1use std::{
85 any::Any,
86 cell::RefCell,
87 fmt::Debug,
88 rc::{Rc, Weak},
89 time::Duration,
90};
91
92use anyhow::Context;
93use nautilus_common::{
94 actor::{self, Actor, DataActor, DataActorNative},
95 cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
96 clients::ExecutionClient,
97 component::Component,
98 enums::{Environment, LogColor},
99 live::{dispatch::DispatchMessage, dst},
100 log_info,
101 messages::{
102 DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
103 data::DataCommand,
104 execution::TradingCommand,
105 system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
106 },
107 msgbus::{self, BusMessage, MessagingSwitchboard},
108 runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
109};
110use nautilus_core::{
111 UUID4,
112 datetime::{mins_to_secs, secs_to_nanos_unchecked},
113};
114#[cfg(test)]
115use nautilus_model::reports::OrderStatusReport;
116use nautilus_model::{
117 events::OrderEventAny,
118 identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
119 orders::Order,
120};
121use nautilus_network::mode::ReconnectRequestOutcome;
122#[cfg(feature = "python")]
123use nautilus_system::trader::Trader;
124use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
125use nautilus_trading::{
126 ExecutionAlgorithm, ExecutionAlgorithmNative,
127 strategy::{Strategy, StrategyNative},
128};
129use tabled::{builder::Builder, settings::Style};
130
131use crate::{
132 dispatch::drain_callbacks,
133 execution::{
134 client::LiveExecutionClient,
135 manager::{
136 ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery, TargetedOrderReportResult,
137 },
138 },
139 runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
140 socket::{SocketReconnectLookup, SocketReconnectRegistry},
141};
142
143pub mod builder;
144pub mod config;
145
146#[cfg(feature = "plugin")]
147pub mod plugin;
148
149mod metrics;
150mod queue;
151mod reconciliation;
152mod state;
153
154use builder::ExternalMessageBusIngress;
155pub use builder::LiveNodeBuilder;
156use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
157pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
158use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
159use queue::{QueueMonitor, QueueStateTransition};
160use reconciliation::{
161 OpenOrderReportResult, OpenOrderReportTask, PositionReportTask, PositionReportTaskResult,
162 ReconciliationCheckIntervals, ReconciliationCheckState, ReportTaskOutcome,
163 TargetedOrderReportTask,
164};
165use state::{EngineConnectionStatus, RunningTransition};
166pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
167
168const DISPATCHES_PER_YIELD: usize = 64;
174
175thread_local! {
176 static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
177}
178
179type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
180
181struct StreamProcessor(Box<StreamProcessorCallback>);
182
183impl Debug for StreamProcessor {
184 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
185 f.debug_struct(stringify!(StreamProcessor)).finish()
186 }
187}
188
189#[derive(Debug)]
198pub struct LiveNode {
199 kernel: NautilusKernel,
200 runner: Option<AsyncRunner>,
201 config: LiveNodeConfig,
202 handle: LiveNodeHandle,
203 exec_manager: ExecutionManager,
204 exec_clients: Vec<LiveExecutionClient>,
205 socket_registry: SocketReconnectRegistry,
206 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
207 external_msgbus: Option<ExternalMessageBusIngress>,
208 stream_processors: Vec<StreamProcessor>,
209 shutdown_deadline: Option<dst::time::Instant>,
210 #[cfg(feature = "plugin")]
211 plugins: plugin::NodePlugins,
212 _thread_owner: Rc<()>,
213}
214
215impl LiveNode {
216 #[must_use]
220 #[allow(
221 clippy::too_many_arguments,
222 reason = "builder components have distinct lifecycle roles"
223 )]
224 pub(crate) fn new_from_builder(
225 kernel: NautilusKernel,
226 runner: AsyncRunner,
227 config: LiveNodeConfig,
228 handle: LiveNodeHandle,
229 mut exec_manager: ExecutionManager,
230 exec_clients: Vec<LiveExecutionClient>,
231 socket_registry: SocketReconnectRegistry,
232 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
233 external_msgbus: Option<ExternalMessageBusIngress>,
234 thread_owner: Rc<()>,
235 ) -> Self {
236 for client in &exec_clients {
237 exec_manager.register_submission_retention(client);
238 }
239
240 Self {
241 kernel,
242 runner: Some(runner),
243 config,
244 handle,
245 exec_manager,
246 exec_clients,
247 socket_registry,
248 cache_database_factory,
249 external_msgbus,
250 stream_processors: Vec::new(),
251 shutdown_deadline: None,
252 #[cfg(feature = "plugin")]
253 plugins: plugin::NodePlugins,
254 _thread_owner: thread_owner,
255 }
256 }
257
258 pub fn builder(
264 trader_id: TraderId,
265 environment: Environment,
266 ) -> anyhow::Result<LiveNodeBuilder> {
267 LiveNodeBuilder::new(trader_id, environment)
268 }
269
270 pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
281 let thread_owner = Self::acquire_thread()?;
282 let config = config.unwrap_or_default();
283 validate_live_environment(config.environment())?;
284
285 config.validate_runtime_support()?;
286
287 if config.event_store.is_some() {
288 anyhow::bail!(
289 "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
290 use LiveNodeBuilder::with_event_store(...) instead"
291 );
292 }
293
294 let handle = LiveNodeHandle::new();
295 let runner = AsyncRunner::new();
296 runner.bind_senders_for_node(handle.clone());
297
298 let kernel = NautilusKernel::new(name, config.clone())?;
299 #[cfg(feature = "python")]
300 if let Some(controller) = config.controller.as_ref() {
301 Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
302 }
303
304 #[cfg(not(feature = "python"))]
305 if let Some(controller) = config.controller.as_ref() {
306 anyhow::bail!(
307 "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
308 controller.controller_path
309 );
310 }
311
312 let exec_manager_config =
313 ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
314
315 let exec_manager = ExecutionManager::new(
316 kernel.clock.clone(),
317 kernel.cache.clone(),
318 exec_manager_config,
319 )?;
320
321 let node = Self {
322 kernel,
323 runner: Some(runner),
324 config,
325 handle,
326 exec_manager,
327 exec_clients: Vec::new(),
328 socket_registry: SocketReconnectRegistry::default(),
329 cache_database_factory: None,
330 external_msgbus: None,
331 stream_processors: Vec::new(),
332 shutdown_deadline: None,
333 #[cfg(feature = "plugin")]
334 plugins: plugin::NodePlugins,
335 _thread_owner: thread_owner,
336 };
337
338 node.load_configured_plugins()?;
339
340 log::info!("LiveNode built successfully with kernel config");
341
342 Ok(node)
343 }
344
345 fn acquire_thread() -> anyhow::Result<Rc<()>> {
346 NODE_THREAD_OWNER.with(|slot| {
347 let mut owner = slot.borrow_mut();
348 anyhow::ensure!(
349 owner.upgrade().is_none(),
350 "A LiveNode already exists or is being built on this thread; drop it before building another"
351 );
352 let token = Rc::new(());
353 *owner = Rc::downgrade(&token);
354 Ok(token)
355 })
356 }
357
358 pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
364 if self.config.plugins.is_empty() {
365 return Ok(());
366 }
367
368 anyhow::bail!(
369 "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
370 )
371 }
372
373 #[expect(
379 clippy::needless_pass_by_value,
380 reason = "signature mirrors the host-enabled API"
381 )]
382 pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
383 #[cfg(feature = "plugin")]
384 config.validate_runtime_support(self.config.plugins.len())?;
385 #[cfg(not(feature = "plugin"))]
386 let _ = config;
387
388 anyhow::bail!(
389 "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
390 )
391 }
392
393 #[must_use]
395 pub fn handle(&self) -> LiveNodeHandle {
396 self.handle.clone()
397 }
398
399 pub fn add_stream_processor<F>(&mut self, callback: F)
408 where
409 F: Fn(&dyn Any) + 'static,
410 {
411 self.add_stream_processor_with_mapping(move |message, _| {
412 callback(message);
413 Ok(())
414 });
415 }
416
417 pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
418 where
419 F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
420 {
421 self.stream_processors
422 .push(StreamProcessor(Box::new(callback)));
423 }
424
425 pub async fn start(&mut self) -> anyhow::Result<()> {
436 if self.state().is_running() {
437 anyhow::bail!("Already running");
438 }
439
440 if self.external_msgbus.is_some() {
441 log::warn!(
442 "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
443 );
444 }
445
446 self.prepare_cache().await?;
447
448 if let Some(runner) = self.runner.as_ref() {
449 runner.bind_senders_for_node(self.handle.clone());
450 }
451
452 self.handle.set_starting();
453
454 self.kernel.reset_shutdown_flag();
455 self.kernel.start_async().await;
456
457 if self.kernel.is_event_store_replay() {
458 log::info!(
459 "Event-store replay loaded; skipping live client connection and reconciliation",
460 );
461
462 if !self.finish_startup_replay().await? {
463 return Ok(());
464 }
465
466 return Ok(());
467 }
468
469 if self.kernel.is_event_store_replay_configured() {
470 self.abort_startup("Event-store replay did not start")
471 .await?;
472 return Ok(());
473 }
474
475 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
476
477 if let Err(e) = self.connect_data_phase(connection_deadline).await {
479 return self
480 .abort_startup_with_error("Data client connection timed out", e)
481 .await;
482 }
483
484 let (startup_system_events, startup_system_commands) =
485 if let Some(runner) = self.runner.as_mut() {
486 runner.flush_pending_data();
487 (
488 runner.drain_pending_system_events(),
489 runner.drain_pending_system_commands(),
490 )
491 } else {
492 (Vec::new(), Vec::new())
493 };
494
495 if let Err(e) = self.connect_exec_clients(connection_deadline).await {
496 return self
497 .abort_startup_with_error("Execution client connection timed out", e)
498 .await;
499 }
500
501 if let Some(reason) = self.startup_abort_reason() {
502 self.abort_startup(reason).await?;
503 return Ok(());
504 }
505
506 match self.await_engines_connected(connection_deadline).await {
507 EngineConnectionStatus::Connected => {}
508 EngineConnectionStatus::TimedOut => {
509 return self
510 .abort_startup_with_error(
511 "Engine readiness timed out",
512 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
513 )
514 .await;
515 }
516 EngineConnectionStatus::StopRequested => {
517 self.abort_startup("Stop signal received during startup")
518 .await?;
519 return Ok(());
520 }
521 EngineConnectionStatus::ShutdownRequested => {
522 self.abort_startup("Shutdown signal received during startup")
523 .await?;
524 return Ok(());
525 }
526 }
527
528 if let Err(e) = self.perform_startup_reconciliation().await {
529 if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
530 anyhow::bail!(
531 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
532 );
533 }
534
535 return Err(e);
536 }
537
538 if let Some(reason) = self.startup_abort_reason() {
539 self.abort_startup(reason).await?;
540 return Ok(());
541 }
542
543 if let Err(e) = self.kernel.start_trader() {
544 return self.abort_after_trader_start_failure(e).await;
545 }
546
547 #[cfg(feature = "plugin")]
548 if let Err(e) = self.plugins.start_controllers() {
549 return self.abort_after_trader_start_failure(e).await;
550 }
551
552 self.process_system_events(startup_system_events);
553 self.process_system_commands(startup_system_commands);
554
555 if !self.finish_startup_trader(None).await? {
556 return Ok(());
557 }
558
559 Ok(())
560 }
561
562 pub async fn stop(&mut self) -> anyhow::Result<()> {
572 if !self.state().is_running() {
573 anyhow::bail!("Not running");
574 }
575
576 self.handle.set_shutting_down();
577
578 #[cfg(feature = "plugin")]
579 let controller_stop_result = self.plugins.stop_controllers();
580 #[cfg(not(feature = "plugin"))]
581 let controller_stop_result: anyhow::Result<()> = Ok(());
582
583 self.kernel.stop_trader();
584 let delay = self.kernel.delay_post_stop();
585 log::info!("Awaiting residual events ({delay:?})...");
586
587 let residual_events = self.process_runner_for(delay).await;
588 if residual_events > 0 {
589 log::debug!("Processed {residual_events} residual events during shutdown");
590 }
591
592 let stop_result = self.finalize_stop().await;
593 let drained_events = self.drain_runner_pending();
594 if drained_events > 0 {
595 log::info!("Drained {drained_events} remaining events during shutdown");
596 }
597
598 match (controller_stop_result, stop_result) {
599 (Ok(()), Ok(())) => Ok(()),
600 (Err(controller_err), Ok(())) => Err(controller_err),
601 (Ok(()), Err(stop_err)) => Err(stop_err),
602 (Err(controller_err), Err(stop_err)) => {
603 log::error!("Error stopping plug-in controllers: {controller_err}");
604 Err(stop_err)
605 }
606 }
607 }
608
609 pub fn dispose(&mut self) {
614 self.close_external_ingress();
615 self.handle.set_stopped();
616 self.kernel.dispose();
617 drop(self.runner.take());
618
619 if let Some(e) = actor::callback_failure() {
620 log::error!("Callback dispatch failed before disposal cleanup: {e}");
621 }
622
623 if let Err(e) = actor::clear_callbacks() {
624 log::error!("Failed to clear callback dispatch during disposal: {e}");
625 }
626 }
627
628 async fn process_runner_for(&mut self, duration: Duration) -> usize {
629 let Some(mut runner) = self.runner.take() else {
630 dst::time::sleep(duration).await;
631 return 0;
632 };
633
634 runner.bind_senders_for_node(self.handle.clone());
635 let deadline = dst::time::Instant::now() + duration;
636 let mut processed = 0;
637
638 loop {
639 tokio::select! {
640 biased;
641
642 () = dst::time::sleep_until(deadline) => break,
643 event = runner.recv() => {
644 let Some(event) = event else {
645 dst::time::sleep_until(deadline).await;
646 break;
647 };
648
649 self.process_runner_event(event);
650 processed += 1;
651 }
652 }
653 }
654
655 self.runner = Some(runner);
656 processed
657 }
658
659 fn drain_runner_pending(&mut self) -> usize {
660 let Some(mut runner) = self.runner.take() else {
661 return 0;
662 };
663
664 let processed = runner.poll_pending(|event| self.process_runner_event(event));
665 self.runner = Some(runner);
666 processed
667 }
668
669 fn process_runner_event(&mut self, event: PendingRunnerEvent) {
670 match event {
671 PendingRunnerEvent::TimeEvent(message) => {
672 let _ = AsyncRunner::handle_time_event(message);
673 }
674 PendingRunnerEvent::SystemEvent(event) => {
675 event.dispatch(|event| self.process_system_event(event));
676 }
677 PendingRunnerEvent::SystemCommand(command) => {
678 command.dispatch(|command| self.process_system_command(command));
679 }
680 PendingRunnerEvent::ExecEvent(event) => {
681 event.dispatch(|event| self.process_exec_event(event));
682 }
683 PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
684 PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
685 PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
686 }
687 }
688
689 fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
690 for event in events {
691 event.dispatch(|event| self.process_system_event(event));
692 }
693 }
694
695 fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
696 for command in commands {
697 command.dispatch(|command| self.process_system_command(command));
698 }
699 }
700
701 fn process_system_command(&self, command: SystemCommand) {
702 match command {
703 SystemCommand::ReconnectSocket(command) => {
704 self.process_socket_reconnect(command);
705 }
706 }
707 }
708
709 fn process_socket_reconnect(&self, command: ReconnectSocket) {
710 let outcome = if command.trader_id == self.config.trader_id {
711 Self::request_socket_reconnect(
712 self.socket_registry
713 .get(command.client_id, command.endpoint),
714 )
715 } else {
716 SocketReconnectDispatchOutcome::InvalidTrader
717 };
718
719 if outcome == SocketReconnectDispatchOutcome::Accepted {
720 log::info!(
721 "Requested socket reconnect for client {} endpoint {}",
722 command.client_id,
723 command.endpoint
724 );
725 } else {
726 log::warn!(
727 "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
728 command.client_id,
729 command.endpoint
730 );
731 }
732 }
733
734 fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
735 match lookup {
736 SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
737 ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
738 ReconnectRequestOutcome::AlreadyReconnecting => {
739 SocketReconnectDispatchOutcome::AlreadyReconnecting
740 }
741 ReconnectRequestOutcome::Disconnected => {
742 SocketReconnectDispatchOutcome::Disconnected
743 }
744 ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
745 ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
746 },
747 SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
748 SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
749 SocketReconnectLookup::EndpointNotFound => {
750 SocketReconnectDispatchOutcome::UnknownEndpoint
751 }
752 SocketReconnectLookup::AmbiguousEndpoint => {
753 SocketReconnectDispatchOutcome::AmbiguousEndpoint
754 }
755 }
756 }
757
758 fn process_system_event(&self, event: SystemEvent) {
759 match event {
760 SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
761 }
762 }
763
764 fn publish_socket_state_change(&self, change: SocketStateChange) {
765 let timestamp = self.kernel.generate_timestamp_ns();
766
767 let event = SocketStateChanged::new(
768 self.config.trader_id,
769 change.client_id,
770 change.venue,
771 change.endpoint,
772 change.state,
773 UUID4::new(),
774 timestamp,
775 timestamp,
776 );
777
778 msgbus::publish_any(
779 MessagingSwitchboard::socket_state_changed_topic(
780 event.client_id,
781 event.endpoint.as_str(),
782 ),
783 event.as_any(),
784 );
785 }
786
787 async fn await_engines_connected(
791 &self,
792 deadline: dst::time::Instant,
793 ) -> EngineConnectionStatus {
794 log::info!(
795 "Awaiting engine connections ({:?} timeout)...",
796 self.config.timeout_connection
797 );
798
799 let interval = Duration::from_millis(100);
800
801 loop {
802 if self.handle.should_stop() {
803 log::warn!("Stop signal received, aborting connection wait");
804 return EngineConnectionStatus::StopRequested;
805 }
806
807 if self.kernel.is_shutdown_requested() {
808 log::warn!("Shutdown signal received, aborting connection wait");
809 return EngineConnectionStatus::ShutdownRequested;
810 }
811
812 if self.kernel.check_engines_connected() {
813 log::info!("All engine clients connected");
814 return EngineConnectionStatus::Connected;
815 }
816
817 let now = dst::time::Instant::now();
818 if now >= deadline {
819 break;
820 }
821
822 dst::time::sleep(interval.min(deadline - now)).await;
823 }
824
825 self.log_connection_status();
826 EngineConnectionStatus::TimedOut
827 }
828
829 async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
833 log::info!(
834 "Awaiting engine disconnections ({:?} timeout)...",
835 self.config.timeout_disconnection
836 );
837
838 let timeout = self.config.timeout_disconnection;
839 let interval = Duration::from_millis(100);
840
841 loop {
842 if self.kernel.check_engines_disconnected() {
843 log::info!("All engine clients disconnected");
844 return Ok(());
845 }
846
847 let now = dst::time::Instant::now();
848 if now >= deadline {
849 break;
850 }
851
852 dst::time::sleep(interval.min(deadline - now)).await;
853 }
854
855 log::error!(
856 "Timed out ({:?}) waiting for engines to disconnect\n\
857 DataEngine.check_disconnected() == {}\n\
858 ExecEngine.check_disconnected() == {}",
859 timeout,
860 self.kernel.data_engine().check_disconnected(),
861 self.kernel.exec_engine().borrow().check_disconnected(),
862 );
863 anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
864 }
865
866 fn log_connection_status(&self) {
867 let data_status = self.kernel.data_client_connection_status();
868 let exec_status = self.kernel.exec_client_connection_status();
869
870 let mut rows: Vec<ClientStatus> = Vec::new();
871
872 for (client_id, connected) in data_status {
873 rows.push(ClientStatus {
874 client: client_id.to_string(),
875 client_type: "Data",
876 connected,
877 });
878 }
879
880 for (client_id, connected) in exec_status {
881 rows.push(ClientStatus {
882 client: client_id.to_string(),
883 client_type: "Execution",
884 connected,
885 });
886 }
887
888 let table = render_client_statuses(rows);
889
890 log::warn!(
891 "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
892 DataEngine.check_connected() == {}\n\
893 ExecEngine.check_connected() == {}",
894 self.config.timeout_connection,
895 self.kernel.data_engine().check_connected(),
896 self.kernel.exec_engine().borrow().check_connected(),
897 );
898 }
899
900 #[expect(clippy::await_holding_refcell_ref)] async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
910 if !self.config.exec_engine.reconciliation {
911 log::info!("Startup reconciliation disabled");
912 self.kernel
913 .portfolio
914 .borrow_mut()
915 .initialize_wallet_orders()?;
916 return Ok(());
917 }
918
919 log_info!(
920 "Starting execution state reconciliation...",
921 color = LogColor::Blue
922 );
923
924 let lookback_mins = self
925 .config
926 .exec_engine
927 .reconciliation_lookback_mins
928 .map(u64::from);
929
930 let timeout = self.config.timeout_reconciliation;
931 let start = dst::time::Instant::now();
932 let client_ids = self.kernel.exec_engine.borrow().client_ids();
933
934 for client_id in client_ids {
935 let elapsed = start.elapsed();
936 if elapsed >= timeout {
937 anyhow::bail!("Startup reconciliation timeout reached");
938 }
939
940 let remaining = timeout
941 .checked_sub(elapsed)
942 .expect("elapsed checked against reconciliation timeout");
943
944 log_info!(
945 "Requesting mass status from {}...",
946 client_id,
947 color = LogColor::Blue
948 );
949
950 let mass_status_result = dst::time::timeout(remaining, async {
951 self.kernel
952 .exec_engine
953 .borrow_mut()
954 .generate_mass_status(&client_id, lookback_mins)
955 .await
956 })
957 .await
958 .map_err(|_| {
959 anyhow::anyhow!(
960 "Startup reconciliation timeout reached while requesting mass status from {client_id}"
961 )
962 })?;
963
964 match mass_status_result {
965 Ok(Some(mass_status)) => {
966 log_info!(
967 "Reconciling ExecutionMassStatus for {}",
968 client_id,
969 color = LogColor::Blue
970 );
971
972 let result = self
973 .exec_manager
974 .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
975
976 anyhow::ensure!(
977 self.kernel
978 .exec_engine
979 .borrow()
980 .get_client(&client_id)
981 .is_some(),
982 "Execution client {client_id} disappeared during startup reconciliation",
983 );
984
985 anyhow::ensure!(
986 result.unresolved_positions.is_empty(),
987 "Unresolved positions during startup reconciliation for {client_id}: {}",
988 result.unresolved_positions.join("; "),
989 );
990
991 if result.events.is_empty() {
992 log_info!(
993 "Reconciliation for {} succeeded",
994 client_id,
995 color = LogColor::Blue
996 );
997 } else {
998 log::info!(
999 color = LogColor::Blue as u8;
1000 "Reconciliation for {} processed {} events",
1001 client_id,
1002 result.events.len()
1003 );
1004 }
1005
1006 if !result.external_orders.is_empty() {
1008 let exec_engine = self.kernel.exec_engine.borrow();
1009
1010 let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1011 anyhow::anyhow!(
1012 "Execution client {client_id} disappeared during startup reconciliation"
1013 )
1014 })?;
1015
1016 for external in result.external_orders {
1017 source_client.register_external_order(
1018 external.client_order_id,
1019 external.venue_order_id,
1020 external.instrument_id,
1021 external.strategy_id,
1022 external.ts_init,
1023 );
1024 }
1025 }
1026 }
1027 Ok(None) => {
1028 log::warn!(
1029 "No mass status available from {client_id} \
1030 (likely adapter error when generating reports)"
1031 );
1032 }
1033 Err(e) => {
1034 return Err(e).context(format!("Failed to get mass status from {client_id}"));
1035 }
1036 }
1037 }
1038
1039 self.kernel.portfolio.borrow_mut().initialize_orders();
1040 self.kernel.portfolio.borrow_mut().initialize_positions();
1041 self.kernel
1042 .portfolio
1043 .borrow_mut()
1044 .initialize_wallet_orders()?;
1045
1046 let elapsed_secs = start.elapsed().as_secs_f64();
1047 log_info!(
1048 "Startup reconciliation completed in {:.2}s",
1049 elapsed_secs,
1050 color = LogColor::Blue
1051 );
1052
1053 Ok(())
1054 }
1055
1056 pub async fn run(&mut self) -> anyhow::Result<()> {
1079 self.run_with_mode(NodeRunMode::Owned).await
1080 }
1081
1082 pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1093 if self.state().is_running() {
1094 anyhow::bail!("Already running");
1095 }
1096
1097 if self.runner.is_none() {
1098 anyhow::bail!("Runner already consumed - run() called twice");
1099 }
1100
1101 self.prepare_cache().await?;
1102
1103 let Some(runner) = self.runner.take() else {
1104 anyhow::bail!("Runner already consumed - run() called twice");
1105 };
1106
1107 runner.bind_senders_for_node(self.handle.clone());
1108
1109 let AsyncRunnerChannels {
1110 mut time_evt_rx,
1111 mut system_evt_rx,
1112 mut system_cmd_rx,
1113 mut exec_evt_rx,
1114 mut exec_cmd_rx,
1115 mut data_evt_rx,
1116 mut data_cmd_rx,
1117 } = runner.take_channels();
1118
1119 log::info!("Event loop starting");
1120
1121 self.handle.set_starting();
1122 self.kernel.reset_shutdown_flag();
1123 self.kernel.start_async().await;
1124
1125 if self.kernel.is_event_store_replay() {
1126 log::info!(
1127 "Event-store replay loaded; skipping live client connection and reconciliation",
1128 );
1129
1130 if !self.finish_startup_replay().await? {
1131 return Ok(());
1132 }
1133
1134 return Ok(());
1135 }
1136
1137 if self.kernel.is_event_store_replay_configured() {
1138 self.abort_startup("Event-store replay did not start")
1139 .await?;
1140 return Ok(());
1141 }
1142
1143 let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1144 Ok(rx) => rx,
1145 Err(e) => {
1146 let result = self
1147 .abort_startup("External message bus ingress failed to start")
1148 .await;
1149 Self::drain_channels(
1150 &mut time_evt_rx,
1151 &mut system_evt_rx,
1152 &mut system_cmd_rx,
1153 &mut exec_evt_rx,
1154 &mut exec_cmd_rx,
1155 &mut data_evt_rx,
1156 &mut data_cmd_rx,
1157 );
1158 log::info!("Event loop stopped");
1159
1160 if let Err(finalize_err) = result {
1161 anyhow::bail!(
1162 "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1163 );
1164 }
1165
1166 return Err(e);
1167 }
1168 };
1169
1170 let stop_handle = self.handle.clone();
1171 let mut pending = PendingEvents::default();
1172 let mut startup_system_events = Vec::new();
1173 let mut startup_system_commands = Vec::new();
1174 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1175
1176 let data_connect_result = drive_with_event_buffering(
1179 self.connect_data_phase(connection_deadline),
1180 &mut pending,
1181 &mut time_evt_rx,
1182 &mut system_evt_rx,
1183 &mut system_cmd_rx,
1184 &mut exec_evt_rx,
1185 &mut exec_cmd_rx,
1186 &mut data_evt_rx,
1187 &mut data_cmd_rx,
1188 )
1189 .await;
1190
1191 if let Err(e) = data_connect_result {
1192 flush_all_pending(
1193 &mut pending,
1194 &mut time_evt_rx,
1195 &mut system_evt_rx,
1196 &mut system_cmd_rx,
1197 &mut exec_evt_rx,
1198 &mut exec_cmd_rx,
1199 &mut data_evt_rx,
1200 &mut data_cmd_rx,
1201 );
1202 let result = self
1203 .abort_startup_with_error("Data client connection timed out", e)
1204 .await;
1205 Self::drain_channels(
1206 &mut time_evt_rx,
1207 &mut system_evt_rx,
1208 &mut system_cmd_rx,
1209 &mut exec_evt_rx,
1210 &mut exec_cmd_rx,
1211 &mut data_evt_rx,
1212 &mut data_cmd_rx,
1213 );
1214 log::info!("Event loop stopped");
1215 return result;
1216 }
1217
1218 flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1222 startup_system_events.extend(pending.take_system_events());
1223 startup_system_commands.extend(pending.take_system_commands());
1224 debug_assert!(
1225 pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1226 "data must be drained into cache before exec clients connect",
1227 );
1228
1229 let engine_connection_result = drive_with_event_buffering(
1231 self.connect_exec_phase(connection_deadline),
1232 &mut pending,
1233 &mut time_evt_rx,
1234 &mut system_evt_rx,
1235 &mut system_cmd_rx,
1236 &mut exec_evt_rx,
1237 &mut exec_cmd_rx,
1238 &mut data_evt_rx,
1239 &mut data_cmd_rx,
1240 )
1241 .await;
1242
1243 flush_all_pending(
1245 &mut pending,
1246 &mut time_evt_rx,
1247 &mut system_evt_rx,
1248 &mut system_cmd_rx,
1249 &mut exec_evt_rx,
1250 &mut exec_cmd_rx,
1251 &mut data_evt_rx,
1252 &mut data_cmd_rx,
1253 );
1254 startup_system_events.extend(pending.take_system_events());
1255 startup_system_commands.extend(pending.take_system_commands());
1256 debug_assert!(
1257 pending.is_empty(),
1258 "all startup events must be processed before reconciliation",
1259 );
1260
1261 let engine_connection_status = match engine_connection_result {
1262 Ok(status) => status,
1263 Err(e) => {
1264 let result = self
1265 .abort_startup_with_error("Execution client connection timed out", e)
1266 .await;
1267 Self::drain_channels(
1268 &mut time_evt_rx,
1269 &mut system_evt_rx,
1270 &mut system_cmd_rx,
1271 &mut exec_evt_rx,
1272 &mut exec_cmd_rx,
1273 &mut data_evt_rx,
1274 &mut data_cmd_rx,
1275 );
1276 log::info!("Event loop stopped");
1277 return result;
1278 }
1279 };
1280
1281 if engine_connection_status == EngineConnectionStatus::TimedOut {
1282 let result = self
1283 .abort_startup_with_error(
1284 "Engine readiness timed out",
1285 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1286 )
1287 .await;
1288 Self::drain_channels(
1289 &mut time_evt_rx,
1290 &mut system_evt_rx,
1291 &mut system_cmd_rx,
1292 &mut exec_evt_rx,
1293 &mut exec_cmd_rx,
1294 &mut data_evt_rx,
1295 &mut data_cmd_rx,
1296 );
1297 log::info!("Event loop stopped");
1298 return result;
1299 }
1300
1301 if let Some(reason) = engine_connection_status
1302 .abort_reason()
1303 .or_else(|| self.startup_abort_reason())
1304 {
1305 self.abort_startup(reason).await?;
1306 Self::drain_channels(
1307 &mut time_evt_rx,
1308 &mut system_evt_rx,
1309 &mut system_cmd_rx,
1310 &mut exec_evt_rx,
1311 &mut exec_cmd_rx,
1312 &mut data_evt_rx,
1313 &mut data_cmd_rx,
1314 );
1315 log::info!("Event loop stopped");
1316 return Ok(());
1317 }
1318
1319 debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1320
1321 if let Err(e) = self.perform_startup_reconciliation().await {
1323 let result = self.abort_startup("Startup reconciliation failed").await;
1324 Self::drain_channels(
1325 &mut time_evt_rx,
1326 &mut system_evt_rx,
1327 &mut system_cmd_rx,
1328 &mut exec_evt_rx,
1329 &mut exec_cmd_rx,
1330 &mut data_evt_rx,
1331 &mut data_cmd_rx,
1332 );
1333 log::info!("Event loop stopped");
1334
1335 if let Err(finalize_err) = result {
1336 anyhow::bail!(
1337 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1338 );
1339 }
1340
1341 return Err(e);
1342 }
1343
1344 if let Some(reason) = self.startup_abort_reason() {
1345 let result = self.abort_startup(reason).await;
1346 Self::drain_channels(
1347 &mut time_evt_rx,
1348 &mut system_evt_rx,
1349 &mut system_cmd_rx,
1350 &mut exec_evt_rx,
1351 &mut exec_cmd_rx,
1352 &mut data_evt_rx,
1353 &mut data_cmd_rx,
1354 );
1355 log::info!("Event loop stopped");
1356 return result;
1357 }
1358
1359 if let Err(e) = self.kernel.start_trader() {
1360 let result = self.abort_after_trader_start_failure(e).await;
1361 Self::drain_channels(
1362 &mut time_evt_rx,
1363 &mut system_evt_rx,
1364 &mut system_cmd_rx,
1365 &mut exec_evt_rx,
1366 &mut exec_cmd_rx,
1367 &mut data_evt_rx,
1368 &mut data_cmd_rx,
1369 );
1370 log::info!("Event loop stopped");
1371 return result;
1372 }
1373
1374 #[cfg(feature = "plugin")]
1375 if let Err(e) = self.plugins.start_controllers() {
1376 let result = self.abort_after_trader_start_failure(e).await;
1377 Self::drain_channels(
1378 &mut time_evt_rx,
1379 &mut system_evt_rx,
1380 &mut system_cmd_rx,
1381 &mut exec_evt_rx,
1382 &mut exec_cmd_rx,
1383 &mut data_evt_rx,
1384 &mut data_cmd_rx,
1385 );
1386 log::info!("Event loop stopped");
1387 return result;
1388 }
1389
1390 self.process_system_events(startup_system_events);
1391 self.process_system_commands(startup_system_commands);
1392
1393 let finish_result = {
1394 let mut receivers = RunnerReceivers {
1395 time_evt: &mut time_evt_rx,
1396 system_evt: &mut system_evt_rx,
1397 system_cmd: &mut system_cmd_rx,
1398 data_evt: &mut data_evt_rx,
1399 data_cmd: &mut data_cmd_rx,
1400 exec_evt: &mut exec_evt_rx,
1401 exec_cmd: &mut exec_cmd_rx,
1402 };
1403
1404 self.finish_startup_trader(Some(&mut receivers)).await
1405 };
1406
1407 match finish_result {
1408 Ok(true) => {}
1409 result => {
1410 log::info!("Event loop stopped");
1411 return result.map(|_| ());
1412 }
1413 }
1414
1415 let exec_config = &self.config.exec_engine;
1416 let inflight_interval =
1417 Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1418
1419 let open_interval = exec_config
1420 .open_check_interval_secs
1421 .filter(|&s| s > 0.0)
1422 .map_or(Duration::ZERO, |secs| {
1423 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1424 });
1425
1426 let position_interval = exec_config
1427 .position_check_interval_secs
1428 .filter(|&s| s > 0.0)
1429 .map_or(Duration::ZERO, |secs| {
1430 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1431 });
1432
1433 let has_clients = !self
1434 .kernel
1435 .exec_engine
1436 .borrow()
1437 .get_all_clients()
1438 .is_empty();
1439 let recon_enabled = has_clients
1440 && (!inflight_interval.is_zero()
1441 || !open_interval.is_zero()
1442 || !position_interval.is_zero());
1443
1444 let recon_min_interval = if recon_enabled {
1445 let mut intervals = Vec::new();
1446
1447 if !inflight_interval.is_zero() {
1448 intervals.push(inflight_interval);
1449 }
1450
1451 if !open_interval.is_zero() {
1452 intervals.push(open_interval);
1453 }
1454
1455 if !position_interval.is_zero() {
1456 intervals.push(position_interval);
1457 }
1458
1459 intervals
1460 .into_iter()
1461 .min()
1462 .unwrap_or(Duration::from_secs(1))
1463 } else {
1464 Duration::from_secs(1) };
1466
1467 let startup_delay = if self.config.exec_engine.reconciliation {
1472 Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1473 } else {
1474 Duration::ZERO
1475 };
1476
1477 let recon_start = dst::time::Instant::now() + startup_delay;
1478
1479 let mut last_inflight_check = dst::time::Instant::now();
1480 let mut last_open_check = last_inflight_check;
1481 let mut last_position_check = last_inflight_check;
1482
1483 let far_future = Duration::from_hours(24 * 365 * 100);
1486
1487 let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1488 let dur = opt_dur.unwrap_or(far_future);
1489 (dur, recon_start + dur)
1490 };
1491
1492 let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1493 Some(recon_min_interval)
1494 } else {
1495 None
1496 });
1497
1498 let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1499 exec_config
1500 .purge_closed_orders_interval_mins
1501 .filter(|&m| m > 0)
1502 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1503 );
1504
1505 let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1506 exec_config
1507 .purge_closed_positions_interval_mins
1508 .filter(|&m| m > 0)
1509 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1510 );
1511
1512 let (purge_account_interval, mut purge_account_next) = make_schedule(
1513 exec_config
1514 .purge_account_events_interval_mins
1515 .filter(|&m| m > 0)
1516 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1517 );
1518
1519 let (own_books_interval, mut own_books_next) = make_schedule(
1520 exec_config
1521 .own_books_audit_interval_secs
1522 .filter(|&s| s > 0.0)
1523 .map(Duration::from_secs_f64),
1524 );
1525
1526 let (prune_fills_interval, mut prune_fills_next) =
1527 make_schedule(Some(Duration::from_mins(1)));
1528
1529 let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1530 maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1531
1532 let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1537 stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1538
1539 let mut residual_events = 0usize;
1541 let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1542 let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1543 let mut position_report_task: Option<PositionReportTask> = None;
1544
1545 let owns_signals = mode.owns_signals();
1548
1549 let ctrl_c = async move {
1550 if owns_signals {
1551 dst::signal::ctrl_c().await
1552 } else {
1553 std::future::pending::<std::io::Result<()>>().await
1554 }
1555 };
1556
1557 let terminate = async move {
1558 if owns_signals {
1559 dst::signal::terminate().await
1560 } else {
1561 std::future::pending::<std::io::Result<()>>().await
1562 }
1563 };
1564
1565 tokio::pin!(ctrl_c);
1566 tokio::pin!(terminate);
1567
1568 let metrics = self.handle.metrics.clone();
1569 let metrics_start = dst::time::Instant::now();
1570 metrics.reset();
1571
1572 let mut queue_monitor = self
1573 .config
1574 .queue_monitor
1575 .as_ref()
1576 .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1577 let mut dispatches_since_yield = 0usize;
1578
1579 let dispatch_result = loop {
1580 let callbacks_pending = match drain_callbacks().await {
1581 Ok(pending) => pending,
1582 Err(e) => {
1583 if self.state() == NodeState::Running {
1584 self.initiate_shutdown();
1585 }
1586
1587 log::warn!(
1588 "Skipping residual events and final buffered dispatch after callback failure"
1589 );
1590
1591 break Err(e);
1592 }
1593 };
1594
1595 let shutdown_deadline = self.shutdown_deadline;
1596 let is_shutting_down = self.state() == NodeState::ShuttingDown;
1597 let is_running = self.state() == NodeState::Running;
1598
1599 tokio::select! {
1600 biased;
1601
1602 result = &mut ctrl_c, if is_running => {
1604 match result {
1605 Ok(()) => log::info!("Received SIGINT, shutting down"),
1606 Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1607 }
1608 self.initiate_shutdown();
1609 }
1610 result = &mut terminate, if is_running => {
1611 match result {
1612 Ok(()) => log::info!("Received SIGTERM, shutting down"),
1613 Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1614 }
1615 self.initiate_shutdown();
1616 }
1617 _ = stop_check_timer.tick(), if is_running => {
1618 if stop_handle.should_stop() {
1619 log::info!("Received stop signal from handle");
1620 self.initiate_shutdown();
1621 } else if self.kernel.is_shutdown_requested() {
1622 log::info!("Received ShutdownSystem command, shutting down");
1623 self.initiate_shutdown();
1624 }
1625 }
1626 () = async {
1627 match shutdown_deadline {
1628 Some(deadline) => dst::time::sleep_until(deadline).await,
1629 None => std::future::pending::<()>().await,
1630 }
1631 }, if self.state() == NodeState::ShuttingDown => {
1632 break Ok(());
1633 }
1634 () = std::future::ready(()), if callbacks_pending => {},
1635 result = async {
1636 match open_order_report_task.as_mut() {
1637 Some(task) => task.future.as_mut().await,
1638 None => std::future::pending::<ReportTaskOutcome<OpenOrderReportResult>>().await,
1639 }
1640 }, if open_order_report_task.is_some() => {
1641 let maintenance_start = dst::time::Instant::now();
1642
1643 drop(open_order_report_task.take());
1644
1645 match result {
1646 ReportTaskOutcome::Completed(result) => {
1647 let client_refs = self
1648 .exec_clients
1649 .iter()
1650 .map(|client| client as &dyn ExecutionClient)
1651 .collect::<Vec<_>>();
1652 let reconciliation = self.exec_manager.reconcile_open_order_reports(
1653 &result.check,
1654 result.reports,
1655 &result.queried_clients,
1656 &result.failed_clients,
1657 &client_refs,
1658 );
1659 self.process_reconciliation_events(&reconciliation.events);
1660 if !reconciliation.targeted_queries.is_empty() {
1661 if is_shutting_down {
1662 let planned_client_order_ids = reconciliation
1663 .targeted_queries
1664 .iter()
1665 .map(TargetedOrderQuery::client_order_id)
1666 .collect::<Vec<_>>();
1667 self.cleanup_cancelled_report_tasks(
1668 &planned_client_order_ids,
1669 );
1670 } else {
1671 targeted_order_report_task = Some(
1672 self.start_targeted_order_report_check(
1673 reconciliation.targeted_queries,
1674 ),
1675 );
1676 }
1677 }
1678 }
1679 ReportTaskOutcome::TimedOut => {
1680 self.cleanup_cancelled_report_tasks(&[]);
1681 log::warn!(
1682 "Open-order report collection expired after {:?}",
1683 self.config.timeout_reconciliation,
1684 );
1685 }
1686 }
1687 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1688 }
1689 result = async {
1690 match targeted_order_report_task.as_mut() {
1691 Some(task) => task.future.as_mut().await,
1692 None => std::future::pending::<ReportTaskOutcome<Vec<TargetedOrderReportResult>>>().await,
1693 }
1694 }, if targeted_order_report_task.is_some() => {
1695 let maintenance_start = dst::time::Instant::now();
1696
1697 let planned_client_order_ids = targeted_order_report_task
1698 .as_ref()
1699 .map(|task| task.planned_client_order_ids.clone())
1700 .unwrap_or_default();
1701 drop(targeted_order_report_task.take());
1702
1703 match result {
1704 ReportTaskOutcome::Completed(result) => {
1705 let client_refs = self
1706 .exec_clients
1707 .iter()
1708 .map(|client| client as &dyn ExecutionClient)
1709 .collect::<Vec<_>>();
1710 let events = self
1711 .exec_manager
1712 .reconcile_targeted_order_reports(result, &client_refs);
1713 self.process_reconciliation_events(&events);
1714 }
1715 ReportTaskOutcome::TimedOut => {
1716 self.cleanup_cancelled_report_tasks(&planned_client_order_ids);
1717 log::warn!(
1718 "Targeted order report collection expired after {:?}",
1719 self.config.timeout_reconciliation,
1720 );
1721 }
1722 }
1723 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1724 }
1725 result = async {
1726 match position_report_task.as_mut() {
1727 Some(task) => task.future.as_mut().await,
1728 None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1729 }
1730 }, if position_report_task.is_some() => {
1731 let maintenance_start = dst::time::Instant::now();
1732
1733 drop(position_report_task.take());
1734
1735 match result {
1736 ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1737 if is_shutting_down {
1738 self.cleanup_cancelled_report_tasks(&[]);
1739 } else {
1740 position_report_task = self.handle_position_report_result(result);
1741 }
1742 }
1743 ReportTaskOutcome::Completed(PositionReportTaskResult::Fills(result)) => {
1744 self.handle_position_fill_report_result(result);
1745 }
1746 ReportTaskOutcome::TimedOut => {
1747 self.cleanup_cancelled_report_tasks(&[]);
1748 log::warn!(
1749 "Position report collection expired after {:?}",
1750 self.config.timeout_reconciliation,
1751 );
1752 }
1753 }
1754 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1755 }
1756
1757 _ = maintenance_timer.tick(), if is_running => {
1760 let maintenance_start = dst::time::Instant::now();
1761 metrics.publish_queue_depths(
1762 RunnerChannelQueueDepths::from_receivers(
1763 &time_evt_rx,
1764 &exec_evt_rx,
1765 &exec_cmd_rx,
1766 &data_evt_rx,
1767 &data_cmd_rx,
1768 ),
1769 metrics_start.elapsed(),
1770 );
1771
1772 if let Some(queue_monitor) = queue_monitor.as_mut() {
1773 let transitions = queue_monitor.evaluate(metrics.snapshot());
1774 self.publish_queue_state_transitions(&transitions);
1775 }
1776
1777 let mut now = dst::time::Instant::now();
1778
1779 if recon_enabled && now >= recon_next {
1780 let recon_intervals = ReconciliationCheckIntervals {
1781 inflight: inflight_interval,
1782 open: open_interval,
1783 position: position_interval,
1784 };
1785 let mut recon_state = ReconciliationCheckState {
1786 last_inflight_check: &mut last_inflight_check,
1787 last_open_check: &mut last_open_check,
1788 last_position_check: &mut last_position_check,
1789 open_order_report_task: &mut open_order_report_task,
1790 targeted_order_report_task: &mut targeted_order_report_task,
1791 position_report_task: &mut position_report_task,
1792 };
1793
1794 self.run_reconciliation_checks(
1795 now,
1796 recon_intervals,
1797 &mut recon_state,
1798 );
1799
1800 now = dst::time::Instant::now();
1801 recon_next = now + recon_interval;
1802 }
1803
1804 if now >= purge_orders_next {
1805 self.exec_manager.purge_closed_orders();
1806 purge_orders_next = now + purge_orders_interval;
1807 }
1808
1809 if now >= purge_positions_next {
1810 self.exec_manager.purge_closed_positions();
1811 purge_positions_next = now + purge_positions_interval;
1812 }
1813
1814 if now >= purge_account_next {
1815 self.exec_manager.purge_account_events();
1816 purge_account_next = now + purge_account_interval;
1817 }
1818
1819 if now >= own_books_next {
1820 self.kernel.cache().borrow_mut().audit_own_order_books();
1821 own_books_next = now + own_books_interval;
1822 }
1823
1824 if now >= prune_fills_next {
1825 self.exec_manager.prune_recent_fills_cache(60.0);
1826 self.exec_manager.prune_processed_fills();
1827 self.exec_manager.prune_order_local_activity();
1828 prune_fills_next = now + prune_fills_interval;
1829 }
1830
1831 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1832 }
1833
1834 Some(handler) = time_evt_rx.recv() => {
1839 let dispatch_start = dst::time::Instant::now();
1840 let dispatched = AsyncRunner::handle_time_event(handler);
1841
1842 if dispatched && is_shutting_down {
1843 log::debug!("Residual time event");
1844 residual_events += 1;
1845 }
1846
1847 if dispatched {
1848 record_runner_dispatch(
1849 &metrics,
1850 SystemChannel::TimeEvents,
1851 dispatch_start,
1852 metrics_start,
1853 );
1854 }
1855 }
1856 Some(event) = system_evt_rx.recv() => {
1857 if is_shutting_down {
1858 log::debug!("Residual system event: {event}");
1859 residual_events += 1;
1860 }
1861 event.dispatch(|event| self.process_system_event(event));
1862 }
1863 Some(command) = system_cmd_rx.recv() => {
1864 if is_shutting_down {
1865 log::debug!("Residual system command: {command}");
1866 residual_events += 1;
1867 }
1868 command.dispatch(|command| self.process_system_command(command));
1869 }
1870 Some(evt) = exec_evt_rx.recv() => {
1871 let dispatch_start = dst::time::Instant::now();
1872
1873 if is_shutting_down {
1874 log::debug!("Residual exec event: {evt}");
1875 residual_events += 1;
1876 }
1877
1878 evt.dispatch(|evt| self.process_exec_event(evt));
1879 record_runner_dispatch(
1880 &metrics,
1881 SystemChannel::ExecEvents,
1882 dispatch_start,
1883 metrics_start,
1884 );
1885 }
1886 Some(cmd) = exec_cmd_rx.recv() => {
1887 let dispatch_start = dst::time::Instant::now();
1888
1889 if is_shutting_down {
1890 log::debug!("Residual exec command: {cmd}");
1891 residual_events += 1;
1892 }
1893
1894 self.process_exec_command(cmd);
1895 record_runner_dispatch(
1896 &metrics,
1897 SystemChannel::ExecCommands,
1898 dispatch_start,
1899 metrics_start,
1900 );
1901 }
1902 message = recv_external_msgbus_message(&mut external_msgbus_rx) => {
1903 let external_msgbus_start = dst::time::Instant::now();
1904
1905 match message {
1906 Some(message) => {
1907 if is_shutting_down {
1908 log::debug!("Residual external message bus message: {message}");
1909 residual_events += 1;
1910 }
1911 self.process_external_msgbus_message(&message);
1912 }
1913 None => {
1914 log::info!("External message bus ingress closed");
1915 external_msgbus_rx = None;
1916 self.close_external_ingress();
1917 }
1918 }
1919
1920 record_runner_external_msgbus(
1921 &metrics,
1922 external_msgbus_start,
1923 metrics_start,
1924 );
1925 }
1926 Some(evt) = data_evt_rx.recv() => {
1927 let dispatch_start = dst::time::Instant::now();
1928
1929 if is_shutting_down {
1930 log::debug!("Residual data event: {evt:?}");
1931 residual_events += 1;
1932 }
1933 AsyncRunner::dispatch_data_event(evt);
1934 record_runner_dispatch(
1935 &metrics,
1936 SystemChannel::DataEvents,
1937 dispatch_start,
1938 metrics_start,
1939 );
1940 }
1941 Some(cmd) = data_cmd_rx.recv() => {
1942 let dispatch_start = dst::time::Instant::now();
1943
1944 if is_shutting_down {
1945 log::debug!("Residual data command: {cmd:?}");
1946 residual_events += 1;
1947 }
1948 AsyncRunner::handle_data_command(cmd);
1949 record_runner_dispatch(
1950 &metrics,
1951 SystemChannel::DataCommands,
1952 dispatch_start,
1953 metrics_start,
1954 );
1955 }
1956 }
1957
1958 dispatches_since_yield += 1;
1959 if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1960 dispatches_since_yield = 0;
1961 tokio::task::yield_now().await;
1962 }
1963 };
1964
1965 if residual_events > 0 {
1966 log::debug!("Processed {residual_events} residual events during shutdown");
1967 }
1968
1969 self.cancel_report_tasks(
1970 &mut open_order_report_task,
1971 &mut targeted_order_report_task,
1972 &mut position_report_task,
1973 );
1974 drop(external_msgbus_rx.take());
1975 let _ = self.kernel.cache().borrow().check_residuals();
1976
1977 let stop_result = self.finalize_stop().await;
1978
1979 if let Err(e) = dispatch_result {
1980 if let Err(stop_err) = stop_result {
1981 log::error!("Failed to finalize node after callback failure: {stop_err}");
1982 let diagnostic = format!("{e}; {stop_err}");
1984 return Err(anyhow::Error::new(e).context(diagnostic));
1985 }
1986
1987 return Err(e.into());
1988 }
1989
1990 Self::drain_channels(
1992 &mut time_evt_rx,
1993 &mut system_evt_rx,
1994 &mut system_cmd_rx,
1995 &mut exec_evt_rx,
1996 &mut exec_cmd_rx,
1997 &mut data_evt_rx,
1998 &mut data_cmd_rx,
1999 );
2000
2001 log::info!("Event loop stopped");
2002
2003 stop_result
2004 }
2005
2006 fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2007 for transition in transitions {
2008 let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2009 let timestamp = self.kernel.generate_timestamp_ns();
2010
2011 let event = QueueStateChanged::new(
2012 self.config.trader_id,
2013 transition.channel,
2014 transition.condition,
2015 transition.state,
2016 transition.queue_depth,
2017 transition.mean_dispatch_ns,
2018 UUID4::new(),
2019 timestamp,
2020 timestamp,
2021 );
2022
2023 msgbus::publish_any(topic, event.as_any());
2024 }
2025 }
2026
2027 #[expect(
2028 clippy::await_holding_refcell_ref,
2029 reason = "cache loading is serialized before the single-threaded live node starts"
2030 )]
2031 async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2032 self.install_cache_database().await?;
2033
2034 let cache = self.kernel.cache();
2035 if !cache.borrow().has_backing() {
2036 return Ok(());
2037 }
2038
2039 if self
2040 .config
2041 .cache
2042 .as_ref()
2043 .is_some_and(|config| config.flush_on_start)
2044 {
2045 cache
2046 .borrow_mut()
2047 .flush_db()
2048 .context("Failed to flush persistent cache")?;
2049 return Ok(());
2050 }
2051
2052 if self.config.exec_engine.load_cache {
2053 self.kernel
2054 .exec_engine()
2055 .borrow_mut()
2056 .load_cache()
2057 .await
2058 .context("Failed to load persistent cache")?;
2059 }
2060
2061 Ok(())
2062 }
2063
2064 #[must_use]
2066 pub const fn has_pending_cache_database(&self) -> bool {
2067 self.cache_database_factory.is_some()
2068 }
2069
2070 async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2075 let Some(factory) = self.cache_database_factory.as_ref() else {
2076 return Ok(());
2077 };
2078
2079 let config = self.config.cache.clone().unwrap_or_default();
2080
2081 let database = factory
2084 .create(self.config.trader_id, self.kernel.instance_id, config)
2085 .await
2086 .context("failed to create cache database backing")?;
2087 self.kernel.cache().borrow_mut().set_database(database);
2088 self.cache_database_factory = None;
2089
2090 Ok(())
2091 }
2092
2093 fn take_external_ingress_receiver(
2094 &mut self,
2095 ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2096 let Some(external_ingress) = self.external_msgbus.as_mut() else {
2097 return Ok(None);
2098 };
2099
2100 let receiver = external_ingress.take_receiver()?;
2101 log::info!("External message bus ingress started");
2102 Ok(Some(receiver))
2103 }
2104
2105 fn republish_external_msgbus_message(message: &BusMessage) {
2106 if let Err(e) = msgbus::republish_external_message(message) {
2107 log::error!(
2108 "Failed to republish external message bus topic '{}': {e:#}",
2109 message.topic
2110 );
2111 }
2112 }
2113
2114 fn process_external_msgbus_message(&self, message: &BusMessage) {
2115 if self.stream_processors.is_empty() {
2116 Self::republish_external_msgbus_message(message);
2117 return;
2118 }
2119
2120 let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2121 for processor in &self.stream_processors {
2122 (processor.0)(value, mapping)?;
2123 }
2124
2125 Ok(())
2126 };
2127
2128 if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2129 log::error!(
2130 "Failed to process external message bus topic '{}': {e:#}",
2131 message.topic
2132 );
2133 }
2134 }
2135
2136 fn close_external_ingress(&mut self) {
2137 if let Some(external_ingress) = self.external_msgbus.as_mut()
2138 && !external_ingress.is_closed()
2139 {
2140 external_ingress.close();
2141 }
2142 }
2143
2144 fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2145 if events.is_empty() {
2146 self.publish_submission_recovery_exhaustions();
2147 return;
2148 }
2149
2150 log::info!(
2151 "Processing {} reconciliation event{}",
2152 events.len(),
2153 if events.len() == 1 { "" } else { "s" }
2154 );
2155
2156 for event in events {
2157 self.exec_manager
2158 .record_local_activity(event.client_order_id());
2159 if let OrderEventAny::Filled(fill) = event {
2160 self.exec_manager
2161 .record_position_activity(fill.instrument_id, fill.account_id);
2162 }
2163
2164 self.kernel.exec_engine.borrow_mut().process(event);
2165 self.exec_manager
2166 .confirm_submission_outcome(&event.client_order_id());
2167 if let OrderEventAny::Filled(fill) = event {
2168 self.exec_manager.commit_recent_fill_if_applied(fill);
2169 }
2170 }
2171
2172 self.publish_submission_recovery_exhaustions();
2173 }
2174
2175 fn publish_submission_recovery_exhaustions(&mut self) {
2176 for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2177 log::warn!(
2178 "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2179 diagnostic.client_order_id,
2180 diagnostic.source,
2181 diagnostic.retry_count,
2182 );
2183 msgbus::publish_any(
2184 MessagingSwitchboard::submission_recovery_exhausted_topic(),
2185 &diagnostic,
2186 );
2187 }
2188 }
2189
2190 fn process_exec_event(&mut self, event: ExecutionEvent) {
2191 let clear_closed_orders = matches!(
2192 &event,
2193 ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2194 );
2195 let closed_report_id = match &event {
2196 ExecutionEvent::Report(
2197 ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2198 ) if report.order_status.is_closed() => report.client_order_id,
2199 _ => None,
2200 };
2201 let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2202 return;
2203 };
2204
2205 self.dispatch_exec_event_and_commit_fill(event);
2206
2207 for client_order_id in &order_ids {
2208 self.exec_manager
2209 .confirm_submission_outcome(client_order_id);
2210 let is_closed = self
2211 .kernel
2212 .cache()
2213 .borrow()
2214 .order(client_order_id)
2215 .is_some_and(|order| order.is_closed());
2216
2217 if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2218 self.exec_manager
2219 .clear_recon_tracking(client_order_id, true);
2220 }
2221 }
2222 }
2223
2224 fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2225 message.dispatch_trading(|message| {
2226 if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2227 self.observe_exec_command_before_dispatch(message.command());
2228 }
2229 });
2230 }
2231
2232 fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2241 let recent_fill_candidate = match &evt {
2242 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2243 _ => None,
2244 };
2245
2246 AsyncRunner::handle_exec_event(evt);
2247
2248 if let Some(fill) = &recent_fill_candidate {
2249 self.exec_manager.commit_recent_fill_if_applied(fill);
2250 }
2251 }
2252
2253 async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2254 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2260 dst::time::timeout(remaining, self.kernel.connect_data_clients())
2261 .await
2262 .map_err(|_| anyhow::anyhow!("data-connect timeout"))
2263 }
2264
2265 async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2266 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2267 dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2268 .await
2269 .map_err(|_| anyhow::anyhow!("exec-connect timeout"))
2270 }
2271
2272 async fn connect_exec_phase(
2277 &mut self,
2278 deadline: dst::time::Instant,
2279 ) -> anyhow::Result<EngineConnectionStatus> {
2280 self.connect_exec_clients(deadline).await?;
2281 Ok(self.await_engines_connected(deadline).await)
2282 }
2283
2284 fn startup_abort_reason(&self) -> Option<&'static str> {
2285 if self.handle.should_stop() {
2286 Some("Stop signal received during startup")
2287 } else if self.kernel.is_shutdown_requested() {
2288 Some("Shutdown signal received during startup")
2289 } else {
2290 None
2291 }
2292 }
2293
2294 async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2295 match self.handle.try_set_running() {
2296 RunningTransition::Entered => Ok(true),
2297 RunningTransition::StopRequested => {
2298 self.abort_startup("Stop signal received during startup")
2299 .await?;
2300 Ok(false)
2301 }
2302 RunningTransition::Invalid(control) => {
2303 self.abort_startup_with_error(
2304 "Invalid lifecycle state during startup",
2305 anyhow::anyhow!(
2306 "Invalid LiveNode control state {control:#04x} while entering Running"
2307 ),
2308 )
2309 .await?;
2310 Ok(false)
2311 }
2312 }
2313 }
2314
2315 async fn finish_startup_trader(
2316 &mut self,
2317 receivers: Option<&mut RunnerReceivers<'_>>,
2318 ) -> anyhow::Result<bool> {
2319 match self.handle.try_set_running() {
2320 RunningTransition::Entered => Ok(true),
2321 RunningTransition::StopRequested => {
2322 self.abort_started_trader("Stop signal received during startup", receivers)
2323 .await?;
2324 Ok(false)
2325 }
2326 RunningTransition::Invalid(control) => {
2327 let state_err = anyhow::anyhow!(
2328 "Invalid LiveNode control state {control:#04x} while entering Running"
2329 );
2330
2331 match self
2332 .abort_started_trader("Invalid lifecycle state during startup", receivers)
2333 .await
2334 {
2335 Ok(()) => Err(state_err),
2336 Err(finalize_err) => {
2337 anyhow::bail!(
2338 "{state_err}; failed to finalize startup abort: {finalize_err}"
2339 )
2340 }
2341 }
2342 }
2343 }
2344 }
2345
2346 async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2347 log::info!("{reason}, aborting startup");
2348 self.handle.set_shutting_down();
2349 self.finalize_stop().await
2350 }
2351
2352 async fn abort_startup_with_error(
2353 &mut self,
2354 reason: &str,
2355 startup_err: anyhow::Error,
2356 ) -> anyhow::Result<()> {
2357 match self.abort_startup(reason).await {
2358 Ok(()) => Err(startup_err),
2359 Err(finalize_err) => {
2360 anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2361 }
2362 }
2363 }
2364
2365 async fn abort_started_trader(
2366 &mut self,
2367 reason: &str,
2368 mut receivers: Option<&mut RunnerReceivers<'_>>,
2369 ) -> anyhow::Result<()> {
2370 log::info!("{reason}, aborting startup");
2371 self.handle.set_shutting_down();
2372
2373 #[cfg(feature = "plugin")]
2374 let controller_stop_result = self.plugins.stop_controllers();
2375 #[cfg(not(feature = "plugin"))]
2376 let controller_stop_result: anyhow::Result<()> = Ok(());
2377
2378 let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2379 let delay = self.kernel.delay_post_stop();
2380 log::info!("Awaiting residual events ({delay:?})...");
2381
2382 let residual_events = match receivers.as_mut() {
2383 Some(receivers) => self.process_receivers_for(delay, receivers).await,
2384 None => self.process_runner_for(delay).await,
2385 };
2386
2387 if residual_events > 0 {
2388 log::debug!("Processed {residual_events} residual events during shutdown");
2389 }
2390
2391 let finalize_result = self.finalize_stop().await;
2392
2393 if let Some(receivers) = receivers {
2394 Self::drain_channels(
2395 receivers.time_evt,
2396 receivers.system_evt,
2397 receivers.system_cmd,
2398 receivers.exec_evt,
2399 receivers.exec_cmd,
2400 receivers.data_evt,
2401 receivers.data_cmd,
2402 );
2403 } else {
2404 let drained_events = self.drain_runner_pending();
2405 if drained_events > 0 {
2406 log::info!("Drained {drained_events} remaining events during shutdown");
2407 }
2408 }
2409
2410 let mut errors = Vec::new();
2411
2412 if let Err(e) = controller_stop_result {
2413 errors.push(format!("Failed to stop plug-in controllers: {e}"));
2414 }
2415
2416 if let Err(e) = trader_stop_result {
2417 errors.push(format!("Failed to stop trader: {e}"));
2418 }
2419
2420 if let Err(e) = finalize_result {
2421 errors.push(format!("Failed to finalize startup abort: {e}"));
2422 }
2423
2424 if errors.is_empty() {
2425 Ok(())
2426 } else {
2427 anyhow::bail!("{}", errors.join("; "))
2428 }
2429 }
2430
2431 async fn process_receivers_for(
2432 &mut self,
2433 duration: Duration,
2434 receivers: &mut RunnerReceivers<'_>,
2435 ) -> usize {
2436 let deadline = dst::time::Instant::now() + duration;
2437 let mut processed = 0;
2438
2439 loop {
2440 tokio::select! {
2441 biased;
2442
2443 () = dst::time::sleep_until(deadline) => break,
2444 Some(message) = receivers.time_evt.recv() => {
2445 let _ = AsyncRunner::handle_time_event(message);
2446 processed += 1;
2447 }
2448 Some(event) = receivers.system_evt.recv() => {
2449 event.dispatch(|event| self.process_system_event(event));
2450 processed += 1;
2451 }
2452 Some(command) = receivers.system_cmd.recv() => {
2453 command.dispatch(|command| self.process_system_command(command));
2454 processed += 1;
2455 }
2456 Some(event) = receivers.exec_evt.recv() => {
2457 event.dispatch(|event| self.process_exec_event(event));
2458 processed += 1;
2459 }
2460 Some(command) = receivers.exec_cmd.recv() => {
2461 self.process_exec_command(command);
2462 processed += 1;
2463 }
2464 Some(event) = receivers.data_evt.recv() => {
2465 AsyncRunner::dispatch_data_event(event);
2466 processed += 1;
2467 }
2468 Some(command) = receivers.data_cmd.recv() => {
2469 AsyncRunner::handle_data_command(command);
2470 processed += 1;
2471 }
2472 }
2473 }
2474
2475 processed
2476 }
2477
2478 async fn abort_after_trader_start_failure(
2479 &mut self,
2480 start_err: anyhow::Error,
2481 ) -> anyhow::Result<()> {
2482 log::info!("Trader startup failed, aborting startup");
2483 self.handle.set_shutting_down();
2484 let stop_result = self.kernel.stop_trader_after_start_failure();
2485 let finalize_result = self.finalize_stop().await;
2486
2487 match (stop_result, finalize_result) {
2488 (Ok(()), Ok(())) => Err(start_err),
2489 (Err(stop_err), Ok(())) => anyhow::bail!(
2490 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2491 {stop_err}"
2492 ),
2493 (Ok(()), Err(finalize_err)) => anyhow::bail!(
2494 "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2495 {finalize_err}"
2496 ),
2497 (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2498 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2499 {stop_err}; failed to finalize startup abort: {finalize_err}"
2500 ),
2501 }
2502 }
2503
2504 fn initiate_shutdown(&mut self) {
2505 #[cfg(feature = "plugin")]
2506 if let Err(e) = self.plugins.stop_controllers() {
2507 log::error!("Error stopping plug-in controllers: {e}");
2508 }
2509
2510 self.kernel.stop_trader();
2511 let delay = self.kernel.delay_post_stop();
2512 log::info!("Awaiting residual events ({delay:?})...");
2513
2514 self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2515 self.handle.set_shutting_down();
2516 }
2517
2518 async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2519 let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2522 self.close_external_ingress();
2523
2524 let timeout = self.config.timeout_disconnection;
2525 let deadline = dst::time::Instant::now() + timeout;
2526
2527 let mut errors = Vec::new();
2528 let (report_result, disconnect_result) = futures_util::join!(
2529 dst::time::timeout(
2530 timeout,
2531 Self::finish_report_tasks(&self.exec_clients, &mut errors)
2532 ),
2533 dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2534 );
2535
2536 if report_result.is_err() {
2537 errors.push("report collection shutdown timeout".to_string());
2538 }
2539
2540 let disconnect_result = match disconnect_result {
2541 Ok(result) => result,
2542 Err(_) => Err(anyhow::anyhow!(
2543 "disconnect timeout while disconnecting clients"
2544 )),
2545 };
2546
2547 if let Err(ref e) = disconnect_result {
2548 log::error!("Error disconnecting clients: {e}");
2549 }
2550
2551 let readiness_result = self.await_engines_disconnected(deadline).await;
2552 let kernel_result = self.kernel.finalize_stop().await;
2553
2554 self.handle.set_stopped();
2555
2556 if !unresolved_submissions.is_empty() {
2557 errors.push(format!(
2558 "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2559 ));
2560 }
2561
2562 if let Err(e) = disconnect_result {
2563 errors.push(e.to_string());
2564 }
2565
2566 if let Err(e) = readiness_result {
2567 errors.push(format!("failed while awaiting engine disconnection: {e}"));
2568 }
2569
2570 if let Err(e) = kernel_result {
2571 errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2572 }
2573
2574 if errors.is_empty() {
2575 Ok(())
2576 } else {
2577 anyhow::bail!("{}", errors.join("; "))
2578 }
2579 }
2580
2581 fn drain_channels(
2582 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2583 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2584 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2585 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2586 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2587 DispatchMessage<TradingCommandMessage>,
2588 >,
2589 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2590 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2591 ) {
2592 let mut drained = 0;
2593
2594 while let Ok(handler) = time_evt_rx.try_recv() {
2595 let _ = AsyncRunner::handle_time_event(handler);
2596 drained += 1;
2597 }
2598
2599 while system_evt_rx.try_recv().is_ok() {
2600 drained += 1;
2601 }
2602
2603 while system_cmd_rx.try_recv().is_ok() {
2604 drained += 1;
2605 }
2606
2607 while let Ok(evt) = data_evt_rx.try_recv() {
2608 AsyncRunner::dispatch_data_event(evt);
2609 drained += 1;
2610 }
2611
2612 while let Ok(cmd) = data_cmd_rx.try_recv() {
2613 AsyncRunner::handle_data_command(cmd);
2614 drained += 1;
2615 }
2616
2617 while let Ok(evt) = exec_evt_rx.try_recv() {
2618 AsyncRunner::dispatch_exec_event(evt);
2619 drained += 1;
2620 }
2621
2622 while let Ok(cmd) = exec_cmd_rx.try_recv() {
2623 AsyncRunner::handle_trading_command(cmd);
2624 drained += 1;
2625 }
2626
2627 if drained > 0 {
2628 log::info!("Drained {drained} remaining events during shutdown");
2629 }
2630 }
2631
2632 fn observe_exec_event_before_dispatch(
2633 &mut self,
2634 evt: &ExecutionEvent,
2635 ) -> Option<Vec<ClientOrderId>> {
2636 let mut order_ids = Vec::new();
2637
2638 match evt {
2639 ExecutionEvent::Order(order_evt) => {
2640 self.exec_manager.observe_order_event(order_evt);
2641 order_ids.push(order_evt.client_order_id());
2642 }
2643 ExecutionEvent::OrderSubmittedBatch(batch) => {
2644 for submitted in &batch.events {
2645 self.exec_manager
2646 .record_local_activity(submitted.client_order_id);
2647 }
2648 }
2649 ExecutionEvent::OrderAcceptedBatch(batch) => {
2650 for accepted in &batch.events {
2651 self.exec_manager
2652 .observe_order_event(&OrderEventAny::Accepted(*accepted));
2653 order_ids.push(accepted.client_order_id);
2654 }
2655 }
2656 ExecutionEvent::OrderCanceledBatch(batch) => {
2657 for canceled in &batch.events {
2658 self.exec_manager
2659 .observe_order_event(&OrderEventAny::Canceled(*canceled));
2660 order_ids.push(canceled.client_order_id);
2661 }
2662 }
2663 ExecutionEvent::Report(report) => {
2664 if let ExecutionReport::Fill(fill_report) = report
2665 && self.exec_manager.is_fill_recently_processed(
2666 fill_report.account_id,
2667 fill_report.instrument_id,
2668 fill_report.trade_id,
2669 )
2670 {
2671 log::debug!(
2672 "Skipping recently processed fill report: {}",
2673 fill_report.trade_id,
2674 );
2675 return None;
2676 }
2677
2678 self.exec_manager.observe_execution_report(report);
2679
2680 if let Some(client_order_id) = self.order_report_client_order_id(report) {
2681 order_ids.push(client_order_id);
2682 }
2683 }
2684 ExecutionEvent::Account(_) => {}
2685 }
2686
2687 Some(order_ids)
2688 }
2689
2690 fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2691 match report {
2692 ExecutionReport::Order(order_report)
2693 | ExecutionReport::OrderWithFills(order_report, _) => {
2694 order_report.client_order_id.or_else(|| {
2695 self.kernel
2696 .cache
2697 .borrow()
2698 .client_order_id(&order_report.venue_order_id)
2699 .copied()
2700 })
2701 }
2702 ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2703 self.kernel
2704 .cache
2705 .borrow()
2706 .client_order_id(&fill_report.venue_order_id)
2707 .copied()
2708 }),
2709 _ => None,
2710 }
2711 }
2712
2713 fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2714 let submission_client_id = if matches!(
2715 cmd,
2716 TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2717 ) {
2718 self.kernel
2719 .exec_engine
2720 .borrow()
2721 .find_client_for_command(cmd)
2722 .map(|client| client.client_id())
2723 } else {
2724 None
2725 };
2726
2727 match cmd {
2728 TradingCommand::SubmitOrder(submit) => {
2729 self.exec_manager.register_submission(
2730 &submit.order_init,
2731 submission_client_id.or(submit.client_id),
2732 );
2733 }
2734 TradingCommand::SubmitOrderList(submit) => {
2735 for order_init in &submit.order_inits {
2736 self.exec_manager
2737 .register_submission(order_init, submission_client_id.or(submit.client_id));
2738 }
2739 }
2740 TradingCommand::ModifyOrder(modify) => {
2741 self.exec_manager.register_inflight(modify.client_order_id);
2742 }
2743 TradingCommand::ModifyOrders(modify) => {
2744 for child in &modify.modifies {
2745 self.exec_manager.register_inflight(child.client_order_id);
2746 }
2747 }
2748 TradingCommand::CancelOrder(cancel) => {
2749 self.exec_manager.register_inflight(cancel.client_order_id);
2750 }
2751 TradingCommand::CancelOrders(cancel) => {
2752 for child in &cancel.cancels {
2753 self.exec_manager.register_inflight(child.client_order_id);
2754 }
2755 }
2756 _ => {}
2757 }
2758 }
2759
2760 #[must_use]
2762 pub fn environment(&self) -> Environment {
2763 self.kernel.environment()
2764 }
2765
2766 #[must_use]
2768 pub const fn kernel(&self) -> &NautilusKernel {
2769 &self.kernel
2770 }
2771
2772 #[must_use]
2774 pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2775 &mut self.kernel
2776 }
2777
2778 #[must_use]
2780 pub fn trader_id(&self) -> TraderId {
2781 self.kernel.trader_id()
2782 }
2783
2784 #[must_use]
2786 pub const fn instance_id(&self) -> UUID4 {
2787 self.kernel.instance_id()
2788 }
2789
2790 #[must_use]
2792 pub fn state(&self) -> NodeState {
2793 self.handle.state()
2794 }
2795
2796 #[must_use]
2798 pub fn is_running(&self) -> bool {
2799 self.state().is_running()
2800 }
2801
2802 pub fn set_cache_database(
2812 &mut self,
2813 database: Box<dyn CacheDatabaseAdapter>,
2814 ) -> anyhow::Result<()> {
2815 if self.state() != NodeState::Idle {
2816 anyhow::bail!(
2817 "Cannot set cache database while node is running, set it before running the node"
2818 );
2819 }
2820
2821 self.kernel.cache().borrow_mut().set_database(database);
2822 Ok(())
2823 }
2824
2825 #[must_use]
2827 pub fn exec_manager(&self) -> &ExecutionManager {
2828 &self.exec_manager
2829 }
2830
2831 #[must_use]
2833 pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2834 &mut self.exec_manager
2835 }
2836
2837 pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2850 where
2851 T: DataActor + DataActorNative + Component + Actor + 'static,
2852 {
2853 if self.state() != NodeState::Idle {
2854 anyhow::bail!(
2855 "Cannot add actor while node is running, add actors before running the node"
2856 );
2857 }
2858
2859 self.kernel.trader.borrow_mut().add_actor(actor)
2860 }
2861
2862 pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2874 where
2875 F: FnOnce() -> anyhow::Result<T>,
2876 T: DataActor + DataActorNative + Component + Actor + 'static,
2877 {
2878 if self.state() != NodeState::Idle {
2879 anyhow::bail!(
2880 "Cannot add actor while node is running, add actors before running the node"
2881 );
2882 }
2883
2884 self.kernel
2885 .trader
2886 .borrow_mut()
2887 .add_actor_from_factory(factory)
2888 }
2889
2890 pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2906 where
2907 T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2908 {
2909 if self.state() != NodeState::Idle {
2910 anyhow::bail!(
2911 "Cannot add strategy while node is running, add strategies before running the node"
2912 );
2913 }
2914
2915 let strategy_id = self
2917 .kernel
2918 .trader
2919 .borrow()
2920 .prepare_strategy_for_registration(&mut strategy)?;
2921 let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
2922 let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
2923
2924 if !instrument_ids.is_empty() {
2925 self.register_external_order_claims(strategy_id, &instrument_ids)?;
2926 }
2927
2928 let mut exec_engine = match oms_type
2929 .map(|_| {
2930 self.kernel
2931 .exec_engine
2932 .try_borrow_mut()
2933 .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
2934 })
2935 .transpose()
2936 {
2937 Ok(exec_engine) => exec_engine,
2938 Err(e) => {
2939 if !instrument_ids.is_empty()
2940 && let Err(rollback_error) =
2941 self.rollback_external_order_claims(strategy_id, &instrument_ids)
2942 {
2943 anyhow::bail!(
2944 "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
2945 );
2946 }
2947
2948 return Err(e);
2949 }
2950 };
2951
2952 if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
2953 drop(exec_engine);
2954
2955 if !instrument_ids.is_empty()
2956 && let Err(rollback_error) =
2957 self.rollback_external_order_claims(strategy_id, &instrument_ids)
2958 {
2959 anyhow::bail!(
2960 "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
2961 );
2962 }
2963
2964 return Err(add_error);
2965 }
2966
2967 if let Some(exec_engine) = &mut exec_engine
2968 && let Some(oms_type) = oms_type
2969 {
2970 exec_engine.register_oms_type(strategy_id, oms_type);
2971 }
2972
2973 Ok(())
2974 }
2975
2976 pub fn register_external_order_claims(
2987 &self,
2988 strategy_id: StrategyId,
2989 instrument_ids: &[InstrumentId],
2990 ) -> anyhow::Result<()> {
2991 self.kernel
2992 .cache
2993 .try_borrow_mut()
2994 .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
2995 .register_external_order_claims(strategy_id, instrument_ids)?;
2996
2997 if !instrument_ids.is_empty() {
2998 log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
2999 }
3000
3001 Ok(())
3002 }
3003
3004 pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3014 self.kernel
3015 .cache
3016 .try_borrow_mut()
3017 .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3018 .set_external_order_claims(strategy_id, &[])?;
3019
3020 Ok(())
3021 }
3022
3023 pub(crate) fn rollback_external_order_claims(
3024 &self,
3025 strategy_id: StrategyId,
3026 instrument_ids: &[InstrumentId],
3027 ) -> anyhow::Result<()> {
3028 let mut cache = self
3029 .kernel
3030 .cache
3031 .try_borrow_mut()
3032 .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3033 let retained: Vec<_> = cache
3034 .external_order_claim_instrument_ids(Some(strategy_id))
3035 .into_iter()
3036 .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3037 .collect();
3038 cache.set_external_order_claims(strategy_id, &retained)
3039 }
3040
3041 pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3052 where
3053 T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3054 {
3055 if self.state() != NodeState::Idle {
3056 anyhow::bail!(
3057 "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3058 );
3059 }
3060
3061 self.kernel
3062 .trader
3063 .borrow_mut()
3064 .add_exec_algorithm(exec_algorithm)
3065 }
3066
3067 }
3071
3072#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3073enum SocketReconnectDispatchOutcome {
3074 Accepted,
3075 AlreadyReconnecting,
3076 Disconnected,
3077 Closed,
3078 Unsupported,
3079 InvalidTrader,
3080 UnknownClient,
3081 UnknownEndpoint,
3082 AmbiguousEndpoint,
3083}
3084
3085fn record_runner_dispatch(
3086 metrics: &RunnerMetrics,
3087 channel: SystemChannel,
3088 dispatch_start: dst::time::Instant,
3089 metrics_start: dst::time::Instant,
3090) {
3091 let dispatch_end = dst::time::Instant::now();
3092 metrics.record_dispatch(
3093 channel,
3094 dispatch_end.duration_since(dispatch_start),
3095 dispatch_end.duration_since(metrics_start),
3096 );
3097}
3098
3099fn record_runner_maintenance(
3100 metrics: &RunnerMetrics,
3101 work_start: dst::time::Instant,
3102 metrics_start: dst::time::Instant,
3103) {
3104 let work_end = dst::time::Instant::now();
3105 metrics.record_maintenance(
3106 work_end.duration_since(work_start),
3107 work_end.duration_since(metrics_start),
3108 );
3109}
3110
3111fn record_runner_external_msgbus(
3112 metrics: &RunnerMetrics,
3113 work_start: dst::time::Instant,
3114 metrics_start: dst::time::Instant,
3115) {
3116 let work_end = dst::time::Instant::now();
3117 metrics.record_external_msgbus(
3118 work_end.duration_since(work_start),
3119 work_end.duration_since(metrics_start),
3120 );
3121}
3122
3123async fn recv_external_msgbus_message(
3124 rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3125) -> Option<BusMessage> {
3126 match rx {
3127 Some(rx) => rx.recv().await,
3128 None => std::future::pending::<Option<BusMessage>>().await,
3129 }
3130}
3131
3132struct RunnerReceivers<'a> {
3133 time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3134 system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3135 system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3136 exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3137 exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3138 data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3139 data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3140}
3141
3142fn flush_pending_data(
3149 pending: &mut PendingEvents,
3150 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3151 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3152) {
3153 loop {
3154 let mut progressed = pending.drain_data();
3155
3156 while let Ok(evt) = data_evt_rx.try_recv() {
3157 AsyncRunner::dispatch_data_event(evt);
3158 progressed = true;
3159 }
3160
3161 while let Ok(cmd) = data_cmd_rx.try_recv() {
3162 AsyncRunner::handle_data_command(cmd);
3163 progressed = true;
3164 }
3165
3166 if !progressed {
3167 break;
3168 }
3169 }
3170}
3171
3172#[expect(
3178 clippy::too_many_arguments,
3179 reason = "all runner receivers are drained together"
3180)]
3181fn flush_all_pending(
3182 pending: &mut PendingEvents,
3183 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3184 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3185 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3186 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3187 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3188 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3189 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3190) {
3191 while let Ok(handler) = time_evt_rx.try_recv() {
3193 let _ = AsyncRunner::handle_time_event(handler);
3194 }
3195
3196 while let Ok(event) = system_evt_rx.try_recv() {
3197 pending.system_events.push(event);
3198 }
3199
3200 while let Ok(command) = system_cmd_rx.try_recv() {
3201 pending.system_commands.push(command);
3202 }
3203
3204 while let Ok(evt) = data_evt_rx.try_recv() {
3205 pending.data_evts.push(evt);
3206 }
3207
3208 while let Ok(cmd) = data_cmd_rx.try_recv() {
3209 pending.data_cmds.push(cmd);
3210 }
3211
3212 while let Ok(evt) = exec_evt_rx.try_recv() {
3213 pending.push_exec_event(evt);
3214 }
3215
3216 while let Ok(cmd) = exec_cmd_rx.try_recv() {
3217 pending.exec_cmds.push(cmd);
3218 }
3219
3220 pending.drain();
3221}
3222
3223#[expect(
3228 clippy::too_many_arguments,
3229 reason = "startup buffering owns one future plus the pending state and all runner receivers"
3230)]
3231async fn drive_with_event_buffering<F: std::future::Future>(
3232 future: F,
3233 pending: &mut PendingEvents,
3234 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3235 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3236 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3237 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3238 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3239 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3240 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3241) -> F::Output {
3242 tokio::pin!(future);
3243
3244 loop {
3245 tokio::select! {
3246 biased;
3247
3248 result = &mut future => {
3249 break result;
3250 }
3251 Some(handler) = time_evt_rx.recv() => {
3252 let _ = AsyncRunner::handle_time_event(handler);
3253 }
3254 Some(event) = system_evt_rx.recv() => {
3255 pending.system_events.push(event);
3256 }
3257 Some(command) = system_cmd_rx.recv() => {
3258 pending.system_commands.push(command);
3259 }
3260 Some(evt) = exec_evt_rx.recv() => {
3261 pending.push_exec_event(evt);
3262 }
3263 Some(cmd) = exec_cmd_rx.recv() => {
3264 pending.exec_cmds.push(cmd);
3265 }
3266 Some(evt) = data_evt_rx.recv() => {
3267 pending.data_evts.push(evt);
3268 }
3269 Some(cmd) = data_cmd_rx.recv() => {
3270 pending.data_cmds.push(cmd);
3271 }
3272 }
3273 }
3274}
3275
3276#[derive(Default)]
3277struct PendingEvents {
3278 system_events: Vec<DispatchMessage<SystemEvent>>,
3279 system_commands: Vec<DispatchMessage<SystemCommand>>,
3280 data_evts: Vec<DispatchMessage<DataEvent>>,
3281 data_cmds: Vec<DispatchMessage<DataCommand>>,
3282 exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3283 order_evts: Vec<DispatchMessage<OrderEventAny>>,
3284 exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3285}
3286
3287impl PendingEvents {
3288 fn is_empty(&self) -> bool {
3289 self.system_events.is_empty()
3290 && self.system_commands.is_empty()
3291 && self.data_evts.is_empty()
3292 && self.data_cmds.is_empty()
3293 && self.exec_reports.is_empty()
3294 && self.order_evts.is_empty()
3295 && self.exec_cmds.is_empty()
3296 }
3297
3298 fn drain_data(&mut self) -> bool {
3302 let total = self.data_evts.len() + self.data_cmds.len();
3303
3304 if total > 0 {
3305 log::debug!(
3306 "Draining {total} data events/commands into cache \
3307 (data_evts={}, data_cmds={})",
3308 self.data_evts.len(),
3309 self.data_cmds.len(),
3310 );
3311 }
3312
3313 for evt in self.data_evts.drain(..) {
3314 AsyncRunner::dispatch_data_event(evt);
3315 }
3316
3317 for cmd in self.data_cmds.drain(..) {
3318 AsyncRunner::handle_data_command(cmd);
3319 }
3320
3321 total > 0
3322 }
3323
3324 fn drain(&mut self) {
3326 let total = self.data_evts.len()
3327 + self.data_cmds.len()
3328 + self.exec_reports.len()
3329 + self.order_evts.len()
3330 + self.exec_cmds.len();
3331
3332 if total > 0 {
3333 log::debug!(
3334 "Processing {total} events/commands queued during startup \
3335 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3336 self.data_evts.len(),
3337 self.data_cmds.len(),
3338 self.exec_reports.len(),
3339 self.order_evts.len(),
3340 self.exec_cmds.len()
3341 );
3342 }
3343
3344 for evt in self.data_evts.drain(..) {
3345 AsyncRunner::dispatch_data_event(evt);
3346 }
3347
3348 for cmd in self.data_cmds.drain(..) {
3349 AsyncRunner::handle_data_command(cmd);
3350 }
3351
3352 for report in self.exec_reports.drain(..) {
3353 report
3354 .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3355 }
3356
3357 for evt in self.order_evts.drain(..) {
3358 evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3359 }
3360
3361 for cmd in self.exec_cmds.drain(..) {
3362 AsyncRunner::handle_trading_command(cmd);
3363 }
3364 }
3365
3366 fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3367 let rooted = event.is_rooted();
3368 event.dispatch(|event| {
3369 let owner = std::thread::current().id();
3370
3371 let order = |event| {
3372 if rooted {
3373 DispatchMessage::new(event, owner)
3374 } else {
3375 DispatchMessage::from(event)
3376 }
3377 };
3378
3379 match event {
3382 ExecutionEvent::Account(_) => AsyncRunner::handle_exec_event(event),
3383 ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3384 DispatchMessage::new(report, owner)
3385 } else {
3386 report.into()
3387 }),
3388 ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3389 ExecutionEvent::OrderSubmittedBatch(batch) => {
3390 for event in batch {
3391 self.order_evts.push(order(OrderEventAny::Submitted(event)));
3392 }
3393 }
3394 ExecutionEvent::OrderAcceptedBatch(batch) => {
3395 for event in batch {
3396 self.order_evts.push(order(OrderEventAny::Accepted(event)));
3397 }
3398 }
3399 ExecutionEvent::OrderCanceledBatch(batch) => {
3400 for event in batch {
3401 self.order_evts.push(order(OrderEventAny::Canceled(event)));
3402 }
3403 }
3404 }
3405 });
3406 }
3407
3408 fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3409 std::mem::take(&mut self.system_events)
3410 }
3411
3412 fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3413 std::mem::take(&mut self.system_commands)
3414 }
3415}
3416
3417struct ClientStatus {
3418 client: String,
3419 client_type: &'static str,
3420 connected: bool,
3421}
3422
3423fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3424 let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3425 builder.push_record(["Client", "Type", "Connected"]);
3426
3427 for row in rows {
3428 builder.push_record([
3429 row.client,
3430 row.client_type.to_string(),
3431 row.connected.to_string(),
3432 ]);
3433 }
3434
3435 builder.build().with(Style::rounded()).to_string()
3436}
3437
3438#[cfg(test)]
3439mod tests {
3440 mod shutdown;
3441
3442 use std::{
3443 cell::{Cell, RefCell},
3444 fmt::Debug,
3445 rc::Rc,
3446 sync::{
3447 Arc,
3448 atomic::{AtomicBool, Ordering},
3449 },
3450 };
3451
3452 use bytes::Bytes;
3453 use indexmap::{IndexMap, IndexSet};
3454 use log::{Level, LevelFilter, Log, Metadata, Record};
3455 #[cfg(feature = "python")]
3456 use nautilus_common::runner::{
3457 SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3458 };
3459 use nautilus_common::{
3460 actor::{
3461 self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3462 },
3463 cache::Cache,
3464 clock::{Clock, VirtualClock},
3465 enums::SerializationEncoding,
3466 live::{
3467 runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3468 sender::DispatchSender,
3469 },
3470 logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3471 messages::{
3472 data::{SubscribeCommand, SubscribeQuotes},
3473 execution::{
3474 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3475 GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3476 TradingCommand,
3477 },
3478 system::{
3479 QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3480 },
3481 },
3482 msgbus::{
3483 self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3484 MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3485 MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3486 },
3487 nautilus_actor,
3488 runner::{SyncTradingCommandSender, TradingCommandSender},
3489 testing::wait_until_async,
3490 timer::{TimeEvent, TimeEventCallback},
3491 };
3492 use nautilus_core::{Params, UUID4, UnixNanos};
3493 use nautilus_execution::{
3494 engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3495 reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3496 };
3497 use nautilus_model::{
3498 accounts::{AccountAny, MarginAccount},
3499 data::QuoteTick,
3500 enums::{
3501 AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3502 TimeInForce,
3503 },
3504 events::{
3505 AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3506 order::spec::{
3507 OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3508 },
3509 },
3510 identifiers::{
3511 AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3512 TradeId, TraderId, Venue, VenueOrderId,
3513 },
3514 instruments::{
3515 Instrument, InstrumentAny,
3516 stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3517 },
3518 orders::{
3519 Order, OrderAny, OrderList, OrderTestBuilder,
3520 stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3521 },
3522 reports::{FillReport, PositionStatusReport},
3523 types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3524 };
3525 use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3526 use nautilus_testkit::{
3527 cache::TestCacheDatabaseControl,
3528 components::{StateActor, StateStrategy},
3529 };
3530 use nautilus_trading::{
3531 nautilus_strategy,
3532 strategy::{config::StrategyConfig, core::StrategyCore},
3533 };
3534 use parking_lot::Mutex;
3535 use rstest::*;
3536 use rust_decimal::Decimal;
3537 use rust_decimal_macros::dec;
3538 use ustr::Ustr;
3539
3540 use super::{
3541 reconciliation::{
3542 PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3543 request_position_fill_reports,
3544 },
3545 *,
3546 };
3547 use crate::{
3548 execution::{
3549 client::LiveExecutionClient,
3550 manager::{PositionFillReportQuery, ReportClientCoverage},
3551 submission::{
3552 SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3553 },
3554 },
3555 socket::SocketControl,
3556 };
3557
3558 struct ExternalIngressLogCapture {
3559 messages: Mutex<Vec<String>>,
3560 }
3561
3562 static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3563 messages: Mutex::new(Vec::new()),
3564 };
3565
3566 #[derive(Debug)]
3567 enum FillReportClientOutcome {
3568 Reports(Vec<FillReport>),
3569 Failure,
3570 }
3571
3572 #[derive(Debug)]
3573 struct FillReportClient {
3574 client_id: ClientId,
3575 account_id: AccountId,
3576 venue: Venue,
3577 outcome: FillReportClientOutcome,
3578 commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3579 }
3580
3581 #[async_trait::async_trait(?Send)]
3582 impl ExecutionClient for FillReportClient {
3583 fn is_connected(&self) -> bool {
3584 true
3585 }
3586
3587 fn client_id(&self) -> ClientId {
3588 self.client_id
3589 }
3590
3591 fn account_id(&self) -> AccountId {
3592 self.account_id
3593 }
3594
3595 fn venue(&self) -> Venue {
3596 self.venue
3597 }
3598
3599 fn oms_type(&self) -> OmsType {
3600 OmsType::Netting
3601 }
3602
3603 fn get_account(&self) -> Option<AccountAny> {
3604 None
3605 }
3606
3607 fn generate_account_state(
3608 &self,
3609 _balances: Vec<AccountBalance>,
3610 _margins: Vec<MarginBalance>,
3611 _reported: bool,
3612 _ts_event: UnixNanos,
3613 _info: Option<Params>,
3614 ) -> anyhow::Result<()> {
3615 Ok(())
3616 }
3617
3618 fn start(&mut self) -> anyhow::Result<()> {
3619 Ok(())
3620 }
3621
3622 fn stop(&mut self) -> anyhow::Result<()> {
3623 Ok(())
3624 }
3625
3626 async fn generate_order_status_reports(
3627 &self,
3628 _cmd: &GenerateOrderStatusReports,
3629 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3630 match self.outcome {
3631 FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3632 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3633 }
3634 }
3635
3636 async fn generate_position_status_reports(
3637 &self,
3638 _cmd: &GeneratePositionStatusReports,
3639 ) -> anyhow::Result<Vec<PositionStatusReport>> {
3640 match self.outcome {
3641 FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3642 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3643 }
3644 }
3645
3646 async fn generate_fill_reports(
3647 &self,
3648 cmd: GenerateFillReports,
3649 ) -> anyhow::Result<Vec<FillReport>> {
3650 self.commands.borrow_mut().push(cmd);
3651
3652 match &self.outcome {
3653 FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3654 FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3655 }
3656 }
3657 }
3658
3659 #[derive(Debug)]
3660 struct OrderReportClient {
3661 client_id: ClientId,
3662 report: OrderStatusReport,
3663 targeted: bool,
3664 queries: Rc<RefCell<Vec<&'static str>>>,
3665 }
3666
3667 #[async_trait::async_trait(?Send)]
3668 impl ExecutionClient for OrderReportClient {
3669 fn is_connected(&self) -> bool {
3670 true
3671 }
3672
3673 fn client_id(&self) -> ClientId {
3674 self.client_id
3675 }
3676
3677 fn account_id(&self) -> AccountId {
3678 self.report.account_id
3679 }
3680
3681 fn venue(&self) -> Venue {
3682 self.report.instrument_id.venue
3683 }
3684
3685 fn oms_type(&self) -> OmsType {
3686 OmsType::Netting
3687 }
3688
3689 fn get_account(&self) -> Option<AccountAny> {
3690 None
3691 }
3692
3693 fn generate_account_state(
3694 &self,
3695 _balances: Vec<AccountBalance>,
3696 _margins: Vec<MarginBalance>,
3697 _reported: bool,
3698 _ts_event: UnixNanos,
3699 _info: Option<Params>,
3700 ) -> anyhow::Result<()> {
3701 Ok(())
3702 }
3703
3704 fn start(&mut self) -> anyhow::Result<()> {
3705 Ok(())
3706 }
3707
3708 fn stop(&mut self) -> anyhow::Result<()> {
3709 Ok(())
3710 }
3711
3712 async fn generate_order_status_reports(
3713 &self,
3714 _command: &GenerateOrderStatusReports,
3715 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3716 self.queries.borrow_mut().push("bulk");
3717 Ok(if self.targeted {
3718 Vec::new()
3719 } else {
3720 vec![self.report.clone()]
3721 })
3722 }
3723
3724 async fn generate_order_status_report(
3725 &self,
3726 _command: &GenerateOrderStatusReport,
3727 ) -> anyhow::Result<Option<OrderStatusReport>> {
3728 self.queries.borrow_mut().push("targeted");
3729 Ok(Some(self.report.clone()))
3730 }
3731 }
3732
3733 #[derive(Debug)]
3734 struct StartupSocketActor {
3735 core: DataActorCore,
3736 received: Rc<RefCell<Vec<SocketStateChanged>>>,
3737 }
3738
3739 impl StartupSocketActor {
3740 fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3741 Self {
3742 core: DataActorCore::new(DataActorConfig {
3743 actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3744 ..Default::default()
3745 }),
3746 received,
3747 }
3748 }
3749 }
3750
3751 impl DataActor for StartupSocketActor {
3752 fn on_start(&mut self) -> anyhow::Result<()> {
3753 self.subscribe_socket_state(None, None, None);
3754 Ok(())
3755 }
3756
3757 fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3758 self.received.borrow_mut().push(event.clone());
3759 Ok(())
3760 }
3761 }
3762
3763 nautilus_actor!(StartupSocketActor);
3764
3765 impl Log for ExternalIngressLogCapture {
3766 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3767 metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3768 }
3769
3770 fn log(&self, record: &Record<'_>) {
3771 if self.enabled(record.metadata()) {
3772 self.messages.lock().push(record.args().to_string());
3773 }
3774 }
3775
3776 fn flush(&self) {}
3777 }
3778
3779 #[derive(Debug)]
3780 struct FailingTimerActor {
3781 core: DataActorCore,
3782 received: Rc<RefCell<Vec<u64>>>,
3783 }
3784
3785 nautilus_actor!(FailingTimerActor);
3786
3787 impl DataActor for FailingTimerActor {
3788 fn on_start(&mut self) -> anyhow::Result<()> {
3789 for timestamp in [17, 23] {
3790 let received = self.received.clone();
3791
3792 let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3793 received.borrow_mut().push(timestamp);
3794
3795 if timestamp == 17 {
3796 crate::dispatch::tests::latch_callback_failure();
3797 }
3798 }));
3799
3800 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3801 TimeEvent::new(
3802 "callback-failure".into(),
3803 UUID4::new(),
3804 timestamp.into(),
3805 timestamp.into(),
3806 ),
3807 callback,
3808 ));
3809 }
3810
3811 Ok(())
3812 }
3813 }
3814
3815 #[rstest]
3816 #[tokio::test]
3817 async fn test_callback_failure_stops_later_live_events(
3818 #[values(false, true)] incomplete_submission: bool,
3819 ) {
3820 actor::clear_callbacks().unwrap();
3821
3822 let config = LiveNodeConfig {
3823 exec_engine: crate::config::LiveExecutionEngineConfig {
3824 reconciliation: false,
3825 submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3826 ..Default::default()
3827 },
3828 timeout_connection: Duration::ZERO,
3829 timeout_reconciliation: Duration::ZERO,
3830 timeout_portfolio: Duration::ZERO,
3831 timeout_disconnection: Duration::ZERO,
3832 delay_post_stop: Duration::ZERO,
3833 timeout_shutdown: Duration::ZERO,
3834 ..Default::default()
3835 };
3836
3837 let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3838
3839 if incomplete_submission {
3840 let order = OrderTestBuilder::new(OrderType::Market)
3841 .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3842 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3843 .quantity(Quantity::from("1"))
3844 .build();
3845 node.exec_manager
3846 .register_submission(order.init_event(), None);
3847 }
3848 let received = Rc::new(RefCell::new(Vec::new()));
3849 node.add_actor(FailingTimerActor {
3850 core: DataActorCore::new(DataActorConfig {
3851 actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3852 ..Default::default()
3853 }),
3854 received: received.clone(),
3855 })
3856 .unwrap();
3857
3858 let result = node.run_with_mode(NodeRunMode::Hosted).await;
3859
3860 let error = result.unwrap_err();
3861 assert!(error.to_string().contains("Callback delivery unwound"));
3862 if incomplete_submission {
3863 assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3864 }
3865 let state = node.state();
3866 let trader_stopped = node.kernel.trader.borrow().is_stopped();
3867 let failure = actor::callback_failure();
3868 node.dispose();
3869
3870 assert_eq!(
3871 error.downcast_ref::<CallbackDispatchError>(),
3872 Some(&CallbackDispatchError::DeliveryUnwound)
3873 );
3874 assert_eq!(*received.borrow(), [17]);
3875 assert_eq!(state, NodeState::Stopped);
3876 assert!(trader_stopped);
3877 assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
3878 assert_eq!(actor::callback_failure(), None);
3879 assert_eq!(actor::clear_callbacks(), Ok(()));
3880 }
3881
3882 #[rstest]
3883 fn test_render_client_statuses() {
3884 let rows = vec![
3885 ClientStatus {
3886 client: "BINANCE".to_string(),
3887 client_type: "Data",
3888 connected: true,
3889 },
3890 ClientStatus {
3891 client: "SIM".to_string(),
3892 client_type: "Execution",
3893 connected: false,
3894 },
3895 ];
3896
3897 let output = render_client_statuses(rows);
3898 let expected = "â•─────────┬───────────┬───────────╮\n\
3899│ Client │ Type │ Connected │\n\
3900├─────────┼───────────┼───────────┤\n\
3901│ BINANCE │ Data │ true │\n\
3902│ SIM │ Execution │ false │\n\
3903╰─────────┴───────────┴───────────╯";
3904
3905 assert_eq!(output, expected);
3906 }
3907
3908 #[rstest]
3909 fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
3910 log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
3911 log::set_max_level(LevelFilter::Error);
3912 EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
3913 let message = BusMessage::with_str_topic(
3914 "data.quotes.AUDUSD.SIM*",
3915 BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
3916 Bytes::new(),
3917 SerializationEncoding::Json,
3918 );
3919
3920 LiveNode::republish_external_msgbus_message(&message);
3921
3922 assert_eq!(
3923 *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
3924 vec![
3925 "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
3926 external message topic: Topic `value` contained invalid characters, was \
3927 data.quotes.AUDUSD.SIM*"
3928 .to_string()
3929 ],
3930 );
3931 }
3932
3933 #[rstest]
3934 #[case(None, 2)]
3935 #[case(Some(SystemChannel::DataEvents), 1)]
3936 #[case(Some(SystemChannel::ExecCommands), 1)]
3937 fn test_publish_queue_state_transitions_reaches_typed_subscriber(
3938 #[case] channel: Option<SystemChannel>,
3939 #[case] expected_count: usize,
3940 ) {
3941 let config = LiveNodeConfig {
3942 trader_id: TraderId::from("QUEUE-001"),
3943 exec_engine: crate::config::LiveExecutionEngineConfig {
3944 reconciliation: false,
3945 ..Default::default()
3946 },
3947 ..Default::default()
3948 };
3949
3950 let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
3951 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
3952
3953 let handler = ShareableMessageHandler::from_typed({
3954 let received = received.clone();
3955 move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
3956 });
3957
3958 msgbus::subscribe_any(
3959 MessagingSwitchboard::queue_state_changed_pattern(channel),
3960 handler,
3961 None,
3962 );
3963
3964 let transitions = [
3965 QueueStateTransition {
3966 channel: SystemChannel::DataEvents,
3967 condition: QueueCondition::Backlogged,
3968 state: QueueState::Triggered,
3969 queue_depth: 17,
3970 mean_dispatch_ns: 23,
3971 },
3972 QueueStateTransition {
3973 channel: SystemChannel::ExecCommands,
3974 condition: QueueCondition::Slow,
3975 state: QueueState::Triggered,
3976 queue_depth: 17,
3977 mean_dispatch_ns: 23,
3978 },
3979 ];
3980
3981 node.publish_queue_state_transitions(&transitions);
3982
3983 let events = received.borrow();
3984 assert_eq!(events.len(), expected_count);
3985
3986 let expected = transitions
3987 .into_iter()
3988 .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
3989
3990 for (event, transition) in events.iter().zip(expected) {
3991 assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
3992 assert_eq!(event.channel, transition.channel);
3993 assert_eq!(event.condition, transition.condition);
3994 assert_eq!(event.state, transition.state);
3995 assert_eq!(event.queue_depth, transition.queue_depth);
3996 assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
3997 assert_ne!(event.event_id, UUID4::default());
3998 assert_ne!(event.ts_event, UnixNanos::default());
3999 assert_eq!(event.ts_init, event.ts_event);
4000 }
4001
4002 if channel.is_none() {
4003 assert_ne!(events[0].event_id, events[1].event_id);
4004 }
4005
4006 drop(events);
4007 msgbus::get_message_bus().borrow_mut().dispose();
4008 }
4009
4010 #[rstest]
4011 #[case(None, None)]
4012 #[case(Some(ClientId::from("BINANCE")), None)]
4013 #[case(None, Some("binance-futures-market-streams"))]
4014 #[case(
4015 Some(ClientId::from("BINANCE")),
4016 Some("binance-futures-market-streams")
4017 )]
4018 fn test_process_socket_state_change_reaches_typed_subscriber(
4019 #[case] client_id: Option<ClientId>,
4020 #[case] endpoint: Option<&str>,
4021 ) {
4022 let config = LiveNodeConfig {
4023 trader_id: TraderId::from("SOCKET-001"),
4024 exec_engine: crate::config::LiveExecutionEngineConfig {
4025 reconciliation: false,
4026 ..Default::default()
4027 },
4028 ..Default::default()
4029 };
4030
4031 let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4032 let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4033
4034 let handler = ShareableMessageHandler::from_typed({
4035 let received = received.clone();
4036 move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4037 });
4038
4039 msgbus::subscribe_any(
4040 MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4041 handler,
4042 None,
4043 );
4044
4045 let change = SocketStateChange::new(
4046 ClientId::from("BINANCE"),
4047 Some(Venue::from("BINANCE")),
4048 Ustr::from("binance-futures-market-streams"),
4049 SocketState::Disconnected,
4050 );
4051
4052 node.process_system_event(SystemEvent::SocketState(change));
4053
4054 let events = received.borrow();
4055 assert_eq!(events.len(), 1);
4056 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4057 assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4058 assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4059 assert_eq!(
4060 events[0].endpoint,
4061 Ustr::from("binance-futures-market-streams")
4062 );
4063 assert_eq!(events[0].state, SocketState::Disconnected);
4064 assert_ne!(events[0].event_id, UUID4::default());
4065 assert_ne!(events[0].ts_event, UnixNanos::default());
4066 assert_eq!(events[0].ts_init, events[0].ts_event);
4067 drop(events);
4068 msgbus::get_message_bus().borrow_mut().dispose();
4069 }
4070
4071 #[rstest]
4072 #[case::accepted(
4073 ReconnectRequestOutcome::Accepted,
4074 SocketReconnectDispatchOutcome::Accepted
4075 )]
4076 #[case::already_reconnecting(
4077 ReconnectRequestOutcome::AlreadyReconnecting,
4078 SocketReconnectDispatchOutcome::AlreadyReconnecting
4079 )]
4080 #[case::disconnected(
4081 ReconnectRequestOutcome::Disconnected,
4082 SocketReconnectDispatchOutcome::Disconnected
4083 )]
4084 #[case::closed(
4085 ReconnectRequestOutcome::Closed,
4086 SocketReconnectDispatchOutcome::Closed
4087 )]
4088 #[case::unsupported(
4089 ReconnectRequestOutcome::Unsupported,
4090 SocketReconnectDispatchOutcome::Unsupported
4091 )]
4092 fn test_request_socket_reconnect_maps_transport_outcome(
4093 #[case] transport: ReconnectRequestOutcome,
4094 #[case] expected: SocketReconnectDispatchOutcome,
4095 ) {
4096 let registry = SocketReconnectRegistry::default();
4097 let client_id = ClientId::from("TEST");
4098 let endpoint = Ustr::from("test-streams");
4099 let control = SocketControl::with_registry(client_id, None, endpoint, ®istry);
4100 let _sink = control.sink();
4101 control.register(move || transport);
4102
4103 let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4104
4105 assert_eq!(outcome, expected);
4106 }
4107
4108 #[rstest]
4109 #[case::client_not_found(
4110 SocketReconnectLookup::ClientNotFound,
4111 SocketReconnectDispatchOutcome::UnknownClient
4112 )]
4113 #[case::unsupported(
4114 SocketReconnectLookup::Unsupported,
4115 SocketReconnectDispatchOutcome::Unsupported
4116 )]
4117 #[case::endpoint_not_found(
4118 SocketReconnectLookup::EndpointNotFound,
4119 SocketReconnectDispatchOutcome::UnknownEndpoint
4120 )]
4121 #[case::ambiguous(
4122 SocketReconnectLookup::AmbiguousEndpoint,
4123 SocketReconnectDispatchOutcome::AmbiguousEndpoint
4124 )]
4125 fn test_request_socket_reconnect_maps_lookup_failure(
4126 #[case] lookup: SocketReconnectLookup,
4127 #[case] expected: SocketReconnectDispatchOutcome,
4128 ) {
4129 assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4130 }
4131
4132 #[rstest]
4133 fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4134 let trader_id = TraderId::from("SOCKET-001");
4135
4136 let config = LiveNodeConfig {
4137 trader_id,
4138 ..Default::default()
4139 };
4140
4141 let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4142 let client_id = ClientId::from("TEST");
4143 let endpoint = Ustr::from("test-streams");
4144 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4145 let sender = DispatchSender::new(tx);
4146 let events = sender.clone();
4147 msgbus::subscribe_any(
4148 MessagingSwitchboard::socket_state_changed_pattern(
4149 Some(client_id),
4150 Some(endpoint.as_str()),
4151 ),
4152 ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4153 events.send(17u32).unwrap();
4154 }),
4155 None,
4156 );
4157
4158 let control =
4159 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4160 let _sink = control.sink();
4161 control.register(move || {
4162 sender.send(23u32).unwrap();
4163 ReconnectRequestOutcome::Accepted
4164 });
4165
4166 let event = SystemEvent::SocketState(SocketStateChange::new(
4167 client_id,
4168 None,
4169 endpoint,
4170 SocketState::Connected,
4171 ));
4172 let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4173 trader_id,
4174 client_id,
4175 endpoint,
4176 31.into(),
4177 ));
4178
4179 if startup {
4180 let mut pending = PendingEvents::default();
4181 pending.system_events.push(event.into());
4182 pending.system_commands.push(command.into());
4183 node.process_system_events(pending.take_system_events());
4184 node.process_system_commands(pending.take_system_commands());
4185 assert!(pending.is_empty());
4186 } else {
4187 node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4188 node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4189 }
4190
4191 let event_child = rx.try_recv().unwrap();
4192 let command_child = rx.try_recv().unwrap();
4193 assert!(event_child.is_rooted());
4194 assert!(command_child.is_rooted());
4195 assert_eq!(event_child.dispatch(|value| value), 17);
4196 assert_eq!(command_child.dispatch(|value| value), 23);
4197 assert!(rx.is_empty());
4198 msgbus::get_message_bus().borrow_mut().dispose();
4199 }
4200
4201 #[rstest]
4202 fn test_process_socket_reconnect_routes_only_matching_trader() {
4203 let trader_id = TraderId::from("SOCKET-001");
4204
4205 let config = LiveNodeConfig {
4206 trader_id,
4207 exec_engine: crate::config::LiveExecutionEngineConfig {
4208 reconciliation: false,
4209 ..Default::default()
4210 },
4211 ..Default::default()
4212 };
4213
4214 let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4215 let client_id = ClientId::from("TEST");
4216 let endpoint = Ustr::from("test-streams");
4217 let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4218 let request_count = Arc::clone(&requests);
4219 let control =
4220 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4221 let _sink = control.sink();
4222 control.register(move || {
4223 request_count.fetch_add(1, Ordering::SeqCst);
4224 ReconnectRequestOutcome::Accepted
4225 });
4226
4227 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4228 TraderId::from("OTHER-001"),
4229 client_id,
4230 endpoint,
4231 UnixNanos::default(),
4232 )));
4233 assert_eq!(requests.load(Ordering::SeqCst), 0);
4234
4235 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4236 trader_id,
4237 client_id,
4238 endpoint,
4239 UnixNanos::default(),
4240 )));
4241 assert_eq!(requests.load(Ordering::SeqCst), 1);
4242 }
4243
4244 #[rstest]
4245 #[tokio::test]
4246 async fn test_start_publishes_socket_change_after_actor_subscribes() {
4247 let config = LiveNodeConfig {
4248 trader_id: TraderId::from("SOCKET-STARTUP-001"),
4249 exec_engine: crate::config::LiveExecutionEngineConfig {
4250 reconciliation: false,
4251 ..Default::default()
4252 },
4253 timeout_connection: Duration::ZERO,
4254 timeout_reconciliation: Duration::ZERO,
4255 timeout_portfolio: Duration::ZERO,
4256 timeout_disconnection: Duration::ZERO,
4257 delay_post_stop: Duration::ZERO,
4258 timeout_shutdown: Duration::ZERO,
4259 ..Default::default()
4260 };
4261
4262 let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4263 let received = Rc::new(RefCell::new(Vec::new()));
4264 node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4265 .unwrap();
4266 node.runner.as_ref().unwrap().bind_senders();
4267
4268 let change = SocketStateChange::new(
4269 ClientId::from("BINANCE"),
4270 Some(Venue::from("BINANCE")),
4271 Ustr::from("binance-futures-market-streams"),
4272 SocketState::Connected,
4273 );
4274 get_system_event_sender()
4275 .send(SystemEvent::SocketState(change))
4276 .unwrap();
4277
4278 node.start().await.unwrap();
4279
4280 {
4281 let events = received.borrow();
4282 assert_eq!(events.len(), 1);
4283 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4284 assert_eq!(events[0].client_id, change.client_id);
4285 assert_eq!(events[0].venue, change.venue);
4286 assert_eq!(events[0].endpoint, change.endpoint);
4287 assert_eq!(events[0].state, change.state);
4288 assert_ne!(events[0].event_id, UUID4::default());
4289 assert_ne!(events[0].ts_event, UnixNanos::default());
4290 assert_eq!(events[0].ts_init, events[0].ts_event);
4291 }
4292
4293 node.stop().await.unwrap();
4294 node.dispose();
4295 }
4296
4297 #[rstest]
4298 #[tokio::test(flavor = "current_thread")]
4299 async fn test_run_publishes_queue_state_after_dispatch_sample() {
4300 let config = LiveNodeConfig {
4301 trader_id: TraderId::from("QUEUE-RUN-001"),
4302 queue_monitor: Some(crate::config::QueueMonitorConfig {
4303 queue_depth_trigger: usize::MAX,
4304 queue_depth_clear: 0,
4305 mean_dispatch_ns_trigger: 1,
4306 mean_dispatch_ns_clear: 0,
4307 }),
4308 exec_engine: crate::config::LiveExecutionEngineConfig {
4309 reconciliation: false,
4310 ..Default::default()
4311 },
4312 delay_post_stop: Duration::ZERO,
4313 ..Default::default()
4314 };
4315
4316 let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4317 let handle = node.handle();
4318 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4319
4320 let handler = ShareableMessageHandler::from_typed({
4321 let received = received.clone();
4322 let stop_handle = handle.clone();
4323
4324 move |event: &QueueStateChanged| {
4325 received.borrow_mut().push(event.clone());
4326 stop_handle.stop();
4327 }
4328 });
4329
4330 msgbus::subscribe_any(
4331 MessagingSwitchboard::queue_state_changed_pattern(None),
4332 handler,
4333 None,
4334 );
4335 let drive_handle = handle.clone();
4336
4337 let result = tokio::time::timeout(Duration::from_secs(5), async {
4338 let run = node.run();
4339 tokio::pin!(run);
4340
4341 let drive = async move {
4342 wait_until_async(
4343 || async { drive_handle.is_running() },
4344 Duration::from_secs(2),
4345 )
4346 .await;
4347 get_data_event_sender().send(stub_data_event()).unwrap();
4348 };
4349
4350 let (run_result, ()) = tokio::join!(run, drive);
4351 run_result
4352 })
4353 .await;
4354
4355 assert!(
4356 result.is_ok(),
4357 "queue state event should arrive before timeout"
4358 );
4359 assert!(result.unwrap().is_ok(), "run() should succeed");
4360 let events = received.borrow();
4361 assert_eq!(events.len(), 1);
4362 assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4363 assert_eq!(events[0].channel, SystemChannel::DataEvents);
4364 assert_eq!(events[0].condition, QueueCondition::Slow);
4365 assert_eq!(events[0].state, QueueState::Triggered);
4366 assert_eq!(events[0].queue_depth, 0);
4367 assert!(events[0].mean_dispatch_ns > 0);
4368 assert_ne!(events[0].event_id, UUID4::default());
4369 assert_ne!(events[0].ts_event, UnixNanos::default());
4370 assert_eq!(events[0].ts_init, events[0].ts_event);
4371 drop(events);
4372 msgbus::get_message_bus().borrow_mut().dispose();
4373 }
4374
4375 #[rstest]
4376 fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4377 let config = LiveNodeConfig {
4378 exec_engine: crate::config::LiveExecutionEngineConfig {
4379 reconciliation: false,
4380 ..Default::default()
4381 },
4382 ..Default::default()
4383 };
4384
4385 let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4386 let event = stub_exec_event();
4387 let account_id = AccountId::from("TEST-001");
4388 let instrument_id = InstrumentId::from("TEST.VENUE");
4389 let trade_id = TradeId::from("T-001");
4390
4391 let close_ids = node.observe_exec_event_before_dispatch(&event);
4392 assert_eq!(close_ids, Some(Vec::new()));
4393 assert!(
4394 !node
4395 .exec_manager
4396 .is_fill_recently_processed(account_id, instrument_id, trade_id)
4397 );
4398
4399 node.exec_manager
4400 .mark_fill_processed(account_id, instrument_id, trade_id);
4401
4402 let close_ids = node.observe_exec_event_before_dispatch(&event);
4403 assert_eq!(close_ids, None);
4404 }
4405
4406 #[rstest]
4407 #[case(false, false, OrderStatus::Canceled, 1)]
4408 #[case(false, true, OrderStatus::Accepted, 0)]
4409 #[case(true, false, OrderStatus::Canceled, 1)]
4410 #[case(true, true, OrderStatus::Accepted, 0)]
4411 fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4412 #[case] with_fills: bool,
4413 #[case] superseded: bool,
4414 #[case] expected_status: OrderStatus,
4415 #[case] expected_query_count: usize,
4416 ) {
4417 let config = LiveNodeConfig {
4418 exec_engine: crate::config::LiveExecutionEngineConfig {
4419 reconciliation: true,
4420 open_check_threshold_ms: 5_000,
4421 single_order_query_delay_ms: 0,
4422 ..Default::default()
4423 },
4424 ..Default::default()
4425 };
4426
4427 let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4428 let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4429 let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4430 let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4431 let account_id = AccountId::from("TEST-001");
4432 let client_id = ClientId::from("TEST");
4433 let instrument = crypto_perpetual_ethusdt();
4434 let instrument_id = instrument.id();
4435 let account = AccountAny::Margin(MarginAccount::new(
4436 AccountState::new(
4437 account_id,
4438 AccountType::Margin,
4439 vec![AccountBalance::new(
4440 Money::from("1000000 USDT"),
4441 Money::from("0 USDT"),
4442 Money::from("1000000 USDT"),
4443 )],
4444 Vec::new(),
4445 true,
4446 UUID4::new(),
4447 UnixNanos::default(),
4448 UnixNanos::default(),
4449 Some(Currency::USDT()),
4450 ),
4451 true,
4452 ));
4453 node.kernel.cache.borrow_mut().add_account(account).unwrap();
4454 node.kernel
4455 .cache
4456 .borrow_mut()
4457 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4458 .unwrap();
4459 insert_accepted_limit_order_in_node(
4460 &node,
4461 account_id,
4462 client_id,
4463 instrument_id,
4464 client_order_id,
4465 old_venue_order_id,
4466 );
4467
4468 if superseded {
4469 let order = node
4470 .kernel
4471 .cache
4472 .borrow()
4473 .order_owned(&client_order_id)
4474 .unwrap();
4475 let pending_update = OrderPendingUpdateSpec::builder()
4476 .trader_id(order.trader_id())
4477 .strategy_id(order.strategy_id())
4478 .instrument_id(order.instrument_id())
4479 .client_order_id(client_order_id)
4480 .account_id(account_id)
4481 .venue_order_id(old_venue_order_id)
4482 .build();
4483 node.kernel
4484 .cache
4485 .borrow_mut()
4486 .update_order(&OrderEventAny::PendingUpdate(pending_update))
4487 .unwrap();
4488 let order = node
4489 .kernel
4490 .cache
4491 .borrow()
4492 .order_owned(&client_order_id)
4493 .unwrap();
4494 let updated = OrderUpdatedSpec::builder()
4495 .trader_id(order.trader_id())
4496 .strategy_id(order.strategy_id())
4497 .instrument_id(order.instrument_id())
4498 .client_order_id(client_order_id)
4499 .quantity(order.quantity())
4500 .venue_order_id(new_venue_order_id)
4501 .account_id(account_id)
4502 .build();
4503 node.kernel
4504 .cache
4505 .borrow_mut()
4506 .update_order(&OrderEventAny::Updated(updated))
4507 .unwrap();
4508 }
4509
4510 let report = OrderStatusReport::new(
4511 account_id,
4512 instrument_id,
4513 Some(client_order_id),
4514 old_venue_order_id,
4515 OrderSide::Buy.into(),
4516 OrderType::Limit,
4517 TimeInForce::Gtc,
4518 OrderStatus::Canceled,
4519 Quantity::from("10.0"),
4520 Quantity::from("0.0"),
4521 UnixNanos::from(1_000),
4522 UnixNanos::from(2_000),
4523 UnixNanos::from(3_000),
4524 None,
4525 );
4526
4527 let report = if with_fills {
4528 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4529 } else {
4530 ExecutionReport::Order(Box::new(report))
4531 };
4532
4533 let event = ExecutionEvent::Report(report);
4534
4535 node.process_exec_event(event);
4536
4537 let order = node
4538 .kernel
4539 .cache
4540 .borrow()
4541 .order_owned(&client_order_id)
4542 .unwrap();
4543
4544 let expected_venue_order_id = if superseded {
4545 new_venue_order_id
4546 } else {
4547 old_venue_order_id
4548 };
4549
4550 assert_eq!(order.status(), expected_status);
4551 assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4552
4553 if !superseded {
4554 let replacement = OrderTestBuilder::new(OrderType::Limit)
4555 .client_order_id(client_order_id)
4556 .instrument_id(instrument_id)
4557 .quantity(Quantity::from("20.0"))
4558 .price(Price::from("200.0"))
4559 .build();
4560 let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4561 node.kernel
4562 .cache
4563 .borrow_mut()
4564 .add_order(replacement, None, Some(client_id), true)
4565 .unwrap();
4566 let replacement = node
4567 .kernel
4568 .cache
4569 .borrow_mut()
4570 .update_order(&submitted)
4571 .unwrap();
4572 let accepted =
4573 TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4574 node.kernel
4575 .cache
4576 .borrow_mut()
4577 .update_order(&accepted)
4578 .unwrap();
4579 }
4580
4581 assert_eq!(
4582 node.exec_manager.check_open_order_queries().len(),
4583 expected_query_count,
4584 );
4585 }
4586
4587 #[rstest]
4588 fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4589 let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4590
4591 let OrderEventAny::Filled(fill) = &mut fill_event else {
4592 unreachable!();
4593 };
4594
4595 fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4596 fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4597 let fill = fill.clone();
4598 let report_event = fill_report_event(&fill);
4599 let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4600
4601 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4602 let marked_before_dispatch = is_recent_fill(&node, &fill);
4603
4604 node.dispatch_exec_event_and_commit_fill(event);
4605
4606 let marked_after_dispatch = is_recent_fill(&node, &fill);
4607 let later_report_is_eligible = node
4608 .observe_exec_event_before_dispatch(&report_event)
4609 .is_some();
4610 assert_eq!(
4611 (
4612 marked_before_dispatch,
4613 marked_after_dispatch,
4614 later_report_is_eligible,
4615 ),
4616 (false, false, true),
4617 );
4618 }
4619
4620 #[rstest]
4621 fn test_applied_direct_fill_commits_and_skips_later_report() {
4622 let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
4623
4624 let OrderEventAny::Filled(fill) = &fill_event else {
4625 unreachable!();
4626 };
4627
4628 let fill = fill.clone();
4629 let report_event = fill_report_event(&fill);
4630 let event = ExecutionEvent::Order(fill_event);
4631
4632 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4633 assert!(!is_recent_fill(&node, &fill));
4634
4635 node.dispatch_exec_event_and_commit_fill(event);
4636
4637 assert!(is_recent_fill(&node, &fill));
4638 assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
4639 }
4640
4641 #[rstest]
4642 fn test_canonical_duplicate_fill_counts_as_applied() {
4643 let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
4644
4645 let OrderEventAny::Filled(fill) = &fill_event else {
4646 unreachable!();
4647 };
4648
4649 let mut fill = fill.clone();
4650 node.kernel
4651 .cache
4652 .borrow_mut()
4653 .update_order(&fill_event)
4654 .unwrap();
4655 fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
4656
4657 node.exec_manager.commit_recent_fill_if_applied(&fill);
4658
4659 assert!(is_recent_fill(&node, &fill));
4660 }
4661
4662 #[rstest]
4663 #[case(false)]
4664 #[case(true)]
4665 fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
4666 let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
4667 "AppliedContinuousFillNode"
4668 } else {
4669 "RejectedContinuousFillNode"
4670 });
4671
4672 if !applied {
4673 let OrderEventAny::Filled(fill) = &mut fill_event else {
4674 unreachable!();
4675 };
4676
4677 fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
4678 fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
4679 }
4680
4681 let OrderEventAny::Filled(fill) = &fill_event else {
4682 unreachable!();
4683 };
4684
4685 let fill = fill.clone();
4686
4687 node.process_reconciliation_events(&[fill_event]);
4688
4689 assert_eq!(is_recent_fill(&node, &fill), applied);
4690 }
4691
4692 #[rstest]
4693 fn test_recent_fill_commit_requires_account_and_instrument_match() {
4694 let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
4695 node.kernel
4696 .cache
4697 .borrow_mut()
4698 .update_order(&fill_event)
4699 .unwrap();
4700
4701 let OrderEventAny::Filled(fill) = fill_event else {
4702 unreachable!();
4703 };
4704
4705 let mut account_mismatch = fill.clone();
4706 account_mismatch.account_id = AccountId::from("OTHER-001");
4707 let mut instrument_mismatch = fill;
4708 instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
4709
4710 node.exec_manager
4711 .commit_recent_fill_if_applied(&account_mismatch);
4712 node.exec_manager
4713 .commit_recent_fill_if_applied(&instrument_mismatch);
4714
4715 assert!(!is_recent_fill(&node, &account_mismatch));
4716 assert!(!is_recent_fill(&node, &instrument_mismatch));
4717 }
4718
4719 #[rstest]
4720 fn test_applied_inferred_fill_remains_recently_processed() {
4721 let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
4722 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
4723 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
4724 let account_id = AccountId::from("TEST-001");
4725 let order = node
4726 .kernel
4727 .cache
4728 .borrow()
4729 .order_owned(&client_order_id)
4730 .unwrap();
4731 let report = OrderStatusReport::new(
4732 account_id,
4733 instrument.id(),
4734 Some(client_order_id),
4735 venue_order_id,
4736 OrderSide::Buy.into(),
4737 OrderType::Limit,
4738 TimeInForce::Gtc,
4739 OrderStatus::PartiallyFilled,
4740 Quantity::from("10.0"),
4741 Quantity::from("1.0"),
4742 UnixNanos::from(1_000),
4743 UnixNanos::from(1_000),
4744 UnixNanos::from(1_000),
4745 None,
4746 )
4747 .with_avg_px(dec!(100.0));
4748 let inferred = create_inferred_fill_for_qty(
4749 &order,
4750 &report,
4751 &account_id,
4752 &instrument,
4753 Quantity::from("1.0"),
4754 UnixNanos::from(1_000),
4755 None,
4756 )
4757 .unwrap();
4758
4759 let OrderEventAny::Filled(fill) = &inferred else {
4760 unreachable!();
4761 };
4762
4763 let fill = fill.clone();
4764
4765 node.process_reconciliation_events(&[inferred]);
4766
4767 assert!(fill.reconciliation);
4768 assert!(is_recent_fill(&node, &fill));
4769 }
4770
4771 #[rstest]
4772 #[tokio::test]
4773 async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
4774 let client_id = ClientId::from("POSITION-FILLS");
4775 let account_id = AccountId::from("POSITION-FILLS-001");
4776 let instrument_id = crypto_perpetual_ethusdt().id();
4777 let commands = Rc::new(RefCell::new(Vec::new()));
4778
4779 let client = LiveExecutionClient::new(Box::new(FillReportClient {
4780 client_id,
4781 account_id,
4782 venue: instrument_id.venue,
4783 outcome: FillReportClientOutcome::Failure,
4784 commands: commands.clone(),
4785 }));
4786
4787 let command = GenerateFillReports::new(
4788 UUID4::new(),
4789 UnixNanos::from(2_000),
4790 Some(instrument_id),
4791 None,
4792 Some(UnixNanos::from(1_000)),
4793 Some(UnixNanos::from(2_000)),
4794 None,
4795 None,
4796 );
4797
4798 let result = request_position_fill_reports(
4799 vec![client],
4800 vec![PositionFillReportQuery {
4801 key: (instrument_id, account_id),
4802 client_id,
4803 command: command.clone(),
4804 }],
4805 )
4806 .await;
4807
4808 assert_eq!(*commands.borrow(), vec![command]);
4809 assert!(result.successful_keys.is_empty());
4810 assert!(result.reports.is_empty());
4811 }
4812
4813 #[rstest]
4814 #[tokio::test]
4815 async fn test_request_position_fill_reports_accepts_scoped_response() {
4816 let client_id = ClientId::from("POSITION-FILLS");
4817 let account_id = AccountId::from("POSITION-FILLS-001");
4818 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4819 let instrument_id = instrument.id();
4820
4821 let report_b = FillReport::new(
4822 account_id,
4823 instrument_id,
4824 VenueOrderId::from("V-POSITION-FILLS"),
4825 TradeId::from("T-POSITION-FILLS-B"),
4826 OrderSide::Buy,
4827 Quantity::from("1.0"),
4828 Price::from("100.0"),
4829 Money::zero(instrument.quote_currency()),
4830 LiquiditySide::Taker,
4831 Some(ClientOrderId::from("O-POSITION-FILLS")),
4832 None,
4833 UnixNanos::from(1_500),
4834 UnixNanos::from(2_000),
4835 None,
4836 );
4837 let mut report_a = report_b.clone();
4838 report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
4839 let commands = Rc::new(RefCell::new(Vec::new()));
4840
4841 let client = LiveExecutionClient::new(Box::new(FillReportClient {
4842 client_id,
4843 account_id,
4844 venue: instrument_id.venue,
4845 outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
4846 commands,
4847 }));
4848
4849 let command = GenerateFillReports::new(
4850 UUID4::new(),
4851 UnixNanos::from(2_000),
4852 Some(instrument_id),
4853 None,
4854 Some(UnixNanos::from(1_000)),
4855 Some(UnixNanos::from(2_000)),
4856 None,
4857 None,
4858 );
4859 let key = (instrument_id, account_id);
4860
4861 let result = request_position_fill_reports(
4862 vec![client],
4863 vec![PositionFillReportQuery {
4864 key,
4865 client_id,
4866 command,
4867 }],
4868 )
4869 .await;
4870
4871 assert_eq!(result.successful_keys, IndexSet::from([key]));
4872 assert_eq!(
4873 result.reports,
4874 IndexMap::from([(key, vec![report_a, report_b])])
4875 );
4876 }
4877
4878 #[rstest]
4879 #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
4880 #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
4881 #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
4882 #[tokio::test]
4883 async fn test_request_position_fill_reports_rejects_out_of_scope_response(
4884 #[case] report_account: &str,
4885 #[case] report_instrument: &str,
4886 #[case] ts_event: u64,
4887 #[case] quantity: &str,
4888 ) {
4889 let client_id = ClientId::from("POSITION-FILLS");
4890 let account_id = AccountId::from("POSITION-FILLS-001");
4891 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4892 let instrument_id = instrument.id();
4893
4894 let report = FillReport::new(
4895 AccountId::from(report_account),
4896 InstrumentId::from(report_instrument),
4897 VenueOrderId::from("V-POSITION-FILLS"),
4898 TradeId::from("T-POSITION-FILLS"),
4899 OrderSide::Buy,
4900 Quantity::from(quantity),
4901 Price::from("100.0"),
4902 Money::zero(instrument.quote_currency()),
4903 LiquiditySide::Taker,
4904 Some(ClientOrderId::from("O-POSITION-FILLS")),
4905 None,
4906 UnixNanos::from(ts_event),
4907 UnixNanos::from(2_000),
4908 None,
4909 );
4910
4911 let client = LiveExecutionClient::new(Box::new(FillReportClient {
4912 client_id,
4913 account_id,
4914 venue: instrument_id.venue,
4915 outcome: FillReportClientOutcome::Reports(vec![report]),
4916 commands: Rc::new(RefCell::new(Vec::new())),
4917 }));
4918
4919 let command = GenerateFillReports::new(
4920 UUID4::new(),
4921 UnixNanos::from(2_000),
4922 Some(instrument_id),
4923 None,
4924 Some(UnixNanos::from(1_000)),
4925 Some(UnixNanos::from(2_000)),
4926 None,
4927 None,
4928 );
4929 let key = (instrument_id, account_id);
4930
4931 let result = request_position_fill_reports(
4932 vec![client],
4933 vec![PositionFillReportQuery {
4934 key,
4935 client_id,
4936 command,
4937 }],
4938 )
4939 .await;
4940
4941 assert!(result.successful_keys.is_empty());
4942 assert!(result.reports.is_empty());
4943 }
4944
4945 #[rstest]
4946 #[case(999)]
4947 #[case(2_001)]
4948 #[tokio::test]
4949 async fn test_request_position_fill_reports_filters_time_window_superset(
4950 #[case] ts_event: u64,
4951 ) {
4952 let client_id = ClientId::from("POSITION-FILLS");
4953 let account_id = AccountId::from("POSITION-FILLS-001");
4954 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4955 let instrument_id = instrument.id();
4956
4957 let report = FillReport::new(
4958 account_id,
4959 instrument_id,
4960 VenueOrderId::from("V-POSITION-FILLS"),
4961 TradeId::from("T-POSITION-FILLS"),
4962 OrderSide::Buy,
4963 Quantity::from("1.0"),
4964 Price::from("100.0"),
4965 Money::zero(instrument.quote_currency()),
4966 LiquiditySide::Taker,
4967 Some(ClientOrderId::from("O-POSITION-FILLS")),
4968 None,
4969 UnixNanos::from(ts_event),
4970 UnixNanos::from(2_000),
4971 None,
4972 );
4973
4974 let client = LiveExecutionClient::new(Box::new(FillReportClient {
4975 client_id,
4976 account_id,
4977 venue: instrument_id.venue,
4978 outcome: FillReportClientOutcome::Reports(vec![report]),
4979 commands: Rc::new(RefCell::new(Vec::new())),
4980 }));
4981
4982 let command = GenerateFillReports::new(
4983 UUID4::new(),
4984 UnixNanos::from(2_000),
4985 Some(instrument_id),
4986 None,
4987 Some(UnixNanos::from(1_000)),
4988 Some(UnixNanos::from(2_000)),
4989 None,
4990 None,
4991 );
4992 let key = (instrument_id, account_id);
4993
4994 let result = request_position_fill_reports(
4995 vec![client],
4996 vec![PositionFillReportQuery {
4997 key,
4998 client_id,
4999 command,
5000 }],
5001 )
5002 .await;
5003
5004 assert_eq!(result.successful_keys, IndexSet::from([key]));
5005 assert_eq!(result.reports, IndexMap::from([(key, Vec::new())]));
5006 }
5007
5008 #[rstest]
5009 #[case::query_error(false)]
5010 #[case::missing_client(true)]
5011 #[tokio::test]
5012 async fn test_position_fill_collection_discards_partial_failure(
5013 #[case] missing_client: bool,
5014 #[values(false, true)] failure_first: bool,
5015 position_fill_query: (FillReport, PositionFillReportQuery),
5016 ) {
5017 let (report, query) = position_fill_query;
5018 let failed_client_id = ClientId::from("FAILED-FILLS");
5019 let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5020 let mut healthy_report = report.clone();
5021 healthy_report.account_id = healthy_key.1;
5022 let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5023 let mut clients = vec![
5024 LiveExecutionClient::new(Box::new(FillReportClient {
5025 client_id: query.client_id,
5026 account_id: query.key.1,
5027 venue: query.key.0.venue,
5028 outcome: FillReportClientOutcome::Reports(vec![report]),
5029 commands: Rc::new(RefCell::new(Vec::new())),
5030 })),
5031 LiveExecutionClient::new(Box::new(FillReportClient {
5032 client_id: healthy_client_id,
5033 account_id: healthy_key.1,
5034 venue: healthy_key.0.venue,
5035 outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5036 commands: Rc::new(RefCell::new(Vec::new())),
5037 })),
5038 ];
5039
5040 if !missing_client {
5041 clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5042 client_id: failed_client_id,
5043 account_id: query.key.1,
5044 venue: query.key.0.venue,
5045 outcome: FillReportClientOutcome::Failure,
5046 commands: Rc::new(RefCell::new(Vec::new())),
5047 })));
5048 }
5049
5050 let failed_query = PositionFillReportQuery {
5051 client_id: failed_client_id,
5052 key: query.key,
5053 command: query.command.clone(),
5054 };
5055
5056 let healthy_query = PositionFillReportQuery {
5057 key: healthy_key,
5058 client_id: healthy_client_id,
5059 command: query.command.clone(),
5060 };
5061
5062 let mut queries = vec![query, failed_query];
5063
5064 if failure_first {
5065 queries.reverse();
5066 }
5067
5068 queries.push(healthy_query);
5069
5070 let result = request_position_fill_reports(clients, queries).await;
5071
5072 assert_eq!(result.successful_keys, IndexSet::from([healthy_key]));
5073 assert_eq!(
5074 result.reports,
5075 IndexMap::from([(healthy_key, vec![healthy_report])])
5076 );
5077 }
5078
5079 #[rstest]
5080 #[case::same_economics("metadata", true)]
5081 #[case::venue_order("venue_order", false)]
5082 #[case::side("side", false)]
5083 #[case::quantity("quantity", false)]
5084 #[case::price("price", false)]
5085 #[case::commission("commission", false)]
5086 #[case::liquidity("liquidity", false)]
5087 #[case::average("average", false)]
5088 #[case::event_time("event_time", false)]
5089 #[case::client_order("client_order", false)]
5090 #[case::position("position", false)]
5091 #[tokio::test]
5092 async fn test_position_fill_collection_validates_duplicate_economics(
5093 #[case] changed: &str,
5094 #[case] accepted: bool,
5095 position_fill_query: (FillReport, PositionFillReportQuery),
5096 ) {
5097 let (report, query) = position_fill_query;
5098 let mut duplicate = report.clone();
5099
5100 match changed {
5101 "metadata" => {
5102 duplicate.report_id = UUID4::new();
5103 duplicate.ts_init = UnixNanos::from(3_000);
5104 }
5105 "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5106 "side" => duplicate.order_side = OrderSide::Sell,
5107 "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5108 "price" => duplicate.last_px = Price::from("101.0"),
5109 "commission" => duplicate.commission = Money::from("0.25 USDT"),
5110 "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5111 "average" => duplicate.avg_px = Some(dec!(100.5)),
5112 "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5113 "client_order" => duplicate.client_order_id = None,
5114 "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5115 _ => unreachable!(),
5116 }
5117
5118 let key = query.key;
5119
5120 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5121 client_id: query.client_id,
5122 account_id: key.1,
5123 venue: key.0.venue,
5124 outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5125 commands: Rc::new(RefCell::new(Vec::new())),
5126 }));
5127
5128 let result = request_position_fill_reports(vec![client], vec![query]).await;
5129
5130 if accepted {
5131 assert_eq!(result.successful_keys, IndexSet::from([key]));
5132 assert_eq!(result.reports, IndexMap::from([(key, vec![report])]));
5133 } else {
5134 assert!(result.successful_keys.is_empty());
5135 assert!(result.reports.is_empty());
5136 }
5137 }
5138
5139 #[fixture]
5140 fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5141 let account_id = AccountId::from("POSITION-FILLS-001");
5142 let instrument_id = crypto_perpetual_ethusdt().id();
5143
5144 let report = FillReport::new(
5145 account_id,
5146 instrument_id,
5147 VenueOrderId::from("V-POSITION-FILLS"),
5148 TradeId::from("T-POSITION-FILLS"),
5149 OrderSide::Buy,
5150 Quantity::from("1.0"),
5151 Price::from("100.0"),
5152 Money::from("0.10 USDT"),
5153 LiquiditySide::Taker,
5154 Some(ClientOrderId::from("O-POSITION-FILLS")),
5155 None,
5156 UnixNanos::from(1_500),
5157 UnixNanos::from(2_000),
5158 None,
5159 );
5160
5161 let query = PositionFillReportQuery {
5162 key: (instrument_id, account_id),
5163 client_id: ClientId::from("POSITION-FILLS"),
5164 command: GenerateFillReports::new(
5165 UUID4::new(),
5166 UnixNanos::from(2_000),
5167 Some(instrument_id),
5168 None,
5169 Some(UnixNanos::from(1_000)),
5170 Some(UnixNanos::from(2_000)),
5171 None,
5172 None,
5173 ),
5174 };
5175
5176 (report, query)
5177 }
5178
5179 #[rstest]
5180 #[case::failed_query(false)]
5181 #[case::local_activity(true)]
5182 fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5183 let (mut node, report, _) =
5184 position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5185 let active_key = (report.instrument_id, report.account_id);
5186 let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5187 let deferred_reports = vec![
5188 PositionStatusReport::new(
5189 deferred_key.1,
5190 deferred_key.0,
5191 PositionSide::Long,
5192 Quantity::from("2.0"),
5193 UnixNanos::from(1_000),
5194 UnixNanos::from(1_000),
5195 None,
5196 None,
5197 Some(dec!(100.0)),
5198 ),
5199 PositionStatusReport::new(
5200 deferred_key.1,
5201 deferred_key.0,
5202 PositionSide::Short,
5203 Quantity::from("1.0"),
5204 UnixNanos::from(1_000),
5205 UnixNanos::from(1_000),
5206 None,
5207 None,
5208 Some(dec!(100.0)),
5209 ),
5210 ];
5211 let mut check = node
5212 .exec_manager
5213 .prepare_position_report_check(UUID4::new(), &[]);
5214 check.client_coverage.clear();
5215 let events = node.exec_manager.reconcile_position_reports(
5216 &check,
5217 deferred_reports.clone(),
5218 &IndexSet::new(),
5219 &IndexSet::new(),
5220 );
5221 assert!(events.is_empty());
5222 assert_eq!(
5223 node.exec_manager.position_recon_retry_count(&deferred_key),
5224 1
5225 );
5226
5227 let mut position_result = position_report_result(&node, report);
5228 position_result.reports.extend(deferred_reports);
5229 position_result.check.client_coverage.insert(
5230 deferred_key,
5231 ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5232 );
5233 position_result
5234 .check
5235 .activity_revisions
5236 .insert(deferred_key, 0);
5237 position_result
5238 .queried_clients
5239 .insert(ClientId::from("SECOND"));
5240 let mut successful_keys = IndexSet::from([active_key]);
5241
5242 if local_activity {
5243 successful_keys.insert(deferred_key);
5244 node.exec_manager
5245 .record_position_activity(deferred_key.0, deferred_key.1);
5246 }
5247
5248 node.handle_position_fill_report_result(PositionFillReportResult {
5249 position_result,
5250 reports: IndexMap::from([(active_key, Vec::new())]),
5251 successful_keys,
5252 });
5253
5254 assert_eq!(
5255 node.exec_manager.position_recon_retry_count(&deferred_key),
5256 1
5257 );
5258 {
5259 let cache = node.kernel.cache.borrow();
5260 let positions =
5261 cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5262 assert_eq!(
5263 positions
5264 .iter()
5265 .map(|position| position.quantity)
5266 .collect::<Vec<_>>(),
5267 vec![Quantity::from("1.0"), Quantity::from("1.0")],
5268 );
5269 assert_eq!(
5270 cache
5271 .positions_open(
5272 None,
5273 Some(&deferred_key.0),
5274 None,
5275 Some(&deferred_key.1),
5276 None,
5277 )
5278 .len(),
5279 0
5280 );
5281 }
5282
5283 let mut fresh_check = node
5284 .exec_manager
5285 .prepare_position_report_check(UUID4::new(), &[]);
5286 node.exec_manager.plan_position_fill_reports(
5287 &mut fresh_check,
5288 &[],
5289 &IndexSet::new(),
5290 &IndexSet::new(),
5291 &[],
5292 );
5293
5294 assert_eq!(
5295 node.exec_manager.position_recon_retry_count(&deferred_key),
5296 0
5297 );
5298 }
5299
5300 #[rstest]
5301 fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5302 let (mut node, venue_report, fill_report) =
5303 position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5304 let key = (venue_report.instrument_id, venue_report.account_id);
5305 let revision = node.exec_manager.position_activity_revision(&key);
5306 let position_result = position_report_result(&node, venue_report);
5307
5308 node.handle_position_fill_report_result(PositionFillReportResult {
5309 position_result,
5310 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5311 successful_keys: IndexSet::from([key]),
5312 });
5313
5314 let cache = node.kernel.cache.borrow();
5315 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5316 assert_eq!(
5317 positions
5318 .iter()
5319 .map(|position| position.quantity)
5320 .sum::<Quantity>(),
5321 Quantity::from("2.0")
5322 );
5323 assert_eq!(
5324 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5325 1
5326 );
5327 drop(positions);
5328 drop(cache);
5329 assert!(
5330 node.exec_manager
5331 .position_contains_fill_report(&fill_report)
5332 );
5333 assert_eq!(
5334 node.exec_manager.position_activity_revision(&key),
5335 revision + 1
5336 );
5337 }
5338
5339 #[rstest]
5340 fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5341 let (mut node, venue_report, fill_report) =
5342 position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5343 let key = (venue_report.instrument_id, venue_report.account_id);
5344 let position_result = position_report_result(&node, venue_report);
5345
5346 node.handle_position_fill_report_result(PositionFillReportResult {
5347 position_result,
5348 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5349 successful_keys: IndexSet::from([key]),
5350 });
5351
5352 let mut rest_report = fill_report;
5353 rest_report.ts_event = UnixNanos::from(2_000);
5354
5355 assert!(
5356 node.exec_manager
5357 .position_contains_fill_report(&rest_report)
5358 );
5359 }
5360
5361 #[rstest]
5362 fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5363 let (mut node, venue_report, fill_report) =
5364 position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5365 let key = (venue_report.instrument_id, venue_report.account_id);
5366 let client_order_id = fill_report.client_order_id.unwrap();
5367 apply_inferred_position_fill(&mut node, &fill_report);
5368
5369 let venue_report = PositionStatusReport::new(
5370 key.1,
5371 key.0,
5372 PositionSide::Long,
5373 Quantity::from("3.0"),
5374 venue_report.ts_last,
5375 venue_report.ts_init,
5376 None,
5377 None,
5378 venue_report.avg_px_open,
5379 );
5380 let mut later_fill_report = fill_report.clone();
5381 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5382 later_fill_report.ts_event = UnixNanos::from(1_001);
5383 let position_result = position_report_result(&node, venue_report);
5384
5385 node.handle_position_fill_report_result(PositionFillReportResult {
5386 position_result,
5387 reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5388 successful_keys: IndexSet::from([key]),
5389 });
5390
5391 let cache = node.kernel.cache.borrow();
5392 let order = cache.order(&client_order_id).unwrap();
5393 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5394 assert_eq!(order.filled_qty(), Quantity::from("2.0"));
5395 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5396 assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
5397 assert_eq!(
5398 positions
5399 .iter()
5400 .map(|position| position.quantity)
5401 .sum::<Quantity>(),
5402 Quantity::from("3.0")
5403 );
5404 assert_eq!(
5405 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5406 2
5407 );
5408 }
5409
5410 #[rstest]
5411 fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
5412 let (mut node, _, _) =
5413 position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
5414 let account_id = AccountId::from("TEST-001");
5415 let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
5416 let key = (instrument.id(), account_id);
5417 let commission = Money::zero(instrument.quote_currency());
5418 node.kernel
5419 .cache
5420 .borrow_mut()
5421 .add_instrument(instrument.clone())
5422 .unwrap();
5423
5424 let order = OrderTestBuilder::new(OrderType::Market)
5425 .strategy_id(StrategyId::external())
5426 .instrument_id(instrument.id())
5427 .client_order_id(ClientOrderId::from("O-RECONCILED"))
5428 .quantity(Quantity::from("1.000000"))
5429 .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
5430 .build();
5431 let submitted = TestOrderEventStubs::submitted(&order, account_id);
5432 let accepted =
5433 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
5434 node.kernel
5435 .cache
5436 .borrow_mut()
5437 .add_order(
5438 order.clone(),
5439 None,
5440 Some(ClientId::from("POSITION-FILLS")),
5441 false,
5442 )
5443 .unwrap();
5444 node.process_reconciliation_events(&[submitted, accepted]);
5445 let order = node
5446 .kernel
5447 .cache
5448 .borrow()
5449 .order_owned(&order.client_order_id())
5450 .unwrap();
5451 let opening_fill = TestOrderEventStubs::filled(
5452 &order,
5453 &instrument,
5454 Some(TradeId::from("T-RECONCILED")),
5455 None,
5456 Some(Price::from("100.00")),
5457 Some(Quantity::from("1.000000")),
5458 Some(LiquiditySide::Taker),
5459 Some(commission),
5460 Some(UnixNanos::from(2_000)),
5461 Some(account_id),
5462 );
5463 node.process_reconciliation_events(&[opening_fill]);
5464
5465 let held_fill_report = FillReport::new(
5467 account_id,
5468 instrument.id(),
5469 VenueOrderId::from("V-HELD"),
5470 TradeId::from("T-HELD"),
5471 OrderSide::Buy,
5472 Quantity::from("1.000000"),
5473 Price::from("100.00"),
5474 commission,
5475 LiquiditySide::Taker,
5476 None,
5477 None,
5478 UnixNanos::from(1_000),
5479 UnixNanos::from(1_000),
5480 None,
5481 );
5482 node.kernel
5483 .exec_engine
5484 .borrow_mut()
5485 .reconcile_fill_report(&held_fill_report);
5486
5487 let venue_report = PositionStatusReport::new(
5488 account_id,
5489 instrument.id(),
5490 PositionSide::Long,
5491 Quantity::from("2.000000"),
5492 UnixNanos::from(3_000),
5493 UnixNanos::from(3_000),
5494 None,
5495 None,
5496 Some(dec!(100.00)),
5497 );
5498 let position_result = position_report_result(&node, venue_report);
5499
5500 node.handle_position_fill_report_result(PositionFillReportResult {
5501 position_result,
5502 reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
5503 successful_keys: IndexSet::from([key]),
5504 });
5505
5506 let cache = node.kernel.cache.borrow();
5507 let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
5508 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5509 assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
5510 assert_eq!(
5511 positions
5512 .iter()
5513 .map(|position| position.quantity)
5514 .sum::<Quantity>(),
5515 Quantity::from("2.000000")
5516 );
5517 assert!(
5518 positions
5519 .iter()
5520 .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
5521 );
5522 }
5523
5524 #[rstest]
5525 fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
5526 let (mut node, _, mut fill_report) =
5527 position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
5528 apply_inferred_position_fill(&mut node, &fill_report);
5529 let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
5530 fill_report.venue_position_id = Some(conflicting_position_id);
5531
5532 let error = node
5533 .exec_manager
5534 .prepare_position_fill_report(&mut fill_report, &[])
5535 .unwrap_err();
5536
5537 assert!(
5538 error.to_string().contains(&format!(
5539 "position ID {conflicting_position_id} conflicts with cached order position"
5540 )),
5541 "{error:#}"
5542 );
5543 }
5544
5545 #[rstest]
5546 #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
5547 #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
5548 fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
5549 #[case] authoritative_qty: &str,
5550 #[case] client_order_id: &str,
5551 ) {
5552 let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
5553 "UnappliedPositionFillNode",
5554 Quantity::from(authoritative_qty),
5555 );
5556 fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
5557 let reports = [fill_report.clone()];
5558
5559 let quantities = (0..2)
5560 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
5561 .collect::<Vec<_>>();
5562
5563 assert_eq!(
5564 quantities,
5565 vec![Quantity::from("1.0"), Quantity::from("2.0")]
5566 );
5567 assert!(
5568 !node
5569 .exec_manager
5570 .position_contains_fill_report(&fill_report)
5571 );
5572 }
5573
5574 #[rstest]
5575 fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
5576 let (mut node, venue_report, fill_report) =
5577 position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
5578 let mut later_fill_report = fill_report.clone();
5579 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5580 later_fill_report.ts_event = UnixNanos::from(1_001);
5581 let reports = [fill_report.clone(), later_fill_report.clone()];
5582
5583 let quantities = (0..3)
5584 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
5585 .collect::<Vec<_>>();
5586
5587 assert_eq!(
5588 quantities,
5589 vec![
5590 Quantity::from("1.0"),
5591 Quantity::from("1.0"),
5592 Quantity::from("2.0"),
5593 ]
5594 );
5595 assert!(
5596 !node
5597 .exec_manager
5598 .position_contains_fill_report(&fill_report)
5599 );
5600 assert!(
5601 !node
5602 .exec_manager
5603 .position_contains_fill_report(&later_fill_report)
5604 );
5605 }
5606
5607 #[rstest]
5608 #[case::within_grace(60_000, true)]
5609 #[case::after_grace(0, false)]
5610 fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
5611 #[case] position_check_threshold_ms: u32,
5612 #[case] expected_applied: bool,
5613 ) {
5614 let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
5615 "TransientPositionFillNode",
5616 Quantity::from("1.0"),
5617 position_check_threshold_ms,
5618 );
5619 let mut conflicting_report = fill_report.clone();
5620 conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
5621
5622 let quantities = [conflicting_report, fill_report.clone()]
5623 .into_iter()
5624 .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
5625 .collect::<Vec<_>>();
5626
5627 assert_eq!(
5628 quantities,
5629 vec![Quantity::from("1.0"), Quantity::from("2.0")]
5630 );
5631 assert_eq!(
5632 node.exec_manager
5633 .position_contains_fill_report(&fill_report),
5634 expected_applied
5635 );
5636 }
5637
5638 #[rstest]
5639 fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
5640 let (mut node, venue_report, fill_report) =
5641 position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
5642 let key = (venue_report.instrument_id, venue_report.account_id);
5643 let first_result = position_report_result(&node, venue_report.clone());
5644
5645 node.handle_position_fill_report_result(PositionFillReportResult {
5646 position_result: first_result,
5647 reports: IndexMap::from([(key, vec![fill_report])]),
5648 successful_keys: IndexSet::from([key]),
5649 });
5650
5651 let fresh_result = position_report_result(&node, venue_report);
5652 node.handle_position_fill_report_result(PositionFillReportResult {
5653 position_result: fresh_result,
5654 reports: IndexMap::from([(key, Vec::new())]),
5655 successful_keys: IndexSet::from([key]),
5656 });
5657
5658 let cache = node.kernel.cache.borrow();
5659 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5660 assert_eq!(
5661 positions
5662 .iter()
5663 .map(|position| position.quantity)
5664 .sum::<Quantity>(),
5665 Quantity::from("2.0")
5666 );
5667 assert_eq!(
5668 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5669 2
5670 );
5671 }
5672
5673 #[rstest]
5674 #[case::below_limit(63)]
5675 #[case::at_limit(64)]
5676 #[case::above_limit(65)]
5677 fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
5678 let (mut node, venue_report, fill_report) =
5679 position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
5680 let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
5681
5682 let venue_report = PositionStatusReport::new(
5683 venue_report.account_id,
5684 venue_report.instrument_id,
5685 PositionSide::Long,
5686 Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
5687 venue_report.ts_last,
5688 venue_report.ts_init,
5689 None,
5690 None,
5691 venue_report.avg_px_open,
5692 );
5693 let key = (venue_report.instrument_id, venue_report.account_id);
5694 let client_order_id = fill_report.client_order_id.unwrap();
5695
5696 let reports = (0..count)
5697 .map(|index| {
5698 let mut report = fill_report.clone();
5699 report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
5700 report.ts_event = UnixNanos::from(1_000 + index as u64);
5701 report
5702 })
5703 .collect::<Vec<_>>();
5704
5705 let position_result = position_report_result(&node, venue_report.clone());
5706
5707 node.handle_position_fill_report_result(PositionFillReportResult {
5708 position_result,
5709 reports: IndexMap::from([(key, reports.clone())]),
5710 successful_keys: IndexSet::from([key]),
5711 });
5712
5713 {
5714 let cache = node.kernel.cache.borrow();
5715 let order = cache.order(&client_order_id).unwrap();
5716 let applied = count.min(64);
5717 assert_eq!(
5718 order.filled_qty().as_decimal(),
5719 dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
5720 );
5721 assert_eq!(order.trade_ids().len(), applied + 1);
5722
5723 for (index, report) in reports.iter().enumerate() {
5724 assert_eq!(
5725 order.trade_ids().contains(&&report.trade_id),
5726 index < applied
5727 );
5728 }
5729
5730 assert_eq!(
5731 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5732 1
5733 );
5734 }
5735
5736 let position_result = position_report_result(&node, venue_report);
5738 node.handle_position_fill_report_result(PositionFillReportResult {
5739 position_result,
5740 reports: IndexMap::from([(key, reports)]),
5741 successful_keys: IndexSet::from([key]),
5742 });
5743
5744 let cache = node.kernel.cache.borrow();
5745 let order = cache.order(&client_order_id).unwrap();
5746 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5747 assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
5748 assert_eq!(order.trade_ids().len(), count + 1);
5749 assert_eq!(positions.len(), 1);
5750 assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
5751 assert_eq!(positions[0].side, PositionSide::Long);
5752 assert_eq!(
5753 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5754 1
5755 );
5756 }
5757
5758 #[rstest]
5759 #[case::client_order(0)]
5760 #[case::venue_order(1)]
5761 #[case::side(2)]
5762 #[case::position(3)]
5763 fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
5764 let (mut node, venue_report, valid_report) =
5765 position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
5766 let key = (venue_report.instrument_id, venue_report.account_id);
5767 let client_order_id = valid_report.client_order_id.unwrap();
5768 let revision = node.exec_manager.position_activity_revision(&key);
5769 let position_result = position_report_result(&node, venue_report);
5770 let mut conflicting_report = valid_report.clone();
5771 conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
5772
5773 match conflict {
5774 0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
5775 1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
5776 2 => conflicting_report.order_side = OrderSide::Sell,
5777 3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
5778 _ => unreachable!(),
5779 }
5780
5781 node.handle_position_fill_report_result(PositionFillReportResult {
5782 position_result,
5783 reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
5784 successful_keys: IndexSet::from([key]),
5785 });
5786
5787 let cache = node.kernel.cache.borrow();
5788 let order = cache.order(&client_order_id).unwrap();
5789 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5790 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
5791 assert_eq!(
5792 order.trade_ids(),
5793 vec![&TradeId::from("T-POSITION-INITIAL")]
5794 );
5795 assert_eq!(positions.len(), 1);
5796 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5797 assert_eq!(
5798 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5799 1
5800 );
5801 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
5802 }
5803
5804 #[rstest]
5805 fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
5806 let (mut node, venue_report, _) =
5807 position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
5808 let key = (venue_report.instrument_id, venue_report.account_id);
5809 let position_result = position_report_result(&node, venue_report);
5810
5811 node.handle_position_fill_report_result(PositionFillReportResult {
5812 position_result,
5813 reports: IndexMap::new(),
5814 successful_keys: IndexSet::new(),
5815 });
5816
5817 let cache = node.kernel.cache.borrow();
5818 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5819 assert_eq!(positions.len(), 1);
5820 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5821 assert_eq!(
5822 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5823 1
5824 );
5825 }
5826
5827 #[rstest]
5828 fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
5829 let (mut node, mut venue_report, mut fill_report) =
5830 position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
5831 node.kernel
5832 .exec_engine
5833 .borrow_mut()
5834 .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
5835 let key = (venue_report.instrument_id, venue_report.account_id);
5836
5837 let position_id = {
5838 let cache = node.kernel.cache.borrow();
5839 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5840 assert_eq!(positions.len(), 1);
5841 positions[0].id
5842 };
5843
5844 venue_report.venue_position_id = Some(position_id);
5845 fill_report.client_order_id = None;
5846 fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
5847 let position_result = position_report_result(&node, venue_report);
5848
5849 node.handle_position_fill_report_result(PositionFillReportResult {
5850 position_result,
5851 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5852 successful_keys: IndexSet::from([key]),
5853 });
5854
5855 let cache = node.kernel.cache.borrow();
5856 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5857 assert_eq!(positions.len(), 1);
5858 assert_eq!(positions[0].id, position_id);
5859 assert_eq!(positions[0].quantity, Quantity::from("2.0"));
5860 assert_eq!(
5861 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5862 2
5863 );
5864 drop(positions);
5865 drop(cache);
5866 assert!(
5867 !node
5868 .exec_manager
5869 .position_contains_fill_report(&fill_report)
5870 );
5871 }
5872
5873 #[rstest]
5874 fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
5875 let (mut node, venue_report, fill_report) =
5876 position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
5877 let key = (venue_report.instrument_id, venue_report.account_id);
5878 let client_order_id = fill_report.client_order_id.unwrap();
5879 let position_result = position_report_result(&node, venue_report);
5880 let revision = node.exec_manager.position_activity_revision(&key);
5881
5882 let order_report = OrderStatusReport::new(
5883 key.1,
5884 key.0,
5885 Some(client_order_id),
5886 fill_report.venue_order_id,
5887 OrderSide::Buy.into(),
5888 OrderType::Limit,
5889 TimeInForce::Gtc,
5890 OrderStatus::PartiallyFilled,
5891 Quantity::from("10.0"),
5892 Quantity::from("2.0"),
5893 UnixNanos::from(1_000),
5894 UnixNanos::from(1_000),
5895 UnixNanos::from(2_000),
5896 None,
5897 );
5898 let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
5899 Box::new(order_report),
5900 vec![fill_report.clone()],
5901 ));
5902
5903 assert_eq!(
5904 node.observe_exec_event_before_dispatch(&event),
5905 Some(vec![client_order_id])
5906 );
5907
5908 assert_eq!(
5909 node.exec_manager.position_activity_revision(&key),
5910 revision + 1
5911 );
5912 assert!(
5913 !node
5914 .exec_manager
5915 .position_report_check_is_current(&position_result.check, &key)
5916 );
5917
5918 node.handle_position_fill_report_result(PositionFillReportResult {
5919 position_result,
5920 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5921 successful_keys: IndexSet::from([key]),
5922 });
5923
5924 let cache = node.kernel.cache.borrow();
5925 let order = cache.order(&client_order_id).unwrap();
5926 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5927 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
5928 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5929 assert_eq!(positions.len(), 1);
5930 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5931 assert_eq!(positions[0].side, PositionSide::Long);
5932 assert_eq!(
5933 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5934 1
5935 );
5936 }
5937
5938 #[rstest]
5939 fn test_position_fill_report_result_defers_after_local_position_activity() {
5940 let (mut node, venue_report, fill_report) =
5941 position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
5942 let key = (venue_report.instrument_id, venue_report.account_id);
5943 let position_result = position_report_result(&node, venue_report);
5944 node.exec_manager.record_position_activity(key.0, key.1);
5945
5946 node.handle_position_fill_report_result(PositionFillReportResult {
5947 position_result,
5948 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5949 successful_keys: IndexSet::from([key]),
5950 });
5951
5952 let cache = node.kernel.cache.borrow();
5953 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5954 assert_eq!(positions.len(), 1);
5955 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5956 assert_eq!(
5957 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5958 1
5959 );
5960 drop(positions);
5961 drop(cache);
5962 assert!(
5963 !node
5964 .exec_manager
5965 .position_contains_fill_report(&fill_report)
5966 );
5967 }
5968
5969 #[rstest]
5970 fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
5971 let config = LiveNodeConfig {
5972 exec_engine: crate::config::LiveExecutionEngineConfig {
5973 reconciliation: true,
5974 open_check_threshold_ms: 5_000,
5975 single_order_query_delay_ms: 0,
5976 ..Default::default()
5977 },
5978 ..Default::default()
5979 };
5980
5981 let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
5982 let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
5983 let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
5984 let instrument = crypto_perpetual_ethusdt();
5985 let instrument_id = instrument.id();
5986 let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
5987 let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
5988
5989 node.kernel
5990 .cache
5991 .borrow_mut()
5992 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
5993 .unwrap();
5994 insert_accepted_limit_order_in_node(
5995 &node,
5996 account_id,
5997 client_id,
5998 instrument_id,
5999 client_order_id,
6000 venue_order_id,
6001 );
6002
6003 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6004
6005 let accepted = OrderAcceptedSpec::builder()
6006 .instrument_id(instrument_id)
6007 .client_order_id(client_order_id)
6008 .venue_order_id(venue_order_id)
6009 .account_id(account_id)
6010 .build();
6011 let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6012
6013 let close_ids = node.observe_exec_event_before_dispatch(&event);
6014
6015 assert_eq!(close_ids, Some(vec![client_order_id]));
6016 assert!(node.exec_manager.check_open_order_queries().is_empty());
6017 }
6018
6019 #[rstest]
6020 #[cfg_attr(
6021 not(all(feature = "simulation", madsim)),
6022 tokio::test(start_paused = true)
6023 )]
6024 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6025 async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6026 use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6027 use nautilus_model::{events::OrderPendingCancel, identifiers::ClientOrderId};
6028
6029 let config = LiveNodeConfig {
6030 exec_engine: crate::config::LiveExecutionEngineConfig {
6031 reconciliation: true,
6032 inflight_check_threshold_ms: 100,
6033 inflight_check_retries: 1,
6034 ..Default::default()
6035 },
6036 ..Default::default()
6037 };
6038
6039 let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6040 let trader_id = TraderId::from("TESTER-001");
6041 let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6042 let account_id = AccountId::from("TEST-001");
6043 let instrument = crypto_perpetual_ethusdt();
6044 let instrument_id = instrument.id();
6045 let child_ids = [
6046 ClientOrderId::from("O-BATCH-CANCEL-1"),
6047 ClientOrderId::from("O-BATCH-CANCEL-2"),
6048 ];
6049 node.kernel
6050 .cache
6051 .borrow_mut()
6052 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6053 .unwrap();
6054
6055 for client_order_id in child_ids {
6056 let order = OrderTestBuilder::new(OrderType::Limit)
6057 .trader_id(trader_id)
6058 .strategy_id(strategy_id)
6059 .client_order_id(client_order_id)
6060 .instrument_id(instrument_id)
6061 .quantity(Quantity::from("10.0"))
6062 .price(Price::from("100.0"))
6063 .build();
6064 let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6065 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6069 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6070 let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
6071 trader_id,
6072 strategy_id,
6073 instrument_id,
6074 client_order_id,
6075 Some(account_id),
6076 UUID4::new(),
6077 UnixNanos::default(),
6078 UnixNanos::default(),
6079 false,
6080 Some(venue_order_id),
6081 ));
6082 let mut cache = node.kernel.cache.borrow_mut();
6083 cache.add_order(order, None, None, false).unwrap();
6084 cache.update_order(&submitted).unwrap();
6085 cache.update_order(&accepted).unwrap();
6086 cache.update_order(&pending_cancel).unwrap();
6087 }
6088
6089 let cancels = child_ids
6090 .into_iter()
6091 .map(|client_order_id| {
6092 CancelOrder::new(
6093 trader_id,
6094 None,
6095 strategy_id,
6096 instrument_id,
6097 client_order_id,
6098 None,
6099 UUID4::new(),
6100 UnixNanos::default(),
6101 None,
6102 None,
6103 )
6104 })
6105 .collect();
6106
6107 let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6108 trader_id,
6109 None,
6110 strategy_id,
6111 instrument_id,
6112 cancels,
6113 UUID4::new(),
6114 UnixNanos::default(),
6115 None,
6116 None,
6117 ));
6118
6119 node.observe_exec_command_before_dispatch(&command);
6120 advance_clock(Duration::from_millis(101)).await;
6121 let result = node.exec_manager.check_inflight_orders();
6122 let timed_out_ids = result
6123 .events
6124 .iter()
6125 .map(OrderEventAny::client_order_id)
6126 .collect::<IndexSet<_>>();
6127
6128 assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6129 assert_eq!(result.events.len(), child_ids.len());
6130 assert!(
6131 result
6132 .events
6133 .iter()
6134 .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6135 "batch-cancel children must time out as Canceled events",
6136 );
6137 }
6138
6139 #[rstest]
6140 #[cfg_attr(
6141 not(all(feature = "simulation", madsim)),
6142 tokio::test(start_paused = true)
6143 )]
6144 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6145 async fn test_risk_bound_command_does_not_register_inflight() {
6146 let config = LiveNodeConfig {
6147 exec_engine: crate::config::LiveExecutionEngineConfig {
6148 reconciliation: true,
6149 inflight_check_threshold_ms: 100,
6150 inflight_check_retries: 2,
6151 ..Default::default()
6152 },
6153 ..Default::default()
6154 };
6155
6156 let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6157 msgbus::register_trading_command_endpoint(
6158 MessagingSwitchboard::risk_engine_execute(),
6159 TypedIntoHandler::from(|_: TradingCommand| {}),
6160 );
6161 let instrument = crypto_perpetual_ethusdt();
6162 let instrument_id = instrument.id();
6163 let order = OrderTestBuilder::new(OrderType::Limit)
6164 .trader_id(node.trader_id())
6165 .strategy_id(StrategyId::from("S-RISK-DENIED"))
6166 .instrument_id(instrument_id)
6167 .side(OrderSide::Buy)
6168 .quantity(Quantity::from("1.000"))
6169 .price(Price::from("100.00"))
6170 .build();
6171 let client_order_id = order.client_order_id();
6172
6173 {
6174 let mut cache = node.kernel.cache.borrow_mut();
6175 cache
6176 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6177 .unwrap();
6178 cache.add_order(order.clone(), None, None, false).unwrap();
6179 }
6180
6181 let submit_order = SubmitOrder::new(
6182 order.trader_id(),
6183 None,
6184 order.strategy_id(),
6185 instrument_id,
6186 client_order_id,
6187 order.init_event().clone(),
6188 None,
6189 None,
6190 None,
6191 UUID4::new(),
6192 UnixNanos::default(),
6193 None,
6194 );
6195 node.process_exec_command(
6196 TradingCommandMessage::new(
6197 MessagingSwitchboard::risk_engine_execute(),
6198 TradingCommand::SubmitOrder(submit_order),
6199 )
6200 .into(),
6201 );
6202
6203 advance_clock(Duration::from_millis(101)).await;
6204 let result = node.exec_manager.check_inflight_orders();
6205 let status = node
6206 .kernel
6207 .cache
6208 .borrow()
6209 .order(&client_order_id)
6210 .unwrap()
6211 .status();
6212
6213 assert_eq!(status, OrderStatus::Initialized);
6214 assert_eq!(
6215 node.exec_manager.recon_check_retry_count(&client_order_id),
6216 0
6217 );
6218 assert!(result.events.is_empty());
6219 assert!(result.queries.is_empty());
6220 }
6221
6222 #[rstest]
6223 #[cfg_attr(
6224 not(all(feature = "simulation", madsim)),
6225 tokio::test(start_paused = true)
6226 )]
6227 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6228 async fn test_risk_approved_command_registers_inflight() {
6229 let config = LiveNodeConfig {
6230 risk_engine: crate::config::LiveRiskEngineConfig {
6231 bypass: true,
6232 ..Default::default()
6233 },
6234 exec_engine: crate::config::LiveExecutionEngineConfig {
6235 reconciliation: true,
6236 inflight_check_threshold_ms: 100,
6237 inflight_check_retries: 2,
6238 ..Default::default()
6239 },
6240 ..Default::default()
6241 };
6242
6243 let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
6244 msgbus::register_trading_command_endpoint(
6245 MessagingSwitchboard::exec_engine_execute(),
6246 TypedIntoHandler::from(|_: TradingCommand| {}),
6247 );
6248 let instrument = crypto_perpetual_ethusdt();
6249 let instrument_id = instrument.id();
6250 let order = OrderTestBuilder::new(OrderType::Limit)
6251 .trader_id(node.trader_id())
6252 .strategy_id(StrategyId::from("S-RISK-APPROVED"))
6253 .instrument_id(instrument_id)
6254 .side(OrderSide::Buy)
6255 .quantity(Quantity::from("1.000"))
6256 .price(Price::from("100.00"))
6257 .build();
6258 let client_order_id = order.client_order_id();
6259
6260 {
6261 let mut cache = node.kernel.cache.borrow_mut();
6262 cache
6263 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6264 .unwrap();
6265 cache.add_order(order.clone(), None, None, false).unwrap();
6266 }
6267
6268 let submit_order = SubmitOrder::new(
6269 order.trader_id(),
6270 None,
6271 order.strategy_id(),
6272 instrument_id,
6273 client_order_id,
6274 order.init_event().clone(),
6275 None,
6276 None,
6277 None,
6278 UUID4::new(),
6279 UnixNanos::default(),
6280 None,
6281 );
6282 node.process_exec_command(
6283 TradingCommandMessage::new(
6284 MessagingSwitchboard::risk_engine_execute(),
6285 TradingCommand::SubmitOrder(submit_order),
6286 )
6287 .into(),
6288 );
6289
6290 advance_clock(Duration::from_millis(101)).await;
6291 let result = node.exec_manager.check_inflight_orders();
6292
6293 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6294 panic!("expected one query order command");
6295 };
6296
6297 assert_eq!(query.client_order_id, client_order_id);
6298 assert_eq!(
6299 node.exec_manager.recon_check_retry_count(&client_order_id),
6300 1
6301 );
6302 assert!(result.events.is_empty());
6303 }
6304
6305 #[rstest]
6306 fn test_live_node_builder_clock_factory_drives_kernel_clock() {
6307 let calls = Rc::new(Cell::new(0usize));
6308 let calls_in_factory = calls.clone();
6309 let sentinel = UnixNanos::from(123_456_789_u64);
6310
6311 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6312 .unwrap()
6313 .with_reconciliation(false)
6314 .with_clock_factory(move || {
6315 calls_in_factory.set(calls_in_factory.get() + 1);
6316 let mut clock = VirtualClock::new();
6317 clock.advance_time(sentinel, true);
6318 Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
6319 })
6320 .build()
6321 .unwrap();
6322
6323 assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
6324 assert_eq!(calls.get(), 1);
6325 }
6326
6327 #[derive(Debug)]
6328 struct ReplayKernelEventStore {
6329 fail_restore: bool,
6330 }
6331
6332 impl KernelEventStore for ReplayKernelEventStore {
6333 fn restore_parent_cache(
6334 &mut self,
6335 _instance_id: UUID4,
6336 _cache: &mut Cache,
6337 ) -> anyhow::Result<()> {
6338 if self.fail_restore {
6339 anyhow::bail!("replay restore failed");
6340 }
6341
6342 Ok(())
6343 }
6344
6345 fn open(
6346 &mut self,
6347 _instance_id: UUID4,
6348 _components: &RegisteredComponents,
6349 _environment: Environment,
6350 ) -> anyhow::Result<()> {
6351 Ok(())
6352 }
6353
6354 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
6355 None
6356 }
6357
6358 fn seal(&mut self, _ts_init: UnixNanos) {}
6359
6360 fn run_id(&self) -> Option<&str> {
6361 Some("replay-child")
6362 }
6363
6364 fn parent_run_id(&self) -> Option<&str> {
6365 Some("seed-run")
6366 }
6367
6368 fn is_event_store_replay_configured(&self) -> bool {
6369 true
6370 }
6371
6372 fn is_halted(&self) -> bool {
6373 false
6374 }
6375 }
6376
6377 #[derive(Debug)]
6378 struct TestStrategy {
6379 core: StrategyCore,
6380 }
6381
6382 impl TestStrategy {
6383 fn new(config: StrategyConfig) -> Self {
6384 Self {
6385 core: StrategyCore::new(config),
6386 }
6387 }
6388 }
6389
6390 impl DataActor for TestStrategy {}
6391
6392 nautilus_strategy!(TestStrategy, {
6393 fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
6394 self.core.config.external_order_instrument_ids.clone()
6395 }
6396 });
6397
6398 fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
6399 let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
6402 .unwrap()
6403 .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
6404 reconciliation: false,
6405 ..Default::default()
6406 })
6407 .with_load_state(true)
6408 .with_name("TestKernel")
6409 .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
6410 Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
6411 });
6412
6413 builder.build().unwrap()
6414 }
6415
6416 #[rstest]
6417 fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
6418 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6419 .unwrap()
6420 .with_reconciliation(false)
6421 .with_delay_post_stop_secs(0)
6422 .with_timeout_connection(1)
6423 .build()
6424 .unwrap();
6425 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6426 let strategy_id = StrategyId::from("CLAIMS-001");
6427
6428 node.add_strategy(TestStrategy::new(StrategyConfig {
6429 strategy_id: Some(strategy_id),
6430 external_order_instrument_ids: Some(vec![instrument_id]),
6431 ..Default::default()
6432 }))
6433 .unwrap();
6434
6435 assert_eq!(
6436 node.exec_manager.get_external_order_claim(&instrument_id),
6437 Some(strategy_id)
6438 );
6439
6440 {
6441 let exec_engine = node.kernel().exec_engine.borrow();
6442 assert_eq!(
6443 exec_engine.get_external_order_claim(&instrument_id),
6444 Some(strategy_id)
6445 );
6446 }
6447 }
6448
6449 #[rstest]
6450 fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
6451 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6452 .unwrap()
6453 .with_reconciliation(false)
6454 .build()
6455 .unwrap();
6456 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6457 let strategy_id = StrategyId::from("CLAIMS-001");
6458
6459 node.register_external_order_claims(strategy_id, &[instrument_id])
6460 .unwrap();
6461
6462 assert_eq!(
6463 node.exec_manager.get_external_order_claim(&instrument_id),
6464 Some(strategy_id)
6465 );
6466 assert_eq!(
6467 node.kernel
6468 .exec_engine
6469 .borrow()
6470 .get_external_order_claim(&instrument_id),
6471 Some(strategy_id)
6472 );
6473 }
6474
6475 #[rstest]
6476 #[tokio::test]
6477 async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
6478 let mut node = live_node_with_replay_store(false);
6479 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6480 let strategy_id = StrategyId::from("CLAIMS-001");
6481
6482 node.start().await.unwrap();
6483 assert_eq!(node.state(), NodeState::Running);
6484
6485 node.register_external_order_claims(strategy_id, &[instrument_id])
6486 .unwrap();
6487
6488 assert_eq!(
6489 node.exec_manager.get_external_order_claim(&instrument_id),
6490 Some(strategy_id)
6491 );
6492 assert_eq!(
6493 node.kernel
6494 .exec_engine
6495 .borrow()
6496 .get_external_order_claim(&instrument_id),
6497 Some(strategy_id)
6498 );
6499 }
6500
6501 #[rstest]
6502 fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
6503 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6504 .unwrap()
6505 .with_reconciliation(false)
6506 .build()
6507 .unwrap();
6508 let existing_instrument = InstrumentId::from("AUDUSD.SIM");
6509 let new_instruments = [
6510 InstrumentId::from("EURUSD.SIM"),
6511 InstrumentId::from("GBPUSD.SIM"),
6512 ];
6513 let existing_strategy_id = StrategyId::from("CLAIMS-001");
6514 let new_strategy_id = StrategyId::from("CLAIMS-002");
6515 node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
6516 .unwrap();
6517
6518 let result = node.register_external_order_claims(
6519 new_strategy_id,
6520 &[new_instruments[0], existing_instrument, new_instruments[1]],
6521 );
6522
6523 assert!(result.is_err());
6524 assert_eq!(
6525 node.exec_manager
6526 .get_external_order_claim(&existing_instrument),
6527 Some(existing_strategy_id)
6528 );
6529
6530 for instrument_id in new_instruments {
6531 assert_eq!(
6532 node.exec_manager.get_external_order_claim(&instrument_id),
6533 None
6534 );
6535 assert_eq!(
6536 node.kernel
6537 .exec_engine
6538 .borrow()
6539 .get_external_order_claim(&instrument_id),
6540 None
6541 );
6542 }
6543 }
6544
6545 #[rstest]
6546 fn test_deregister_external_order_claims_allows_successor_to_claim() {
6547 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6548 .unwrap()
6549 .with_reconciliation(false)
6550 .build()
6551 .unwrap();
6552 let instruments = [
6553 InstrumentId::from("AUDUSD.SIM"),
6554 InstrumentId::from("EURUSD.SIM"),
6555 ];
6556 let first_strategy_id = StrategyId::from("CLAIMS-001");
6557 let successor_strategy_id = StrategyId::from("CLAIMS-002");
6558 node.register_external_order_claims(first_strategy_id, &instruments)
6559 .unwrap();
6560
6561 node.deregister_external_order_claims(first_strategy_id)
6562 .unwrap();
6563 node.register_external_order_claims(successor_strategy_id, &instruments)
6564 .unwrap();
6565
6566 for instrument_id in instruments {
6567 assert_eq!(
6568 node.exec_manager.get_external_order_claim(&instrument_id),
6569 Some(successor_strategy_id)
6570 );
6571 assert_eq!(
6572 node.kernel
6573 .exec_engine
6574 .borrow()
6575 .get_external_order_claim(&instrument_id),
6576 Some(successor_strategy_id)
6577 );
6578 }
6579 }
6580
6581 #[rstest]
6582 fn test_deregister_external_order_claims_without_claims_is_idempotent() {
6583 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6584 .unwrap()
6585 .with_reconciliation(false)
6586 .build()
6587 .unwrap();
6588 let strategy_id = StrategyId::from("CLAIMS-001");
6589
6590 node.deregister_external_order_claims(strategy_id).unwrap();
6591 node.deregister_external_order_claims(strategy_id).unwrap();
6592 }
6593
6594 #[rstest]
6595 fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
6596 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6597 .unwrap()
6598 .with_reconciliation(false)
6599 .with_delay_post_stop_secs(0)
6600 .with_timeout_connection(1)
6601 .build()
6602 .unwrap();
6603 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6604 let strategy_id = StrategyId::from("CLAIMS-001");
6605 let duplicate_strategy_id = StrategyId::from("OTHER-002");
6606
6607 node.add_strategy(TestStrategy::new(StrategyConfig {
6608 strategy_id: Some(strategy_id),
6609 external_order_instrument_ids: Some(vec![instrument_id]),
6610 ..Default::default()
6611 }))
6612 .unwrap();
6613
6614 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
6615 strategy_id: Some(duplicate_strategy_id),
6616 external_order_instrument_ids: Some(vec![instrument_id]),
6617 ..Default::default()
6618 }));
6619
6620 assert!(result.is_err());
6621 assert!(
6622 result
6623 .unwrap_err()
6624 .to_string()
6625 .contains("already exists for CLAIMS-001")
6626 );
6627 assert_eq!(
6628 node.exec_manager.get_external_order_claim(&instrument_id),
6629 Some(strategy_id)
6630 );
6631
6632 {
6633 let exec_engine = node.kernel().exec_engine.borrow();
6634 assert_eq!(
6635 exec_engine.get_external_order_claim(&instrument_id),
6636 Some(strategy_id)
6637 );
6638 }
6639 }
6640
6641 #[rstest]
6642 fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
6643 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6644 .unwrap()
6645 .with_reconciliation(false)
6646 .with_delay_post_stop_secs(0)
6647 .with_timeout_connection(1)
6648 .build()
6649 .unwrap();
6650 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6651 let strategy_id = StrategyId::from("CLAIMS-001");
6652
6653 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
6654 strategy_id: Some(strategy_id),
6655 external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
6656 ..Default::default()
6657 }));
6658
6659 assert!(result.is_err());
6660 assert!(
6661 result
6662 .unwrap_err()
6663 .to_string()
6664 .contains("appears more than once for CLAIMS-001")
6665 );
6666 assert_eq!(
6667 node.exec_manager.get_external_order_claim(&instrument_id),
6668 None
6669 );
6670
6671 {
6672 let exec_engine = node.kernel().exec_engine.borrow();
6673 assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
6674 }
6675 }
6676
6677 #[rstest]
6678 fn test_add_strategy_failure_restores_external_order_claims() {
6679 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6680 .unwrap()
6681 .with_reconciliation(false)
6682 .with_delay_post_stop_secs(0)
6683 .with_timeout_connection(1)
6684 .build()
6685 .unwrap();
6686 let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
6687 let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
6688 let strategy_id = StrategyId::from("CLAIMS-001");
6689 node.register_external_order_claims(strategy_id, &[existing_instrument_id])
6690 .unwrap();
6691
6692 let mut strategy = TestStrategy::new(StrategyConfig {
6693 strategy_id: Some(strategy_id),
6694 external_order_instrument_ids: Some(vec![configured_instrument_id]),
6695 ..Default::default()
6696 });
6697
6698 strategy
6699 .core
6700 .register(
6701 node.trader_id(),
6702 node.kernel.clock(),
6703 node.kernel.cache.clone(),
6704 node.kernel.portfolio.clone(),
6705 )
6706 .unwrap();
6707
6708 let result = node.add_strategy(strategy);
6709
6710 assert!(result.is_err());
6711 assert!(
6712 result
6713 .unwrap_err()
6714 .to_string()
6715 .contains("already registered with trader")
6716 );
6717 assert_eq!(
6718 node.exec_manager
6719 .get_external_order_claim(&existing_instrument_id),
6720 Some(strategy_id)
6721 );
6722 assert_eq!(
6723 node.kernel
6724 .exec_engine
6725 .borrow()
6726 .get_external_order_claim(&existing_instrument_id),
6727 Some(strategy_id)
6728 );
6729 assert_eq!(
6730 node.exec_manager
6731 .get_external_order_claim(&configured_instrument_id),
6732 None
6733 );
6734 assert_eq!(
6735 node.kernel
6736 .exec_engine
6737 .borrow()
6738 .get_external_order_claim(&configured_instrument_id),
6739 None
6740 );
6741 }
6742
6743 #[rstest]
6744 fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
6745 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6746 .unwrap()
6747 .with_reconciliation(false)
6748 .build()
6749 .unwrap();
6750 let exec_engine = node.kernel.exec_engine.clone();
6751 let _engine_borrow = exec_engine.borrow_mut();
6752
6753 node.add_strategy(TestStrategy::new(StrategyConfig {
6754 strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
6755 ..Default::default()
6756 }))
6757 .unwrap();
6758 }
6759
6760 #[rstest]
6761 fn test_add_strategy_registers_configured_hedging_oms_type() {
6762 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6763 .unwrap()
6764 .with_reconciliation(false)
6765 .with_delay_post_stop_secs(0)
6766 .with_timeout_connection(1)
6767 .build()
6768 .unwrap();
6769 let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
6770
6771 node.add_strategy(TestStrategy::new(StrategyConfig {
6772 strategy_id: Some(strategy_id),
6773 oms_type: Some(OmsType::Hedging),
6774 ..Default::default()
6775 }))
6776 .unwrap();
6777
6778 let instrument = crypto_perpetual_ethusdt();
6779 let instrument_id = instrument.id();
6780 let client_id = ClientId::from("STUB");
6781
6782 node.kernel
6783 .cache
6784 .borrow_mut()
6785 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6786 .unwrap();
6787 node.kernel
6788 .exec_engine
6789 .borrow_mut()
6790 .register_client(Box::new(StubExecutionClient::new(
6791 client_id,
6792 AccountId::from("TEST-ACCOUNT"),
6793 instrument_id.venue,
6794 OmsType::Netting,
6795 None,
6796 )))
6797 .unwrap();
6798
6799 let order = OrderTestBuilder::new(OrderType::Market)
6800 .trader_id(node.trader_id())
6801 .strategy_id(strategy_id)
6802 .instrument_id(instrument_id)
6803 .quantity(Quantity::from("1.000"))
6804 .build();
6805 let position_id = PositionId::new("CUSTOM-POSITION-001");
6806
6807 node.kernel
6808 .cache
6809 .borrow_mut()
6810 .add_order(order.clone(), Some(position_id), Some(client_id), true)
6811 .unwrap();
6812
6813 let submit_order = SubmitOrder::new(
6814 order.trader_id(),
6815 Some(client_id),
6816 strategy_id,
6817 instrument_id,
6818 order.client_order_id(),
6819 order.init_event().clone(),
6820 order.exec_algorithm_id(),
6821 Some(position_id),
6822 None,
6823 UUID4::new(),
6824 UnixNanos::default(),
6825 None,
6826 );
6827
6828 node.kernel
6829 .exec_engine
6830 .borrow()
6831 .execute(TradingCommand::SubmitOrder(submit_order));
6832
6833 let exec_engine = node.kernel.exec_engine.borrow();
6834 let cache = exec_engine.cache().borrow();
6835 let cached_order = cache
6836 .order(&order.client_order_id())
6837 .expect("Order should be cached");
6838
6839 assert_eq!(cached_order.status(), OrderStatus::Initialized);
6840 }
6841
6842 #[cfg(all(feature = "simulation", madsim))]
6843 async fn advance_clock(d: Duration) {
6844 madsim::time::advance(d);
6845 madsim::task::yield_now().await;
6846 }
6847
6848 #[cfg(not(all(feature = "simulation", madsim)))]
6849 async fn advance_clock(d: Duration) {
6850 tokio::time::advance(d).await;
6851 }
6852
6853 #[rstest]
6854 #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
6855 #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
6856 #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
6857 #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
6858 #[case::unapplied_cancel_default(
6859 SubmissionRecoveryPolicy::ResolveLocally,
6860 false,
6861 Some(OrderStatus::PendingCancel),
6862 false
6863 )]
6864 #[case::unapplied_modify_default(
6865 SubmissionRecoveryPolicy::ResolveLocally,
6866 false,
6867 Some(OrderStatus::PendingUpdate),
6868 false
6869 )]
6870 #[case::unapplied_cancel_diagnostics(
6871 SubmissionRecoveryPolicy::RetainUnresolved,
6872 false,
6873 Some(OrderStatus::PendingCancel),
6874 false
6875 )]
6876 #[case::unapplied_modify_diagnostics(
6877 SubmissionRecoveryPolicy::RetainUnresolved,
6878 false,
6879 Some(OrderStatus::PendingUpdate),
6880 false
6881 )]
6882 #[case::first_fill_cancel_default(
6883 SubmissionRecoveryPolicy::ResolveLocally,
6884 false,
6885 Some(OrderStatus::PendingCancel),
6886 true
6887 )]
6888 #[case::first_fill_modify_default(
6889 SubmissionRecoveryPolicy::ResolveLocally,
6890 false,
6891 Some(OrderStatus::PendingUpdate),
6892 true
6893 )]
6894 #[case::first_fill_cancel_diagnostics(
6895 SubmissionRecoveryPolicy::RetainUnresolved,
6896 false,
6897 Some(OrderStatus::PendingCancel),
6898 true
6899 )]
6900 #[case::first_fill_modify_diagnostics(
6901 SubmissionRecoveryPolicy::RetainUnresolved,
6902 false,
6903 Some(OrderStatus::PendingUpdate),
6904 true
6905 )]
6906 #[cfg_attr(
6907 not(all(feature = "simulation", madsim)),
6908 tokio::test(start_paused = true)
6909 )]
6910 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6911 async fn test_submission_registry_publishes_after_native_resolution(
6912 #[case] policy: SubmissionRecoveryPolicy,
6913 #[case] order_list: bool,
6914 #[case] pending_status: Option<OrderStatus>,
6915 #[case] first_fill: bool,
6916 ) {
6917 let config = LiveNodeConfig {
6918 exec_engine: crate::config::LiveExecutionEngineConfig {
6919 inflight_check_threshold_ms: 100,
6920 inflight_check_retries: 2,
6921 submission_recovery_policy: policy,
6922 ..Default::default()
6923 },
6924 ..Default::default()
6925 };
6926 let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
6927 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6928 let client_id = ClientId::from("TEST-SUBMISSION");
6929 let account_id = AccountId::from("TEST-001");
6930 let strategy_id = StrategyId::from("SUBMISSION-001");
6931 let account = AccountAny::Margin(MarginAccount::new(
6932 AccountState::new(
6933 account_id,
6934 AccountType::Margin,
6935 vec![AccountBalance::new(
6936 Money::from("1000000 USDT"),
6937 Money::from("0 USDT"),
6938 Money::from("1000000 USDT"),
6939 )],
6940 Vec::new(),
6941 true,
6942 UUID4::new(),
6943 UnixNanos::default(),
6944 UnixNanos::default(),
6945 Some(Currency::USDT()),
6946 ),
6947 true,
6948 ));
6949 node.kernel.cache.borrow_mut().add_account(account).unwrap();
6950 node.kernel
6951 .cache
6952 .borrow_mut()
6953 .add_instrument(instrument.clone())
6954 .unwrap();
6955 let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
6956 .map(|index| {
6957 OrderTestBuilder::new(OrderType::Limit)
6958 .trader_id(node.trader_id())
6959 .strategy_id(strategy_id)
6960 .client_order_id(ClientOrderId::from(
6961 format!("O-SUBMISSION-{index}").as_str(),
6962 ))
6963 .instrument_id(instrument.id())
6964 .quantity(Quantity::from("1.000"))
6965 .price(Price::from("100.00"))
6966 .build()
6967 })
6968 .collect();
6969 let command = if order_list {
6970 TradingCommand::SubmitOrderList(SubmitOrderList::new(
6971 node.trader_id(),
6972 Some(client_id),
6973 strategy_id,
6974 OrderList::new(
6975 OrderListId::from("OL-SUBMISSION"),
6976 instrument.id(),
6977 strategy_id,
6978 orders.iter().map(Order::client_order_id).collect(),
6979 UnixNanos::default(),
6980 ),
6981 orders
6982 .iter()
6983 .map(|order| order.init_event().clone())
6984 .collect(),
6985 None,
6986 None,
6987 None,
6988 UUID4::new(),
6989 UnixNanos::default(),
6990 None,
6991 ))
6992 } else {
6993 TradingCommand::SubmitOrder(SubmitOrder::from_order(
6994 &orders[0],
6995 node.trader_id(),
6996 Some(client_id),
6997 None,
6998 UUID4::new(),
6999 UnixNanos::default(),
7000 ))
7001 };
7002 node.observe_exec_command_before_dispatch(&command);
7003
7004 for order in &orders {
7005 let submitted = TestOrderEventStubs::submitted(order, account_id);
7006 node.kernel
7007 .cache
7008 .borrow_mut()
7009 .add_order(order.clone(), None, Some(client_id), false)
7010 .unwrap();
7011 node.process_exec_event(ExecutionEvent::Order(submitted));
7012 }
7013
7014 let expected_status = match pending_status {
7015 Some(status) if !first_fill => status,
7016 Some(_) => OrderStatus::Canceled,
7017 None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7018 None => OrderStatus::Rejected,
7019 };
7020 let diagnostics = Rc::new(RefCell::new(Vec::new()));
7021 let handler = ShareableMessageHandler::from_typed({
7022 let diagnostics = diagnostics.clone();
7023 let cache = node.kernel.cache.clone();
7024 move |diagnostic: &SubmissionRecoveryExhausted| {
7025 let cache = cache.borrow_mut();
7026 let order = cache.order(&diagnostic.client_order_id).unwrap();
7027 assert_eq!(order.status(), expected_status);
7028 assert!(diagnostic.ts_event >= order.ts_last());
7029 diagnostics.borrow_mut().push(diagnostic.clone());
7030 }
7031 });
7032 let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7033 msgbus::subscribe_any(topic.into(), handler.clone(), None);
7034 let queries = Rc::new(RefCell::new(Vec::new()));
7035 msgbus::register_trading_command_endpoint(
7036 MessagingSwitchboard::exec_engine_execute(),
7037 TypedIntoHandler::from({
7038 let queries = queries.clone();
7039 move |command: TradingCommand| queries.borrow_mut().push(command)
7040 }),
7041 );
7042
7043 let mut last_inflight_check = dst::time::Instant::now();
7044 let mut last_open_check = last_inflight_check;
7045 let mut last_position_check = last_inflight_check;
7046 let mut open_order_report_task = None;
7047 let mut targeted_order_report_task = None;
7048 let mut position_report_task = None;
7049
7050 for check in 1..=3 {
7051 advance_clock(Duration::from_millis(101)).await;
7052 node.run_reconciliation_checks(
7053 dst::time::Instant::now(),
7054 ReconciliationCheckIntervals {
7055 inflight: Duration::from_millis(100),
7056 open: Duration::ZERO,
7057 position: Duration::ZERO,
7058 },
7059 &mut ReconciliationCheckState {
7060 last_inflight_check: &mut last_inflight_check,
7061 last_open_check: &mut last_open_check,
7062 last_position_check: &mut last_position_check,
7063 open_order_report_task: &mut open_order_report_task,
7064 targeted_order_report_task: &mut targeted_order_report_task,
7065 position_report_task: &mut position_report_task,
7066 },
7067 );
7068
7069 if check == 1 {
7070 assert!(diagnostics.borrow().is_empty());
7071
7072 if let Some(pending_status) = pending_status {
7073 let order = &orders[0];
7074 let pending = if pending_status == OrderStatus::PendingCancel {
7075 OrderEventAny::PendingCancel(
7076 OrderPendingCancelSpec::builder()
7077 .trader_id(node.trader_id())
7078 .strategy_id(strategy_id)
7079 .instrument_id(instrument.id())
7080 .client_order_id(order.client_order_id())
7081 .account_id(account_id)
7082 .build(),
7083 )
7084 } else {
7085 OrderEventAny::PendingUpdate(
7086 OrderPendingUpdateSpec::builder()
7087 .trader_id(node.trader_id())
7088 .strategy_id(strategy_id)
7089 .instrument_id(instrument.id())
7090 .client_order_id(order.client_order_id())
7091 .account_id(account_id)
7092 .build(),
7093 )
7094 };
7095 node.process_exec_event(ExecutionEvent::Order(pending));
7096 node.exec_manager.register_inflight(order.client_order_id());
7097 if first_fill {
7098 let cached = node
7099 .kernel
7100 .cache
7101 .borrow()
7102 .order(&order.client_order_id())
7103 .unwrap()
7104 .clone();
7105 let OrderEventAny::Filled(mut fill) =
7106 OrderFilledTestBuilder::new(&cached, &instrument)
7107 .last_qty(Quantity::from("0.400"))
7108 .last_px(Price::from("100.00"))
7109 .commission(Money::from("0 USDT"))
7110 .without_position_id()
7111 .build()
7112 else {
7113 unreachable!()
7114 };
7115 fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7116 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7117 } else {
7118 let report = OrderStatusReport::new(
7119 account_id,
7120 instrument.id(),
7121 Some(order.client_order_id()),
7122 VenueOrderId::from("V-SUBMISSION"),
7123 Some(order.order_side()),
7124 order.order_type(),
7125 order.time_in_force(),
7126 OrderStatus::Submitted,
7127 order.quantity(),
7128 Quantity::zero(3),
7129 UnixNanos::default(),
7130 UnixNanos::default(),
7131 UnixNanos::default(),
7132 None,
7133 )
7134 .with_price(Price::from("100.00"));
7135 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
7136 Box::new(report),
7137 )));
7138 }
7139 let cache = node.kernel.cache.borrow();
7140 let cached = cache.order(&order.client_order_id()).unwrap();
7141 assert_eq!(cached.status(), pending_status);
7142 assert_eq!(
7143 cached.filled_qty(),
7144 if first_fill {
7145 Quantity::from("0.400")
7146 } else {
7147 Quantity::zero(3)
7148 }
7149 );
7150 assert!(
7151 !cached
7152 .events()
7153 .iter()
7154 .any(|event| matches!(event, OrderEventAny::Accepted(_)))
7155 );
7156 }
7157 }
7158 }
7159
7160 assert_eq!(
7161 queries.borrow().len(),
7162 orders.len()
7163 + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
7164 );
7165
7166 for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
7167 assert!(matches!(command, TradingCommand::QueryOrder(query)
7168 if query.client_order_id == order.client_order_id()
7169 && query.client_id == Some(client_id)));
7170 assert_eq!(
7171 node.kernel
7172 .cache
7173 .borrow()
7174 .order(&order.client_order_id())
7175 .unwrap()
7176 .status(),
7177 if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
7178 pending_status.unwrap()
7179 } else {
7180 expected_status
7181 }
7182 );
7183 }
7184 let diagnostics = diagnostics.borrow();
7185 assert_eq!(
7186 diagnostics.len(),
7187 if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
7188 orders.len()
7189 } else {
7190 0
7191 }
7192 );
7193
7194 for (diagnostic, order) in diagnostics.iter().zip(&orders) {
7195 assert_eq!(diagnostic.trader_id, node.trader_id());
7196 assert_eq!(diagnostic.client_id, Some(client_id));
7197 assert_eq!(diagnostic.strategy_id, strategy_id);
7198 assert_eq!(diagnostic.instrument_id, instrument.id());
7199 assert_eq!(diagnostic.client_order_id, order.client_order_id());
7200 assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
7201 assert_eq!(diagnostic.retry_count, 2);
7202 }
7203 msgbus::unsubscribe_any(topic.into(), &handler);
7204 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7205 }
7206
7207 #[derive(Debug)]
7208 struct SubmissionCallbackStrategy {
7209 core: StrategyCore,
7210 cancel: bool,
7211 batch: bool,
7212 callback: &'static str,
7213 client_id: ClientId,
7214 }
7215
7216 impl DataActor for SubmissionCallbackStrategy {}
7217
7218 impl SubmissionCallbackStrategy {
7219 fn queue_command(&mut self, client_order_id: ClientOrderId) {
7220 match (self.cancel, self.batch) {
7221 (true, true) => {
7222 self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
7223 .unwrap();
7224 }
7225 (true, false) => {
7226 self.cancel_order(client_order_id, Some(self.client_id), None)
7227 .unwrap();
7228 }
7229 (false, true) => {
7230 self.modify_orders(
7231 vec![(client_order_id, None, Some(Price::from("101.00")), None)],
7232 Some(self.client_id),
7233 None,
7234 )
7235 .unwrap();
7236 }
7237 (false, false) => self
7238 .modify_order(
7239 client_order_id,
7240 None,
7241 Some(Price::from("101.00")),
7242 None,
7243 Some(self.client_id),
7244 None,
7245 )
7246 .unwrap(),
7247 }
7248 }
7249 }
7250
7251 nautilus_strategy!(SubmissionCallbackStrategy, {
7252 fn on_order_accepted(&mut self, event: OrderAccepted) {
7253 if self.callback == "accepted" {
7254 self.queue_command(event.client_order_id);
7255 }
7256 }
7257
7258 fn on_order_updated(&mut self, event: OrderUpdated) {
7259 if self.callback == "updated" {
7260 self.queue_command(event.client_order_id);
7261 }
7262 }
7263 });
7264
7265 #[rstest]
7266 #[case::native(false, "accepted")]
7267 #[case::report(true, "accepted")]
7268 #[case::updated(false, "updated")]
7269 #[cfg_attr(
7270 not(all(feature = "simulation", madsim)),
7271 tokio::test(start_paused = true)
7272 )]
7273 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7274 async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
7275 #[values(
7276 SubmissionRecoveryPolicy::ResolveLocally,
7277 SubmissionRecoveryPolicy::RetainUnresolved
7278 )]
7279 policy: SubmissionRecoveryPolicy,
7280 #[values(false, true)] cancel: bool,
7281 #[values(false, true)] batch: bool,
7282 #[case] report: bool,
7283 #[case] callback: &'static str,
7284 ) {
7285 let config = LiveNodeConfig {
7286 exec_engine: crate::config::LiveExecutionEngineConfig {
7287 inflight_check_threshold_ms: 100,
7288 inflight_check_retries: 2,
7289 submission_recovery_policy: policy,
7290 ..Default::default()
7291 },
7292 ..Default::default()
7293 };
7294 let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
7295 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7296 let account_id = AccountId::from("TEST-001");
7297 let strategy_id = StrategyId::from("CALLBACK-001");
7298 let client_id = ClientId::from("TEST-CALLBACK");
7299 let account = AccountAny::Margin(MarginAccount::new(
7300 AccountState::new(
7301 account_id,
7302 AccountType::Margin,
7303 vec![AccountBalance::new(
7304 Money::from("1000000 USDT"),
7305 Money::from("0 USDT"),
7306 Money::from("1000000 USDT"),
7307 )],
7308 Vec::new(),
7309 true,
7310 UUID4::new(),
7311 UnixNanos::default(),
7312 UnixNanos::default(),
7313 Some(Currency::USDT()),
7314 ),
7315 true,
7316 ));
7317 node.kernel.cache.borrow_mut().add_account(account).unwrap();
7318 node.kernel
7319 .cache
7320 .borrow_mut()
7321 .add_instrument(instrument.clone())
7322 .unwrap();
7323 node.kernel
7324 .exec_engine
7325 .borrow_mut()
7326 .register_client(Box::new(StubExecutionClient::new(
7327 client_id,
7328 account_id,
7329 instrument.id().venue,
7330 OmsType::Netting,
7331 None,
7332 )))
7333 .unwrap();
7334 node.add_strategy(SubmissionCallbackStrategy {
7335 core: StrategyCore::new(StrategyConfig {
7336 strategy_id: Some(strategy_id),
7337 ..Default::default()
7338 }),
7339 cancel,
7340 batch,
7341 callback,
7342 client_id,
7343 })
7344 .unwrap();
7345 node.kernel.trader.borrow_mut().initialize().unwrap();
7346 node.kernel.start_trader().unwrap();
7347 let runner = node.runner.take().unwrap();
7348 runner.bind_senders_for_node(node.handle.clone());
7349 let mut channels = runner.take_channels();
7350 let order = OrderTestBuilder::new(OrderType::Limit)
7351 .trader_id(node.trader_id())
7352 .strategy_id(strategy_id)
7353 .client_order_id(ClientOrderId::from("O-CALLBACK"))
7354 .instrument_id(instrument.id())
7355 .quantity(Quantity::from("1.000"))
7356 .price(Price::from("100.00"))
7357 .build();
7358 let client_order_id = order.client_order_id();
7359 let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
7360 &order,
7361 node.trader_id(),
7362 Some(client_id),
7363 None,
7364 UUID4::new(),
7365 UnixNanos::default(),
7366 ));
7367 node.observe_exec_command_before_dispatch(&submit);
7368 node.kernel
7369 .cache
7370 .borrow_mut()
7371 .add_order(order.clone(), None, Some(client_id), false)
7372 .unwrap();
7373 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
7374 &order, account_id,
7375 )));
7376 advance_clock(Duration::from_millis(101)).await;
7377 let first = node.exec_manager.check_inflight_orders();
7378 assert_eq!(first.queries.len(), 1);
7379 assert!(first.events.is_empty());
7380
7381 advance_clock(Duration::from_millis(101)).await;
7382 let accepted =
7383 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
7384
7385 if callback == "updated" {
7386 let mut update = OrderUpdatedSpec::builder()
7387 .trader_id(node.trader_id())
7388 .strategy_id(strategy_id)
7389 .instrument_id(instrument.id())
7390 .client_order_id(client_order_id)
7391 .account_id(account_id)
7392 .quantity(order.quantity())
7393 .price(order.price().unwrap())
7394 .build();
7395 update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
7396 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
7397 } else if report {
7398 let accepted = OrderStatusReport::new(
7399 account_id,
7400 instrument.id(),
7401 Some(client_order_id),
7402 VenueOrderId::from("V-CALLBACK"),
7403 Some(order.order_side()),
7404 order.order_type(),
7405 order.time_in_force(),
7406 OrderStatus::Accepted,
7407 order.quantity(),
7408 Quantity::zero(3),
7409 UnixNanos::default(),
7410 UnixNanos::default(),
7411 UnixNanos::default(),
7412 None,
7413 )
7414 .with_price(Price::from("100.00"));
7415 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
7416 accepted,
7417 ))));
7418 } else {
7419 node.process_exec_event(ExecutionEvent::Order(accepted));
7420 }
7421
7422 let pending_status = if cancel {
7423 OrderStatus::PendingCancel
7424 } else {
7425 OrderStatus::PendingUpdate
7426 };
7427 assert_eq!(channels.exec_cmd_rx.len(), 1);
7428 assert_eq!(
7429 node.kernel
7430 .cache
7431 .borrow()
7432 .order(&client_order_id)
7433 .unwrap()
7434 .status(),
7435 pending_status
7436 );
7437
7438 for _ in 0..3 {
7440 advance_clock(Duration::from_millis(101)).await;
7441 let maintenance = node.exec_manager.check_inflight_orders();
7442 assert!(maintenance.events.is_empty());
7443 assert!(maintenance.queries.is_empty());
7444 assert!(
7445 node.exec_manager
7446 .take_submission_recovery_exhaustions()
7447 .is_empty()
7448 );
7449 }
7450
7451 let command = channels.exec_cmd_rx.try_recv().unwrap();
7452 node.process_exec_command(command);
7453 assert!(channels.exec_cmd_rx.is_empty());
7454 let immediate = node.exec_manager.check_inflight_orders();
7455 assert!(immediate.events.is_empty());
7456 assert!(immediate.queries.is_empty());
7457 advance_clock(Duration::from_millis(101)).await;
7458 let first_command_check = node.exec_manager.check_inflight_orders();
7459 assert!(first_command_check.events.is_empty());
7460 assert_eq!(first_command_check.queries.len(), 1);
7461 advance_clock(Duration::from_millis(101)).await;
7462 let exhausted = node.exec_manager.check_inflight_orders();
7463 assert!(exhausted.queries.is_empty());
7464 assert!(
7465 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
7466 if event.client_order_id == client_order_id)
7467 );
7468 assert!(
7469 node.exec_manager
7470 .take_submission_recovery_exhaustions()
7471 .is_empty()
7472 );
7473 node.process_reconciliation_events(&exhausted.events);
7474 assert_eq!(
7475 node.kernel
7476 .cache
7477 .borrow()
7478 .order(&client_order_id)
7479 .unwrap()
7480 .status(),
7481 OrderStatus::Canceled
7482 );
7483 }
7484
7485 #[cfg_attr(
7486 not(all(feature = "simulation", madsim)),
7487 tokio::test(start_paused = true)
7488 )]
7489 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7490 async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
7491 let last = dst::time::Instant::now();
7492 let interval = Duration::from_millis(100);
7493
7494 assert!(!reconciliation_check_due(last, last, Duration::ZERO));
7495 assert!(!reconciliation_check_due(last, last, interval));
7496
7497 advance_clock(Duration::from_millis(99)).await;
7498 let before_interval = dst::time::Instant::now();
7499 assert!(!reconciliation_check_due(before_interval, last, interval));
7500
7501 advance_clock(Duration::from_millis(1)).await;
7502 let at_interval = dst::time::Instant::now();
7503 assert!(reconciliation_check_due(at_interval, last, interval));
7504
7505 assert!(!reconciliation_check_due(last, at_interval, interval));
7506 }
7507
7508 #[cfg_attr(
7509 not(all(feature = "simulation", madsim)),
7510 tokio::test(start_paused = true)
7511 )]
7512 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7513 async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
7514 let config = LiveNodeConfig {
7515 exec_engine: crate::config::LiveExecutionEngineConfig {
7516 reconciliation: true,
7517 open_check_interval_secs: Some(1.0),
7518 position_check_interval_secs: Some(1.0),
7519 max_single_order_queries_per_cycle: 5,
7520 ..Default::default()
7521 },
7522 ..Default::default()
7523 };
7524
7525 let mut node =
7526 LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
7527 let client_id = ClientId::from("TEST-QUERY");
7528 let account_id = AccountId::from("TEST-QUERY-001");
7529
7530 let trading_commands = Rc::new(RefCell::new(Vec::new()));
7531 msgbus::register_trading_command_endpoint(
7532 MessagingSwitchboard::exec_engine_execute(),
7533 TypedIntoHandler::from({
7534 let trading_commands = trading_commands.clone();
7535 move |command: TradingCommand| {
7536 trading_commands.borrow_mut().push(command);
7537 }
7538 }),
7539 );
7540
7541 let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
7542 let instrument = crypto_perpetual_ethusdt();
7543 let instrument_id = instrument.id();
7544 let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
7545
7546 node.kernel
7547 .cache
7548 .borrow_mut()
7549 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7550 .unwrap();
7551 insert_accepted_limit_order_in_node(
7552 &node,
7553 account_id,
7554 client_id,
7555 instrument_id,
7556 client_order_id,
7557 venue_order_id,
7558 );
7559
7560 let last = dst::time::Instant::now();
7561 advance_clock(Duration::from_nanos(1)).await;
7562 let now = dst::time::Instant::now();
7563 let mut last_inflight_check = last;
7564 let mut last_open_check = last;
7565 let mut last_position_check = last;
7566 let mut open_order_report_task = None;
7567 let mut targeted_order_report_task = None;
7568 let mut position_report_task = None;
7569
7570 node.run_reconciliation_checks(
7571 now,
7572 ReconciliationCheckIntervals {
7573 inflight: Duration::ZERO,
7574 open: Duration::from_nanos(1),
7575 position: Duration::ZERO,
7576 },
7577 &mut ReconciliationCheckState {
7578 last_inflight_check: &mut last_inflight_check,
7579 last_open_check: &mut last_open_check,
7580 last_position_check: &mut last_position_check,
7581 open_order_report_task: &mut open_order_report_task,
7582 targeted_order_report_task: &mut targeted_order_report_task,
7583 position_report_task: &mut position_report_task,
7584 },
7585 );
7586
7587 let commands = trading_commands.borrow();
7588
7589 assert!(commands.is_empty());
7590 assert!(open_order_report_task.is_none());
7591 assert!(targeted_order_report_task.is_none());
7592 assert!(position_report_task.is_none());
7593
7594 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7595 }
7596
7597 #[rstest]
7598 #[tokio::test]
7599 async fn test_recurring_report_failure_preserves_client_coverage(
7600 #[values(false, true)] positions: bool,
7601 #[values(false, true)] failure_first: bool,
7602 ) {
7603 let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
7604 let failed_id = ClientId::from("FAILED");
7605 let healthy_id = ClientId::from("HEALTHY");
7606
7607 for (client_id, outcome) in [
7608 (failed_id, FillReportClientOutcome::Failure),
7609 (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
7610 ] {
7611 node.exec_clients
7612 .push(LiveExecutionClient::new(Box::new(FillReportClient {
7613 client_id,
7614 account_id: AccountId::from("TEST-001"),
7615 venue: Venue::from("BINANCE"),
7616 outcome,
7617 commands: Rc::new(RefCell::new(Vec::new())),
7618 })));
7619 }
7620
7621 if !failure_first {
7622 node.exec_clients.reverse();
7623 }
7624
7625 let last = dst::time::Instant::now();
7626 let now = last + Duration::from_secs(1);
7627 let mut last_inflight = last;
7628 let mut last_open = last;
7629 let mut last_position = last;
7630 let mut open_task = None;
7631 let mut targeted_task = None;
7632 let mut position_task = None;
7633
7634 node.run_reconciliation_checks(
7635 now,
7636 ReconciliationCheckIntervals {
7637 inflight: Duration::ZERO,
7638 open: if positions {
7639 Duration::ZERO
7640 } else {
7641 Duration::from_secs(1)
7642 },
7643 position: if positions {
7644 Duration::from_secs(1)
7645 } else {
7646 Duration::ZERO
7647 },
7648 },
7649 &mut ReconciliationCheckState {
7650 last_inflight_check: &mut last_inflight,
7651 last_open_check: &mut last_open,
7652 last_position_check: &mut last_position,
7653 open_order_report_task: &mut open_task,
7654 targeted_order_report_task: &mut targeted_task,
7655 position_report_task: &mut position_task,
7656 },
7657 );
7658
7659 let (queried_clients, failed_clients) = if positions {
7660 let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
7661 position_task.unwrap().future.await
7662 else {
7663 panic!("position collection should complete despite one client failure");
7664 };
7665
7666 assert!(result.reports.is_empty());
7667 assert!(open_task.is_none());
7668 (result.queried_clients, result.failed_clients)
7669 } else {
7670 let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
7671 panic!("order collection should complete despite one client failure");
7672 };
7673
7674 assert!(result.reports.is_empty());
7675 assert!(position_task.is_none());
7676 (result.queried_clients, result.failed_clients)
7677 };
7678
7679 assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
7680 assert_eq!(failed_clients, IndexSet::from([failed_id]));
7681 assert!(targeted_task.is_none());
7682 }
7683
7684 #[rstest]
7685 #[case::inflight(true, false)]
7686 #[case::open(false, false)]
7687 #[case::positions(false, true)]
7688 #[tokio::test]
7689 async fn test_reconciliation_shutdown_preserves_pending_checks(
7690 #[case] inflight: bool,
7691 #[case] positions: bool,
7692 ) {
7693 let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
7694 node.handle.set_shutting_down();
7695 let last = dst::time::Instant::now();
7696 let now = last + Duration::from_secs(1);
7697 let mut last_inflight = last;
7698 let mut last_open = last;
7699 let mut last_position = last;
7700 let mut open_task = None;
7701 let mut targeted_task = None;
7702 let mut position_task = None;
7703
7704 node.run_reconciliation_checks(
7705 now,
7706 ReconciliationCheckIntervals {
7707 inflight: if inflight {
7708 Duration::from_secs(1)
7709 } else {
7710 Duration::ZERO
7711 },
7712 open: if !inflight && !positions {
7713 Duration::from_secs(1)
7714 } else {
7715 Duration::ZERO
7716 },
7717 position: if positions {
7718 Duration::from_secs(1)
7719 } else {
7720 Duration::ZERO
7721 },
7722 },
7723 &mut ReconciliationCheckState {
7724 last_inflight_check: &mut last_inflight,
7725 last_open_check: &mut last_open,
7726 last_position_check: &mut last_position,
7727 open_order_report_task: &mut open_task,
7728 targeted_order_report_task: &mut targeted_task,
7729 position_report_task: &mut position_task,
7730 },
7731 );
7732
7733 assert_eq!(
7734 (last_inflight, last_open, last_position),
7735 (last, last, last)
7736 );
7737 assert!(open_task.is_none());
7738 assert!(targeted_task.is_none());
7739 assert!(position_task.is_none());
7740 }
7741
7742 fn insert_accepted_limit_order_in_node(
7743 node: &LiveNode,
7744 account_id: AccountId,
7745 client_id: ClientId,
7746 instrument_id: InstrumentId,
7747 client_order_id: ClientOrderId,
7748 venue_order_id: VenueOrderId,
7749 ) {
7750 let order = OrderTestBuilder::new(OrderType::Limit)
7751 .client_order_id(client_order_id)
7752 .instrument_id(instrument_id)
7753 .quantity(Quantity::from("10.0"))
7754 .price(Price::from("100.0"))
7755 .build();
7756 let submitted = TestOrderEventStubs::submitted(&order, account_id);
7757 node.kernel
7758 .cache
7759 .borrow_mut()
7760 .add_order(order, None, Some(client_id), false)
7761 .unwrap();
7762 let order = node
7763 .kernel
7764 .cache
7765 .borrow_mut()
7766 .update_order(&submitted)
7767 .unwrap();
7768 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
7769 node.kernel
7770 .cache
7771 .borrow_mut()
7772 .update_order(&accepted)
7773 .unwrap();
7774 }
7775
7776 fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
7777 let config = LiveNodeConfig {
7778 exec_engine: crate::config::LiveExecutionEngineConfig {
7779 reconciliation: true,
7780 ..Default::default()
7781 },
7782 ..Default::default()
7783 };
7784
7785 let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
7786 let account_id = AccountId::from("TEST-001");
7787 let client_id = ClientId::from("TEST-RECENT-FILL");
7788 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
7789 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
7790 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7791 node.kernel
7792 .cache
7793 .borrow_mut()
7794 .add_instrument(instrument.clone())
7795 .unwrap();
7796 insert_accepted_limit_order_in_node(
7797 &node,
7798 account_id,
7799 client_id,
7800 instrument.id(),
7801 client_order_id,
7802 venue_order_id,
7803 );
7804 let order = node
7805 .kernel
7806 .cache
7807 .borrow()
7808 .order_owned(&client_order_id)
7809 .unwrap();
7810 let fill = TestOrderEventStubs::filled(
7811 &order,
7812 &instrument,
7813 Some(TradeId::from("T-RECENT-FILL")),
7814 None,
7815 Some(Price::from("100.0")),
7816 Some(Quantity::from("1.0")),
7817 Some(LiquiditySide::Taker),
7818 None,
7819 None,
7820 Some(account_id),
7821 );
7822
7823 (node, fill, instrument)
7824 }
7825
7826 fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
7827 let client_order_id = fill_report.client_order_id.unwrap();
7828 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7829 let order = node
7830 .kernel
7831 .cache
7832 .borrow()
7833 .order_owned(&client_order_id)
7834 .unwrap();
7835 let order_report = OrderStatusReport::new(
7836 fill_report.account_id,
7837 fill_report.instrument_id,
7838 Some(client_order_id),
7839 fill_report.venue_order_id,
7840 OrderSide::Buy.into(),
7841 OrderType::Limit,
7842 TimeInForce::Gtc,
7843 OrderStatus::PartiallyFilled,
7844 Quantity::from("10.0"),
7845 Quantity::from("2.0"),
7846 UnixNanos::from(1_000),
7847 UnixNanos::from(1_000),
7848 UnixNanos::from(1_000),
7849 None,
7850 )
7851 .with_avg_px(dec!(100.0));
7852 let inferred = create_inferred_fill_for_qty(
7853 &order,
7854 &order_report,
7855 &fill_report.account_id,
7856 &instrument,
7857 Quantity::from("1.0"),
7858 UnixNanos::from(1_000),
7859 None,
7860 )
7861 .unwrap();
7862
7863 node.process_reconciliation_events(&[inferred]);
7864 }
7865
7866 #[rstest]
7867 #[case::rejected(false, "10.0", "10.0", "1.0")]
7868 #[case::dust(true, "9.004", "10.004", "10.004")]
7869 #[case::price_improvement(true, "12.5", "13.5", "13.5")]
7870 fn test_position_check_overfill_raises_before_fill_when_client_allows(
7871 #[case] allow: bool,
7872 #[case] fill_qty: &str,
7873 #[case] expected_qty: &str,
7874 #[case] expected_filled: &str,
7875 ) {
7876 let (mut node, venue_report, mut fill_report) =
7877 position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
7878 fill_report.commission = Money::from("1.25 USDT");
7879 let key = (fill_report.instrument_id, fill_report.account_id);
7880
7881 let client = StubExecutionClient::new(
7882 ClientId::from("POSITION-FILLS"),
7883 AccountId::from("TEST-001"),
7884 fill_report.instrument_id.venue,
7885 OmsType::Netting,
7886 None,
7887 );
7888
7889 let client = if allow {
7890 client.with_allows_reconciliation_overfill()
7891 } else {
7892 client
7893 };
7894
7895 node.kernel
7896 .exec_engine
7897 .borrow_mut()
7898 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
7899 .unwrap();
7900
7901 let position_result = position_report_result(&node, venue_report);
7902 node.handle_position_fill_report_result(PositionFillReportResult {
7903 position_result,
7904 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
7905 successful_keys: IndexSet::from([key]),
7906 });
7907
7908 let order = node
7909 .kernel
7910 .cache
7911 .borrow()
7912 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
7913 .unwrap();
7914 assert_eq!(order.quantity(), Quantity::from(expected_qty));
7915 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
7916 let applied = order
7917 .trade_ids()
7918 .iter()
7919 .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
7920 assert_eq!(applied, allow);
7921 let revision = node.exec_manager.position_activity_revision(&key);
7922
7923 if !allow {
7924 return;
7925 }
7926
7927 assert_eq!(revision, 2);
7928
7929 {
7930 let cache = node.kernel.cache.borrow();
7931 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
7932 assert_eq!(positions.len(), 1);
7933 assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
7934 assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
7935 assert_eq!(positions[0].trade_ids.len(), 2);
7936 }
7937
7938 let replay_report = position_report_result(
7939 &node,
7940 PositionStatusReport::new(
7941 key.1,
7942 key.0,
7943 PositionSide::Long,
7944 Quantity::from(expected_qty),
7945 UnixNanos::from(2_000),
7946 UnixNanos::from(2_000),
7947 None,
7948 None,
7949 Some(dec!(100.0)),
7950 ),
7951 );
7952 node.handle_position_fill_report_result(PositionFillReportResult {
7953 position_result: replay_report,
7954 reports: IndexMap::from([(key, vec![fill_report])]),
7955 successful_keys: IndexSet::from([key]),
7956 });
7957
7958 let order = node
7959 .kernel
7960 .cache
7961 .borrow()
7962 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
7963 .unwrap();
7964 assert_eq!(order.quantity(), Quantity::from(expected_qty));
7965 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
7966 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
7967 }
7968
7969 #[rstest]
7970 fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
7971 let (mut node, venue_report, mut overfill) =
7972 position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
7973 overfill.commission = Money::from("0.20 USDT");
7974 let key = (overfill.instrument_id, overfill.account_id);
7975 let client = StubExecutionClient::new(
7976 ClientId::from("POSITION-FILLS"),
7977 AccountId::from("TEST-001"),
7978 overfill.instrument_id.venue,
7979 OmsType::Netting,
7980 None,
7981 )
7982 .with_allows_reconciliation_overfill();
7983 node.kernel
7984 .exec_engine
7985 .borrow_mut()
7986 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
7987 .unwrap();
7988
7989 let mut fitting = overfill.clone();
7990 fitting.trade_id = TradeId::from("T-POSITION-FITTING");
7991 fitting.last_qty = Quantity::from("8.0");
7992 fitting.commission = Money::from("0.80 USDT");
7993 let position_result = position_report_result(&node, venue_report);
7994 node.handle_position_fill_report_result(PositionFillReportResult {
7995 position_result,
7996 reports: IndexMap::from([(key, vec![fitting, overfill])]),
7997 successful_keys: IndexSet::from([key]),
7998 });
7999
8000 let order = node
8001 .kernel
8002 .cache
8003 .borrow()
8004 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8005 .unwrap();
8006 assert_eq!(order.quantity(), Quantity::from("11.0"));
8007 assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8008 assert_eq!(order.trade_ids().len(), 3);
8009 assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8010 {
8011 let cache = node.kernel.cache.borrow();
8012 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8013 assert_eq!(positions.len(), 1);
8014 assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8015 assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8016 }
8017 }
8018
8019 #[rstest]
8020 fn test_position_check_second_overfill_does_not_raise_filled_order() {
8021 let (mut node, venue_report, mut first) =
8022 position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8023 first.commission = Money::from("1.20 USDT");
8024 let key = (first.instrument_id, first.account_id);
8025 let client = StubExecutionClient::new(
8026 ClientId::from("POSITION-FILLS"),
8027 AccountId::from("TEST-001"),
8028 first.instrument_id.venue,
8029 OmsType::Netting,
8030 None,
8031 )
8032 .with_allows_reconciliation_overfill();
8033 node.kernel
8034 .exec_engine
8035 .borrow_mut()
8036 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8037 .unwrap();
8038
8039 let mut second = first.clone();
8040 second.trade_id = TradeId::from("T-POSITION-LATER");
8041 second.last_qty = Quantity::from("1.0");
8042 second.commission = Money::from("0.10 USDT");
8043 second.ts_event = UnixNanos::from(1_001);
8044 let position_result = position_report_result(&node, venue_report);
8045 node.handle_position_fill_report_result(PositionFillReportResult {
8046 position_result,
8047 reports: IndexMap::from([(key, vec![first, second])]),
8048 successful_keys: IndexSet::from([key]),
8049 });
8050
8051 let order = node
8052 .kernel
8053 .cache
8054 .borrow()
8055 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8056 .unwrap();
8057 assert_eq!(order.quantity(), Quantity::from("13.0"));
8058 assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8059 assert_eq!(order.status(), OrderStatus::Filled);
8060 assert_eq!(order.trade_ids().len(), 2);
8061 assert!(
8062 order
8063 .trade_ids()
8064 .iter()
8065 .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8066 );
8067 {
8068 let cache = node.kernel.cache.borrow();
8069 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8070 assert_eq!(positions.len(), 1);
8071 assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8072 assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8073 }
8074 }
8075
8076 #[rstest]
8077 fn test_allow_overfills_applies_fill_without_raising_quantity() {
8078 let config = LiveNodeConfig {
8079 exec_engine: crate::config::LiveExecutionEngineConfig {
8080 reconciliation: true,
8081 allow_overfills: true,
8082 ..Default::default()
8083 },
8084 ..Default::default()
8085 };
8086
8087 let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8088 let account_id = AccountId::from("TEST-001");
8089 let client_id = ClientId::from("POSITION-FILLS");
8090 let client_order_id = ClientOrderId::from("O-ALLOW");
8091 let venue_order_id = VenueOrderId::from("V-ALLOW");
8092 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8093 node.kernel
8094 .cache
8095 .borrow_mut()
8096 .add_instrument(instrument.clone())
8097 .unwrap();
8098 insert_accepted_limit_order_in_node(
8099 &node,
8100 account_id,
8101 client_id,
8102 instrument.id(),
8103 client_order_id,
8104 venue_order_id,
8105 );
8106 let client = StubExecutionClient::new(
8107 client_id,
8108 account_id,
8109 instrument.id().venue,
8110 OmsType::Netting,
8111 None,
8112 )
8113 .with_allows_reconciliation_overfill();
8114 node.kernel
8115 .exec_engine
8116 .borrow_mut()
8117 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8118 .unwrap();
8119
8120 let fill = FillReport::new(
8121 account_id,
8122 instrument.id(),
8123 venue_order_id,
8124 TradeId::from("T-ALLOW"),
8125 OrderSide::Buy,
8126 Quantity::from("12.0"),
8127 Price::from("100.0"),
8128 Money::from("1.00 USDT"),
8129 LiquiditySide::Taker,
8130 Some(client_order_id),
8131 None,
8132 UnixNanos::from(1_000),
8133 UnixNanos::from(1_000),
8134 None,
8135 );
8136 node.kernel
8137 .exec_engine
8138 .borrow_mut()
8139 .reconcile_fill_report(&fill);
8140
8141 let order = node
8142 .kernel
8143 .cache
8144 .borrow()
8145 .order_owned(&client_order_id)
8146 .unwrap();
8147 assert_eq!(order.quantity(), Quantity::from("10.0"));
8148 assert_eq!(order.filled_qty(), Quantity::from("12.0"));
8149 assert_eq!(order.trade_ids().len(), 1);
8150 }
8151
8152 fn position_fill_test_fixture(
8153 name: &str,
8154 authoritative_qty: Quantity,
8155 ) -> (LiveNode, PositionStatusReport, FillReport) {
8156 position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
8157 }
8158
8159 fn position_fill_test_fixture_with_threshold(
8160 name: &str,
8161 authoritative_qty: Quantity,
8162 position_check_threshold_ms: u32,
8163 ) -> (LiveNode, PositionStatusReport, FillReport) {
8164 let config = LiveNodeConfig {
8165 exec_engine: crate::config::LiveExecutionEngineConfig {
8166 reconciliation: true,
8167 position_check_threshold_ms,
8168 ..Default::default()
8169 },
8170 ..Default::default()
8171 };
8172
8173 let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8174 let account_id = AccountId::from("TEST-001");
8175 let client_id = ClientId::from("POSITION-FILLS");
8176 let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
8177 let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
8178 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8179 let account = AccountAny::Margin(MarginAccount::new(
8180 AccountState::new(
8181 account_id,
8182 AccountType::Margin,
8183 vec![AccountBalance::new(
8184 Money::from("1000000 USDT"),
8185 Money::from("0 USDT"),
8186 Money::from("1000000 USDT"),
8187 )],
8188 Vec::new(),
8189 true,
8190 UUID4::new(),
8191 UnixNanos::default(),
8192 UnixNanos::default(),
8193 Some(Currency::USDT()),
8194 ),
8195 true,
8196 ));
8197 node.kernel.cache.borrow_mut().add_account(account).unwrap();
8198 node.kernel
8199 .cache
8200 .borrow_mut()
8201 .add_instrument(instrument.clone())
8202 .unwrap();
8203 insert_accepted_limit_order_in_node(
8204 &node,
8205 account_id,
8206 client_id,
8207 instrument.id(),
8208 client_order_id,
8209 venue_order_id,
8210 );
8211 let order = node
8212 .kernel
8213 .cache
8214 .borrow()
8215 .order_owned(&client_order_id)
8216 .unwrap();
8217 let mut initial_fill = TestOrderEventStubs::filled(
8218 &order,
8219 &instrument,
8220 Some(TradeId::from("T-POSITION-INITIAL")),
8221 None,
8222 Some(Price::from("100.0")),
8223 Some(Quantity::from("1.0")),
8224 Some(LiquiditySide::Taker),
8225 None,
8226 None,
8227 Some(account_id),
8228 );
8229
8230 let OrderEventAny::Filled(fill) = &mut initial_fill else {
8231 unreachable!();
8232 };
8233
8234 fill.commission = Some(Money::zero(instrument.quote_currency()));
8235 node.process_reconciliation_events(&[initial_fill]);
8236
8237 let ts_event = UnixNanos::from(1_000);
8238
8239 let fill_report = FillReport::new(
8240 account_id,
8241 instrument.id(),
8242 venue_order_id,
8243 TradeId::from("T-POSITION-AUTHORITATIVE"),
8244 OrderSide::Buy,
8245 authoritative_qty,
8246 Price::from("100.0"),
8247 Money::zero(instrument.quote_currency()),
8248 LiquiditySide::Taker,
8249 Some(client_order_id),
8250 None,
8251 ts_event,
8252 ts_event,
8253 None,
8254 );
8255
8256 let venue_report = PositionStatusReport::new(
8257 account_id,
8258 instrument.id(),
8259 PositionSide::Long,
8260 Quantity::from("2.0"),
8261 ts_event,
8262 ts_event,
8263 None,
8264 None,
8265 Some(dec!(100.0)),
8266 );
8267
8268 (node, venue_report, fill_report)
8269 }
8270
8271 fn position_report_result(
8272 node: &LiveNode,
8273 report: PositionStatusReport,
8274 ) -> PositionReportResult {
8275 let client_id = ClientId::from("POSITION-FILLS");
8276 let key = (report.instrument_id, report.account_id);
8277 let mut check = node
8278 .exec_manager
8279 .prepare_position_report_check(UUID4::new(), &[]);
8280 check.client_coverage.insert(
8281 key,
8282 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
8283 );
8284
8285 PositionReportResult {
8286 check,
8287 reports: vec![report],
8288 queried_clients: IndexSet::from([client_id]),
8289 failed_clients: IndexSet::new(),
8290 }
8291 }
8292
8293 fn position_quantity_after_check(
8294 node: &mut LiveNode,
8295 venue_report: &PositionStatusReport,
8296 reports: &[FillReport],
8297 ) -> Quantity {
8298 let key = (venue_report.instrument_id, venue_report.account_id);
8299 let position_result = position_report_result(node, venue_report.clone());
8300 node.handle_position_fill_report_result(PositionFillReportResult {
8301 position_result,
8302 reports: IndexMap::from([(key, reports.to_vec())]),
8303 successful_keys: IndexSet::from([key]),
8304 });
8305
8306 node.kernel
8307 .cache
8308 .borrow()
8309 .positions_open(None, Some(&key.0), None, Some(&key.1), None)
8310 .iter()
8311 .map(|position| position.quantity)
8312 .sum()
8313 }
8314
8315 fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
8316 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
8317 fill.account_id,
8318 fill.instrument_id,
8319 fill.venue_order_id,
8320 fill.trade_id,
8321 fill.order_side,
8322 fill.last_qty,
8323 fill.last_px,
8324 fill.commission
8325 .unwrap_or_else(|| Money::zero(fill.currency)),
8326 fill.liquidity_side,
8327 Some(fill.client_order_id),
8328 fill.position_id,
8329 fill.ts_event,
8330 fill.ts_init,
8331 None,
8332 ))))
8333 }
8334
8335 fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
8336 node.exec_manager.is_fill_recently_processed(
8337 fill.account_id,
8338 fill.instrument_id,
8339 fill.trade_id,
8340 )
8341 }
8342
8343 #[rstest]
8344 #[case(0, NodeState::Idle)]
8345 #[case(1, NodeState::Starting)]
8346 #[case(2, NodeState::Running)]
8347 #[case(3, NodeState::ShuttingDown)]
8348 #[case(4, NodeState::Stopped)]
8349 fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
8350 assert_eq!(NodeState::from_u8(value), expected);
8351 }
8352
8353 #[rstest]
8354 #[case(5)]
8355 #[case(255)]
8356 #[should_panic(expected = "Invalid NodeState value")]
8357 fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
8358 let _ = NodeState::from_u8(value);
8359 }
8360
8361 #[rstest]
8362 fn test_node_state_roundtrip() {
8363 for state in [
8364 NodeState::Idle,
8365 NodeState::Starting,
8366 NodeState::Running,
8367 NodeState::ShuttingDown,
8368 NodeState::Stopped,
8369 ] {
8370 assert_eq!(NodeState::from_u8(state.as_u8()), state);
8371 }
8372 }
8373
8374 #[rstest]
8375 fn test_node_state_is_running_only_for_running() {
8376 assert!(!NodeState::Idle.is_running());
8377 assert!(!NodeState::Starting.is_running());
8378 assert!(NodeState::Running.is_running());
8379 assert!(!NodeState::ShuttingDown.is_running());
8380 assert!(!NodeState::Stopped.is_running());
8381 }
8382
8383 #[rstest]
8384 #[tokio::test]
8385 async fn test_await_engines_connected_returns_stop_requested() {
8386 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8387 let handle = node.handle();
8388
8389 handle.stop();
8390
8391 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8392 let status = node.await_engines_connected(deadline).await;
8393
8394 assert_eq!(status, EngineConnectionStatus::StopRequested);
8395 assert!(handle.should_stop());
8396 }
8397
8398 #[rstest]
8399 #[tokio::test]
8400 async fn test_await_engines_connected_returns_shutdown_requested() {
8401 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8402
8403 node.kernel().shutdown_flag().set(true);
8404
8405 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8406 let status = node.await_engines_connected(deadline).await;
8407
8408 assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
8409 }
8410
8411 #[rstest]
8412 #[tokio::test]
8413 async fn test_start_stop_request_aborts_startup_without_running() {
8414 let config = LiveNodeConfig {
8415 exec_engine: crate::config::LiveExecutionEngineConfig {
8416 reconciliation: false,
8417 ..Default::default()
8418 },
8419 timeout_disconnection: Duration::from_millis(50),
8420 ..Default::default()
8421 };
8422
8423 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8424 let handle = node.handle();
8425
8426 handle.stop();
8427 node.start().await.unwrap();
8428
8429 assert_eq!(handle.state(), NodeState::Stopped);
8430 assert!(handle.should_stop());
8431 assert!(!handle.is_running());
8432 }
8433
8434 #[rstest]
8435 #[tokio::test(start_paused = true)]
8436 async fn test_stop_processes_residual_exec_event_during_grace_period() {
8437 let config = LiveNodeConfig {
8438 exec_engine: crate::config::LiveExecutionEngineConfig {
8439 reconciliation: false,
8440 ..Default::default()
8441 },
8442 timeout_connection: Duration::ZERO,
8443 timeout_reconciliation: Duration::ZERO,
8444 timeout_portfolio: Duration::ZERO,
8445 timeout_disconnection: Duration::ZERO,
8446 delay_post_stop: Duration::from_millis(20),
8447 timeout_shutdown: Duration::ZERO,
8448 ..Default::default()
8449 };
8450
8451 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8452 let order = OrderTestBuilder::new(OrderType::Market)
8453 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
8454 .quantity(Quantity::from("2"))
8455 .build();
8456 let client_order_id = order.client_order_id();
8457 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
8458
8459 node.kernel
8460 .cache()
8461 .borrow_mut()
8462 .add_order(order, None, None, false)
8463 .unwrap();
8464
8465 node.start().await.unwrap();
8466 let exec_event_sender = get_exec_event_sender();
8467
8468 let send_residual = tokio::spawn(async move {
8469 tokio::time::sleep(Duration::from_millis(1)).await;
8470 exec_event_sender
8471 .send(ExecutionEvent::Order(submitted))
8472 .unwrap();
8473 });
8474
8475 node.stop().await.unwrap();
8476 send_residual.await.unwrap();
8477
8478 assert_eq!(
8479 node.kernel
8480 .cache()
8481 .borrow()
8482 .order(&client_order_id)
8483 .unwrap()
8484 .status(),
8485 OrderStatus::Submitted
8486 );
8487
8488 node.dispose();
8489 }
8490
8491 #[rstest]
8492 #[tokio::test]
8493 async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
8494 let actor_id = ActorId::from("LIVE-STATE-ACTOR");
8495 let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
8496 let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
8497 let strategy_load =
8498 IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
8499 let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
8500 let strategy_save =
8501 IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
8502 let (database, control) = TestCacheDatabaseControl::create();
8503 control.set_actor_state(actor_id, &actor_load);
8504 control.set_strategy_state(strategy_id, &strategy_load);
8505
8506 let config = LiveNodeConfig {
8507 load_state: true,
8508 save_state: true,
8509 exec_engine: crate::config::LiveExecutionEngineConfig {
8510 reconciliation: false,
8511 ..Default::default()
8512 },
8513 timeout_connection: Duration::ZERO,
8514 timeout_reconciliation: Duration::ZERO,
8515 timeout_portfolio: Duration::ZERO,
8516 timeout_disconnection: Duration::ZERO,
8517 delay_post_stop: Duration::ZERO,
8518 timeout_shutdown: Duration::ZERO,
8519 ..Default::default()
8520 };
8521
8522 let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
8523 node.set_cache_database(Box::new(database)).unwrap();
8524 node.add_actor(StateActor::new(
8525 actor_id,
8526 control.clone(),
8527 actor_save.clone(),
8528 ))
8529 .unwrap();
8530 node.add_strategy(StateStrategy::new(
8531 strategy_id,
8532 control.clone(),
8533 strategy_save.clone(),
8534 ))
8535 .unwrap();
8536
8537 node.start().await.unwrap();
8538 node.stop().await.unwrap();
8539 node.dispose();
8540
8541 assert_eq!(
8542 control.events(),
8543 vec![
8544 "actor.load:LIVE-STATE-ACTOR",
8545 "actor.on_load",
8546 "strategy.load:LIVE-STATE-STRATEGY-001",
8547 "strategy.on_load",
8548 "actor.on_start",
8549 "strategy.on_start",
8550 "actor.on_stop",
8551 "strategy.on_stop",
8552 "actor.on_save",
8553 "actor.update:LIVE-STATE-ACTOR",
8554 "strategy.on_save",
8555 "strategy.update:LIVE-STATE-STRATEGY-001",
8556 "database.close",
8557 ]
8558 );
8559 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
8560 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
8561 assert_eq!(node.state(), NodeState::Stopped);
8562 }
8563
8564 #[rstest]
8565 #[tokio::test]
8566 async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
8567 let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
8568 let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
8569 let (database, control) = TestCacheDatabaseControl::create();
8570
8571 let config = LiveNodeConfig {
8572 save_state: true,
8573 exec_engine: crate::config::LiveExecutionEngineConfig {
8574 reconciliation: false,
8575 ..Default::default()
8576 },
8577 timeout_connection: Duration::ZERO,
8578 timeout_reconciliation: Duration::ZERO,
8579 timeout_portfolio: Duration::ZERO,
8580 timeout_disconnection: Duration::ZERO,
8581 delay_post_stop: Duration::ZERO,
8582 timeout_shutdown: Duration::ZERO,
8583 ..Default::default()
8584 };
8585
8586 let mut node =
8587 LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
8588 node.set_cache_database(Box::new(database)).unwrap();
8589 node.add_actor(
8590 StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
8591 )
8592 .unwrap();
8593 node.add_strategy(
8594 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
8595 )
8596 .unwrap();
8597
8598 node.start().await.unwrap();
8599 let error = node.stop().await.unwrap_err();
8600 node.dispose();
8601
8602 assert_eq!(
8603 error.to_string(),
8604 "failed while finalizing kernel shutdown: Failed to save component state: actor \
8605 LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
8606 LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
8607 );
8608 assert_eq!(
8609 control.events(),
8610 vec![
8611 "actor.on_start",
8612 "strategy.on_start",
8613 "actor.on_stop",
8614 "strategy.on_stop",
8615 "actor.on_save",
8616 "strategy.on_save",
8617 "database.close",
8618 ]
8619 );
8620 assert_eq!(node.state(), NodeState::Stopped);
8621 }
8622
8623 #[rstest]
8624 #[tokio::test]
8625 async fn test_stop_drains_queued_exec_event_after_zero_grace() {
8626 let config = LiveNodeConfig {
8627 exec_engine: crate::config::LiveExecutionEngineConfig {
8628 reconciliation: false,
8629 ..Default::default()
8630 },
8631 timeout_connection: Duration::ZERO,
8632 timeout_reconciliation: Duration::ZERO,
8633 timeout_portfolio: Duration::ZERO,
8634 timeout_disconnection: Duration::ZERO,
8635 delay_post_stop: Duration::ZERO,
8636 timeout_shutdown: Duration::ZERO,
8637 ..Default::default()
8638 };
8639
8640 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8641 let order = OrderTestBuilder::new(OrderType::Market)
8642 .instrument_id(InstrumentId::from("GBP/USD.SIM"))
8643 .quantity(Quantity::from("3"))
8644 .build();
8645 let client_order_id = order.client_order_id();
8646 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
8647
8648 node.kernel
8649 .cache()
8650 .borrow_mut()
8651 .add_order(order, None, None, false)
8652 .unwrap();
8653
8654 node.start().await.unwrap();
8655 get_exec_event_sender()
8656 .send(ExecutionEvent::Order(submitted))
8657 .unwrap();
8658
8659 node.stop().await.unwrap();
8660
8661 assert_eq!(
8662 node.kernel
8663 .cache()
8664 .borrow()
8665 .order(&client_order_id)
8666 .unwrap()
8667 .status(),
8668 OrderStatus::Submitted
8669 );
8670
8671 node.dispose();
8672 }
8673
8674 #[rstest]
8675 #[tokio::test]
8676 async fn test_start_event_store_replay_skips_live_connections() {
8677 let mut node = live_node_with_replay_store(false);
8678 let handle = node.handle();
8679
8680 node.start().await.unwrap();
8681
8682 assert_eq!(handle.state(), NodeState::Running);
8683 assert!(handle.is_running());
8684 assert!(node.kernel.is_event_store_replay());
8685 assert!(node.runner.is_some());
8686 }
8687
8688 #[rstest]
8689 #[tokio::test]
8690 async fn test_start_event_store_replay_preserves_stop_request() {
8691 let mut node = live_node_with_replay_store(false);
8692 let handle = node.handle();
8693 handle.stop();
8694
8695 node.start().await.unwrap();
8696
8697 assert_eq!(handle.state(), NodeState::Stopped);
8698 assert!(handle.should_stop());
8699 assert!(node.kernel.is_event_store_replay());
8700 assert!(node.runner.is_some());
8701 }
8702
8703 #[rstest]
8704 #[tokio::test]
8705 async fn test_start_event_store_replay_config_failure_aborts_startup() {
8706 let mut node = live_node_with_replay_store(true);
8707 let handle = node.handle();
8708
8709 node.start().await.unwrap();
8710
8711 assert_eq!(handle.state(), NodeState::Stopped);
8712 assert!(!handle.is_running());
8713 assert!(node.kernel.is_event_store_replay_configured());
8714 assert!(!node.kernel.is_event_store_replay());
8715 assert!(node.runner.is_some());
8716 }
8717
8718 #[rstest]
8719 #[tokio::test]
8720 async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
8721 let mut node = live_node_with_replay_store(false);
8722 let handle = node.handle();
8723
8724 node.run().await.unwrap();
8725
8726 assert_eq!(handle.state(), NodeState::Running);
8727 assert!(handle.is_running());
8728 assert!(node.kernel.is_event_store_replay());
8729 assert!(node.runner.is_none());
8730 }
8731
8732 #[rstest]
8733 #[tokio::test]
8734 async fn test_run_event_store_replay_preserves_stop_request() {
8735 let mut node = live_node_with_replay_store(false);
8736 let handle = node.handle();
8737 handle.stop();
8738
8739 node.run().await.unwrap();
8740
8741 assert_eq!(handle.state(), NodeState::Stopped);
8742 assert!(handle.should_stop());
8743 assert!(node.kernel.is_event_store_replay());
8744 assert!(node.runner.is_none());
8745 }
8746
8747 #[rstest]
8748 #[tokio::test]
8749 async fn test_run_event_store_replay_config_failure_aborts_startup() {
8750 let mut node = live_node_with_replay_store(true);
8751 let handle = node.handle();
8752
8753 node.run().await.unwrap();
8754
8755 assert_eq!(handle.state(), NodeState::Stopped);
8756 assert!(!handle.is_running());
8757 assert!(node.kernel.is_event_store_replay_configured());
8758 assert!(!node.kernel.is_event_store_replay());
8759 assert!(node.runner.is_none());
8760 }
8761
8762 #[rstest]
8763 fn test_build_rejects_event_store_config_without_factory() {
8764 let config = LiveNodeConfig {
8765 event_store: Some(EventStoreConfig::default()),
8766 exec_engine: crate::config::LiveExecutionEngineConfig {
8767 reconciliation: false,
8768 ..Default::default()
8769 },
8770 ..Default::default()
8771 };
8772
8773 let err = LiveNodeBuilder::from_config(config)
8774 .expect("builder")
8775 .build()
8776 .expect_err("should reject event_store config without factory");
8777
8778 assert!(
8779 err.to_string().contains("with_event_store"),
8780 "error message should mention with_event_store, was: {err}"
8781 );
8782 }
8783
8784 #[rstest]
8785 fn test_direct_build_rejects_event_store_config() {
8786 let config = LiveNodeConfig {
8787 event_store: Some(EventStoreConfig::default()),
8788 exec_engine: crate::config::LiveExecutionEngineConfig {
8789 reconciliation: false,
8790 ..Default::default()
8791 },
8792 ..Default::default()
8793 };
8794
8795 let err = LiveNode::build("TestNode".to_string(), Some(config))
8796 .expect_err("LiveNode::build should reject event_store config");
8797
8798 assert!(
8799 err.to_string().contains("with_event_store"),
8800 "error message should mention with_event_store, was: {err}"
8801 );
8802 }
8803
8804 #[rstest]
8805 fn test_dispose_before_start_is_idempotent() {
8806 let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
8807 node.add_strategy(TestStrategy::new(StrategyConfig {
8808 strategy_id: Some(StrategyId::from("DISPOSAL-001")),
8809 ..Default::default()
8810 }))
8811 .unwrap();
8812
8813 node.dispose();
8814 node.dispose();
8815
8816 assert!(node.kernel.trader().borrow().is_disposed());
8817 assert_eq!(node.kernel.trader().borrow().component_count(), 0);
8818 assert_eq!(node.state(), NodeState::Stopped);
8819 }
8820
8821 #[rstest]
8822 fn test_dispose_releases_retained_callback_roots(
8823 #[values(false, true)] fatal: bool,
8824 #[values(false, true)] external: bool,
8825 ) {
8826 actor::clear_callbacks().unwrap();
8827 let directory =
8828 std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
8829
8830 let config = LiveNodeConfig {
8831 logging: LoggerConfig {
8832 fileout_level: LevelFilter::Info,
8833 file_config: Some(FileWriterConfig {
8834 directory: Some(directory.to_str().unwrap().to_string()),
8835 file_name: Some("disposal".to_string()),
8836 ..Default::default()
8837 }),
8838 ..Default::default()
8839 },
8840 ..Default::default()
8841 };
8842
8843 let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
8844 let received = Rc::new(RefCell::new(Vec::new()));
8845 let observed = received.clone();
8846
8847 let retained = DispatchMessage::from(()).dispatch(|()| {
8848 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
8849 TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
8850 TimeEventCallback::RustLocal(Rc::new(move |_| {
8851 observed.borrow_mut().push("delivered");
8852 })),
8853 ));
8854
8855 external.then(|| DispatchMessage::new((), std::thread::current().id()))
8856 });
8857
8858 if let Some(retained) = &retained {
8859 assert!(retained.is_rooted());
8860 }
8861
8862 assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
8863
8864 if fatal {
8865 crate::dispatch::tests::latch_callback_failure();
8866 }
8867
8868 node.dispose();
8869
8870 assert!(node.runner.is_none());
8871 assert!(received.borrow().is_empty());
8872 assert_eq!(Rc::strong_count(&received), 1);
8873 assert_eq!(
8874 actor::callback_failure(),
8875 (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
8876 );
8877 assert_eq!(
8878 actor::clear_callbacks(),
8879 if external {
8880 Err(CallbackDispatchError::Active)
8881 } else {
8882 Ok(())
8883 }
8884 );
8885
8886 logging_sync_to_disk().unwrap();
8887 let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
8888 drop(retained);
8889 node.dispose();
8890
8891 assert_eq!(actor::callback_failure(), None);
8892 assert_eq!(actor::clear_callbacks(), Ok(()));
8893 drop(node);
8894 std::fs::remove_dir_all(directory).unwrap();
8895
8896 let errors: Vec<_> = output
8897 .lines()
8898 .filter(|line| line.contains("[ERROR]"))
8899 .filter_map(|line| {
8900 line.split_once(".nautilus_live::node: ")
8901 .map(|(_, text)| text)
8902 })
8903 .collect();
8904
8905 let mut expected = Vec::new();
8906
8907 if fatal {
8908 expected.push(
8909 "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
8910 );
8911 }
8912
8913 if external {
8914 expected.push(
8915 "Failed to clear callback dispatch during disposal: Callback work or access is still active",
8916 );
8917 }
8918
8919 assert_eq!(errors, expected);
8920 }
8921
8922 #[tokio::test]
8923 async fn test_dispose_releases_stop_generated_callback_roots() {
8924 actor::clear_callbacks().unwrap();
8925
8926 let config = LiveNodeConfig {
8927 exec_engine: crate::config::LiveExecutionEngineConfig {
8928 reconciliation: false,
8929 ..Default::default()
8930 },
8931 timeout_connection: Duration::ZERO,
8932 timeout_reconciliation: Duration::ZERO,
8933 timeout_portfolio: Duration::ZERO,
8934 timeout_shutdown: Duration::ZERO,
8935 ..Default::default()
8936 };
8937
8938 let mut node =
8939 LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
8940 let received = Rc::new(RefCell::new(Vec::new()));
8941 node.add_actor(StopCallbackActor {
8942 core: DataActorCore::new(DataActorConfig {
8943 actor_id: Some(ActorId::from("STOP-CALLBACK")),
8944 ..Default::default()
8945 }),
8946 received: received.clone(),
8947 })
8948 .unwrap();
8949
8950 node.start().await.unwrap();
8951
8952 assert!(node.kernel.trader().borrow().is_running());
8953 assert_eq!(actor::clear_callbacks(), Ok(()));
8954
8955 node.dispose();
8956
8957 assert_eq!(*received.borrow(), ["stop", "queued"]);
8958 assert_eq!(Rc::strong_count(&received), 1);
8959 assert!(node.runner.is_none());
8960 assert!(node.kernel.trader().borrow().is_disposed());
8961 assert_eq!(node.state(), NodeState::Stopped);
8962 assert_eq!(actor::clear_callbacks(), Ok(()));
8963 }
8964
8965 #[derive(Debug)]
8966 struct StopCallbackActor {
8967 core: DataActorCore,
8968 received: Rc<RefCell<Vec<&'static str>>>,
8969 }
8970
8971 nautilus_actor!(StopCallbackActor);
8972
8973 impl DataActor for StopCallbackActor {
8974 fn on_stop(&mut self) -> anyhow::Result<()> {
8975 self.received.borrow_mut().push("stop");
8976 let received = self.received.clone();
8977 DispatchMessage::from(()).dispatch(|()| {
8978 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
8979 TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
8980 TimeEventCallback::RustLocal(Rc::new(move |_| {
8981 received.borrow_mut().push("delivered");
8982 })),
8983 ));
8984 });
8985
8986 let clear_result = actor::clear_callbacks();
8987 anyhow::ensure!(
8988 clear_result == Err(CallbackDispatchError::Active),
8989 "Expected active callback roots during stop, received {clear_result:?}"
8990 );
8991 self.received.borrow_mut().push("queued");
8992 Ok(())
8993 }
8994 }
8995
8996 #[rstest]
8997 fn test_handle_initial_state() {
8998 let handle = LiveNodeHandle::new();
8999
9000 assert_eq!(handle.state(), NodeState::Idle);
9001 assert!(!handle.should_stop());
9002 assert!(!handle.is_running());
9003 }
9004
9005 #[rstest]
9006 fn test_handle_initial_metrics_snapshot_is_zero() {
9007 let handle = LiveNodeHandle::new();
9008
9009 assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9010 }
9011
9012 #[rstest]
9013 fn test_record_runner_dispatch_updates_selected_channel() {
9014 let metrics = RunnerMetrics::default();
9015 let dispatch_start = dst::time::Instant::now();
9016 let metrics_start = dispatch_start
9017 .checked_sub(Duration::from_micros(1))
9018 .expect("test instant should support a one-microsecond lookback");
9019
9020 record_runner_dispatch(
9021 &metrics,
9022 SystemChannel::DataCommands,
9023 dispatch_start,
9024 metrics_start,
9025 );
9026 let snapshot = metrics.snapshot();
9027
9028 assert_eq!(snapshot.time_events.dispatched, 0);
9029 assert_eq!(snapshot.exec_events.dispatched, 0);
9030 assert_eq!(snapshot.exec_commands.dispatched, 0);
9031 assert_eq!(snapshot.data_events.dispatched, 0);
9032 assert_eq!(snapshot.data_commands.dispatched, 1);
9033 assert_eq!(
9034 snapshot.data_commands.last_dispatch_at_ns,
9035 snapshot.elapsed_ns
9036 );
9037 assert_eq!(
9038 snapshot.data_commands.dispatch_busy_ns,
9039 snapshot.dispatch_busy_ns
9040 );
9041 assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9042 }
9043
9044 #[rstest]
9045 fn test_handle_stop_sets_flag() {
9046 let handle = LiveNodeHandle::new();
9047
9048 handle.stop();
9049
9050 assert!(handle.should_stop());
9051 }
9052
9053 #[rstest]
9054 fn test_handle_stop_blocks_running_transition() {
9055 let handle = LiveNodeHandle::new();
9056 handle.set_starting();
9057 handle.stop();
9058
9059 let transition = handle.try_set_running();
9060
9061 assert_eq!(transition, RunningTransition::StopRequested);
9062 assert_eq!(handle.state(), NodeState::Starting);
9063 assert!(handle.should_stop());
9064 assert!(!handle.is_running());
9065 }
9066
9067 #[rstest]
9068 fn test_handle_stop_after_running_transition_remains_pending() {
9069 let handle = LiveNodeHandle::new();
9070 handle.set_starting();
9071
9072 let transition = handle.try_set_running();
9073 handle.stop();
9074
9075 assert_eq!(transition, RunningTransition::Entered);
9076 assert_eq!(handle.state(), NodeState::Running);
9077 assert!(handle.should_stop());
9078 assert!(handle.is_running());
9079 }
9080
9081 #[rstest]
9082 fn test_handle_node_state_transitions() {
9083 let handle = LiveNodeHandle::new();
9084 assert_eq!(handle.state(), NodeState::Idle);
9085
9086 handle.set_starting();
9087 assert_eq!(handle.state(), NodeState::Starting);
9088 assert!(!handle.is_running());
9089
9090 assert_eq!(handle.try_set_running(), RunningTransition::Entered);
9091 assert_eq!(handle.state(), NodeState::Running);
9092 assert!(handle.is_running());
9093
9094 handle.set_shutting_down();
9095 assert_eq!(handle.state(), NodeState::ShuttingDown);
9096 assert!(!handle.is_running());
9097
9098 handle.set_stopped();
9099 assert_eq!(handle.state(), NodeState::Stopped);
9100 assert!(!handle.is_running());
9101 }
9102
9103 #[rstest]
9104 fn test_handle_clone_shares_state_bidirectionally() {
9105 let handle1 = LiveNodeHandle::new();
9106 let handle2 = handle1.clone();
9107
9108 handle1.set_starting();
9109 let transition = handle2.try_set_running();
9110 handle1.stop();
9111
9112 assert_eq!(transition, RunningTransition::Entered);
9113 assert_eq!(handle1.state(), NodeState::Running);
9114 assert!(handle2.should_stop());
9115 }
9116
9117 #[rstest]
9118 fn test_handle_stop_flag_survives_non_running_state_changes() {
9119 let handle = LiveNodeHandle::new();
9120
9121 handle.set_starting();
9122 handle.stop();
9123 handle.set_shutting_down();
9124 handle.set_stopped();
9125
9126 assert_eq!(handle.state(), NodeState::Stopped);
9127 assert!(handle.should_stop());
9128 }
9129
9130 #[rstest]
9131 fn test_builder_creation() {
9132 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9133
9134 assert!(result.is_ok());
9135 }
9136
9137 #[rstest]
9138 fn test_stream_processor_receives_unregistered_typed_payload() {
9139 let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9140 .unwrap()
9141 .build()
9142 .unwrap();
9143 let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
9144 InstrumentId::from("AUD/USD.SIM"),
9145 Some(ClientId::from("EXTERNAL")),
9146 Some(Venue::from("SIM")),
9147 UUID4::from("00000000-0000-4000-8000-000000000001"),
9148 UnixNanos::from(1),
9149 None,
9150 None,
9151 ));
9152 let expected = serde_json::to_value(&command).unwrap();
9153 let received = Rc::new(RefCell::new(Vec::new()));
9154 let steps = Rc::new(RefCell::new(Vec::new()));
9155 let received_processor = received.clone();
9156 let first_steps = steps.clone();
9157 node.add_stream_processor(move |message| {
9158 let command = message
9159 .downcast_ref::<SubscribeCommand>()
9160 .expect("processor must receive the decoded concrete command");
9161 received_processor
9162 .borrow_mut()
9163 .push(serde_json::to_value(command).unwrap());
9164 first_steps.borrow_mut().push(1);
9165 });
9166
9167 let second_steps = steps.clone();
9168 node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
9169 let republished = Rc::new(RefCell::new(Vec::new()));
9170 let republished_handler = republished.clone();
9171 let subscriber_steps = steps.clone();
9172 msgbus::subscribe_any(
9173 "external.test".into(),
9174 ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
9175 republished_handler
9176 .borrow_mut()
9177 .push(serde_json::to_value(command).unwrap());
9178 subscriber_steps.borrow_mut().push(3);
9179 }),
9180 None,
9181 );
9182
9183 let message = BusMessage::with_str_topic(
9184 "external.test",
9185 BusPayloadType::SubscribeCommand,
9186 Bytes::from(serde_json::to_vec(&command).unwrap()),
9187 SerializationEncoding::Json,
9188 );
9189
9190 assert!(
9191 !msgbus::get_message_bus()
9192 .borrow()
9193 .is_streaming_type(BusPayloadType::SubscribeCommand)
9194 );
9195 node.process_external_msgbus_message(&message);
9196
9197 assert_eq!(*received.borrow(), vec![expected.clone()]);
9198 assert_eq!(*steps.borrow(), vec![1, 2]);
9199 assert!(republished.borrow().is_empty());
9200
9201 received.borrow_mut().clear();
9202 steps.borrow_mut().clear();
9203 msgbus::get_message_bus()
9204 .borrow_mut()
9205 .add_streaming_type(BusPayloadType::SubscribeCommand);
9206 node.process_external_msgbus_message(&message);
9207
9208 assert_eq!(*received.borrow(), vec![expected.clone()]);
9209 assert_eq!(*republished.borrow(), vec![expected]);
9210 assert_eq!(*steps.borrow(), vec![1, 2, 3]);
9211 msgbus::get_message_bus().borrow_mut().dispose();
9212 }
9213
9214 #[rstest]
9215 fn test_builder_rejects_backtest() {
9216 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
9217
9218 assert!(result.is_err());
9219 assert!(result.unwrap_err().to_string().contains("Backtest"));
9220 }
9221
9222 #[rstest]
9223 fn test_builder_accepts_live_environment() {
9224 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
9225
9226 assert!(result.is_ok());
9227 }
9228
9229 #[rstest]
9230 fn test_builder_accepts_sandbox_environment() {
9231 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9232
9233 assert!(result.is_ok());
9234 }
9235
9236 #[rstest]
9237 fn test_builder_fluent_api_chaining() {
9238 let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
9239 .unwrap()
9240 .with_name("TestNode")
9241 .with_instance_id(UUID4::new())
9242 .with_load_state(false)
9243 .with_save_state(true)
9244 .with_timeout_connection(30)
9245 .with_timeout_reconciliation(60)
9246 .with_reconciliation(true)
9247 .with_reconciliation_lookback_mins(120)
9248 .with_timeout_portfolio(10)
9249 .with_timeout_disconnection_secs(5)
9250 .with_delay_post_stop_secs(3)
9251 .with_delay_shutdown_secs(10);
9252
9253 assert_eq!(builder.name(), "TestNode");
9254 }
9255
9256 #[rstest]
9257 fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
9258 let (external_egress, publications, closed) = CapturingExternalEgress::new();
9259
9260 let msgbus_config = MessageBusConfig {
9261 encoding: SerializationEncoding::Json,
9262 ..Default::default()
9263 };
9264
9265 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9266 .unwrap()
9267 .with_msgbus_config(msgbus_config)
9268 .with_external_msgbus_egress(Box::new(external_egress))
9269 .build()
9270 .expect("node builds with external message bus egress");
9271 let quote = QuoteTick::default();
9272
9273 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9274
9275 let publications = publications.borrow();
9276 assert_eq!(publications.len(), 1);
9277 assert_eq!(publications[0].topic, "data.quotes.TEST");
9278 assert_eq!(
9279 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9280 .expect("JSON payload must decode as QuoteTick"),
9281 quote
9282 );
9283 drop(publications);
9284
9285 msgbus::get_message_bus().borrow_mut().dispose();
9286 assert!(closed.get());
9287 drop(node);
9288 }
9289
9290 #[rstest]
9291 #[tokio::test(flavor = "current_thread")]
9292 async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
9293 let quote = QuoteTick::default();
9294 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9295 let publications = Arc::new(Mutex::new(Vec::new()));
9296 let closed = Arc::new(AtomicBool::new(false));
9297 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
9298
9299 let msgbus_config = MessageBusConfig {
9300 external_streams: Some(vec!["stream".to_string()]),
9301 ..Default::default()
9302 };
9303
9304 let config = LiveNodeConfig {
9305 environment: Environment::Sandbox,
9306 msgbus: Some(msgbus_config),
9307 exec_engine: crate::config::LiveExecutionEngineConfig {
9308 reconciliation: false,
9309 ..Default::default()
9310 },
9311 delay_post_stop: Duration::ZERO,
9312 timeout_connection: Duration::from_millis(500),
9313 timeout_disconnection: Duration::from_millis(500),
9314 ..Default::default()
9315 };
9316
9317 let mut node = LiveNodeBuilder::from_config(config)
9318 .unwrap()
9319 .with_external_msgbus_factory(Box::new(factory))
9320 .build()
9321 .expect("node builds with external message bus factory");
9322
9323 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9324 {
9325 let publications = publications.lock();
9326 assert_eq!(publications.len(), 1);
9327 assert_eq!(publications[0].topic, "data.quotes.TEST");
9328 assert_eq!(
9329 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9330 .expect("JSON payload must decode as QuoteTick"),
9331 quote
9332 );
9333 }
9334
9335 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9336 let handle = node.handle();
9337
9338 let handler = TypedHandler::from({
9341 let received = received.clone();
9342 move |quote: &QuoteTick| {
9343 received.borrow_mut().push(*quote);
9344 }
9345 });
9346
9347 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9348 msgbus::get_message_bus()
9349 .borrow_mut()
9350 .add_streaming_type(BusPayloadType::QuoteTick);
9351
9352 let payload =
9353 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
9354 let message = BusMessage::with_str_topic(
9355 "data.quotes.TEST",
9356 BusPayloadType::QuoteTick,
9357 payload,
9358 SerializationEncoding::Json,
9359 );
9360
9361 tokio::time::timeout(Duration::from_secs(30), async {
9362 let run = node.run();
9363 tokio::pin!(run);
9364
9365 let drive = async {
9366 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9367
9368 tx.send(message)
9369 .await
9370 .expect("external ingress receiver should be open");
9371
9372 wait_until_async(
9373 || async { received.borrow().len() == 1 },
9374 Duration::from_secs(10),
9375 )
9376 .await;
9377 assert_eq!(*received.borrow(), vec![quote]);
9378 handle.stop();
9379 };
9380
9381 tokio::select! {
9382 biased;
9383
9384 () = drive => {}
9385 result = &mut run => {
9386 panic!("node stopped before factory ingress was republished: {result:?}");
9387 }
9388 }
9389
9390 run.await.expect("node should stop cleanly");
9391 })
9392 .await
9393 .expect("live node should republish factory ingress and stop before timeout");
9394
9395 assert_eq!(handle.state(), NodeState::Stopped);
9396 assert!(closed.load(Ordering::Relaxed));
9397 msgbus::get_message_bus().borrow_mut().dispose();
9398 }
9399
9400 #[rstest]
9401 #[tokio::test(flavor = "current_thread")]
9402 async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
9403 let quote = QuoteTick::default();
9404 let publications = Arc::new(Mutex::new(Vec::new()));
9405 let closed = Arc::new(AtomicBool::new(false));
9406 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
9407
9408 let config = LiveNodeConfig {
9409 environment: Environment::Sandbox,
9410 msgbus: Some(MessageBusConfig::default()),
9411 exec_engine: crate::config::LiveExecutionEngineConfig {
9412 reconciliation: false,
9413 ..Default::default()
9414 },
9415 delay_post_stop: Duration::ZERO,
9416 timeout_connection: Duration::from_millis(500),
9417 timeout_disconnection: Duration::from_millis(500),
9418 ..Default::default()
9419 };
9420
9421 let mut node = LiveNodeBuilder::from_config(config)
9422 .unwrap()
9423 .with_external_msgbus_factory(Box::new(factory))
9424 .build()
9425 .expect("node builds with egress-only message bus factory");
9426 let handle = node.handle();
9427
9428 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9429 {
9430 let publications = publications.lock();
9431 assert_eq!(publications.len(), 1);
9432 assert_eq!(publications[0].topic, "data.quotes.TEST");
9433 }
9434
9435 tokio::time::timeout(Duration::from_secs(30), async {
9436 let run = node.run();
9437 tokio::pin!(run);
9438
9439 let drive = async {
9440 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9441 handle.stop();
9442 };
9443
9444 tokio::select! {
9445 biased;
9446
9447 () = drive => {}
9448 result = &mut run => {
9449 panic!("node stopped before egress-only factory run was observed: {result:?}");
9450 }
9451 }
9452
9453 run.await.expect("node should stop cleanly");
9454 })
9455 .await
9456 .expect("live node should run without external ingress before timeout");
9457
9458 assert_eq!(handle.state(), NodeState::Stopped);
9459 msgbus::get_message_bus().borrow_mut().dispose();
9460 assert!(closed.load(Ordering::Relaxed));
9461 }
9462
9463 #[rstest]
9464 fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
9465 let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
9466
9467 let egress_factory = CapturingBackingFactory::new(
9468 Arc::new(Mutex::new(Vec::new())),
9469 Arc::new(AtomicBool::new(false)),
9470 None,
9471 );
9472 let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9473 .unwrap()
9474 .with_external_msgbus_factory(Box::new(egress_factory))
9475 .with_external_msgbus_egress(Box::new(external_egress))
9476 .build()
9477 .expect_err("builder should reject factory plus injected egress");
9478
9479 assert!(
9480 egress_error
9481 .to_string()
9482 .contains("cannot be combined with injected egress or ingress")
9483 );
9484
9485 let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9486
9487 let ingress_factory = CapturingBackingFactory::new(
9488 Arc::new(Mutex::new(Vec::new())),
9489 Arc::new(AtomicBool::new(false)),
9490 None,
9491 );
9492 let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
9493 let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9494 .unwrap()
9495 .with_external_msgbus_factory(Box::new(ingress_factory))
9496 .with_external_ingress(Box::new(ingress))
9497 .build()
9498 .expect_err("builder should reject factory plus injected ingress");
9499
9500 assert!(
9501 ingress_error
9502 .to_string()
9503 .contains("cannot be combined with injected egress or ingress")
9504 );
9505 }
9506
9507 #[rstest]
9508 #[tokio::test(flavor = "current_thread")]
9509 async fn test_run_republishes_external_ingress_on_local_msgbus() {
9510 let quote = QuoteTick::default();
9511 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9512 let payload =
9513 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
9514 let message = BusMessage::with_str_topic(
9515 "data.quotes.TEST",
9516 BusPayloadType::QuoteTick,
9517 payload,
9518 SerializationEncoding::Json,
9519 );
9520 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9521 let closed = Rc::new(Cell::new(false));
9522 let ingress = CapturingExternalIngress::new(rx, closed.clone());
9523
9524 let config = LiveNodeConfig {
9525 environment: Environment::Sandbox,
9526 exec_engine: crate::config::LiveExecutionEngineConfig {
9527 reconciliation: false,
9528 ..Default::default()
9529 },
9530 delay_post_stop: Duration::ZERO,
9531 timeout_connection: Duration::from_millis(500),
9532 timeout_disconnection: Duration::from_millis(500),
9533 ..Default::default()
9534 };
9535
9536 let mut node = LiveNodeBuilder::from_config(config)
9537 .unwrap()
9538 .with_external_ingress(Box::new(ingress))
9539 .build()
9540 .expect("node builds with external message bus ingress");
9541 let handle = node.handle();
9542
9543 let handler = TypedHandler::from({
9544 let received = received.clone();
9545 move |quote: &QuoteTick| {
9546 received.borrow_mut().push(*quote);
9547 }
9548 });
9549
9550 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9551 msgbus::get_message_bus()
9552 .borrow_mut()
9553 .add_streaming_type(BusPayloadType::QuoteTick);
9554
9555 tokio::time::timeout(Duration::from_secs(30), async {
9556 let run = node.run();
9557 tokio::pin!(run);
9558
9559 let drive = async {
9560 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9561
9562 tx.send(message)
9563 .await
9564 .expect("external ingress receiver should be open");
9565
9566 wait_until_async(
9567 || async { received.borrow().len() == 1 },
9568 Duration::from_secs(10),
9569 )
9570 .await;
9571 assert_eq!(*received.borrow(), vec![quote]);
9572 handle.stop();
9573 };
9574
9575 tokio::select! {
9576 biased;
9577
9578 () = drive => {}
9579 result = &mut run => {
9580 panic!("node stopped before external message was republished: {result:?}");
9581 }
9582 }
9583
9584 run.await.expect("node should stop cleanly");
9585 })
9586 .await
9587 .expect("live node should republish ingress and stop before timeout");
9588
9589 assert_eq!(handle.state(), NodeState::Stopped);
9590 assert!(closed.get());
9591 msgbus::get_message_bus().borrow_mut().dispose();
9592 }
9593
9594 #[rstest]
9595 #[tokio::test(flavor = "current_thread")]
9596 async fn test_run_closes_external_ingress_when_receiver_closes() {
9597 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9598 let closed = Rc::new(Cell::new(false));
9599 let ingress = CapturingExternalIngress::new(rx, closed.clone());
9600
9601 let config = LiveNodeConfig {
9602 environment: Environment::Sandbox,
9603 exec_engine: crate::config::LiveExecutionEngineConfig {
9604 reconciliation: false,
9605 ..Default::default()
9606 },
9607 delay_post_stop: Duration::ZERO,
9608 timeout_connection: Duration::from_millis(500),
9609 timeout_disconnection: Duration::from_millis(500),
9610 ..Default::default()
9611 };
9612
9613 let mut node = LiveNodeBuilder::from_config(config)
9614 .unwrap()
9615 .with_external_ingress(Box::new(ingress))
9616 .build()
9617 .expect("node builds with external message bus ingress");
9618 let handle = node.handle();
9619
9620 tokio::time::timeout(Duration::from_secs(30), async {
9621 let run = node.run();
9622 tokio::pin!(run);
9623
9624 let drive = async {
9625 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9626
9627 drop(tx);
9628
9629 wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
9630 assert!(
9631 handle.is_running(),
9632 "node should keep running after ingress closes"
9633 );
9634 handle.stop();
9635 };
9636
9637 tokio::select! {
9638 biased;
9639
9640 () = drive => {}
9641 result = &mut run => {
9642 panic!("node stopped before ingress close was observed: {result:?}");
9643 }
9644 }
9645
9646 run.await.expect("node should stop cleanly");
9647 })
9648 .await
9649 .expect("live node should close ingress and stop before timeout");
9650
9651 assert_eq!(handle.state(), NodeState::Stopped);
9652 }
9653
9654 #[rstest]
9655 #[tokio::test(flavor = "current_thread")]
9656 async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
9657 let closed = Rc::new(Cell::new(false));
9658 let ingress = FailingExternalIngress::new(closed.clone());
9659
9660 let config = LiveNodeConfig {
9661 environment: Environment::Sandbox,
9662 exec_engine: crate::config::LiveExecutionEngineConfig {
9663 reconciliation: false,
9664 ..Default::default()
9665 },
9666 delay_post_stop: Duration::ZERO,
9667 timeout_connection: Duration::from_millis(500),
9668 timeout_disconnection: Duration::from_millis(500),
9669 ..Default::default()
9670 };
9671
9672 let mut node = LiveNodeBuilder::from_config(config)
9673 .unwrap()
9674 .with_external_ingress(Box::new(ingress))
9675 .build()
9676 .expect("node builds with external message bus ingress");
9677 let handle = node.handle();
9678
9679 let err = node.run().await.expect_err("run should fail");
9680
9681 assert!(
9682 err.to_string()
9683 .contains("external ingress receiver unavailable")
9684 );
9685 assert_eq!(handle.state(), NodeState::Stopped);
9686 assert!(closed.get());
9687 }
9688
9689 #[rstest]
9690 fn test_node_build_rejects_existing_node_preserving_account_delivery(
9691 #[values(false, true)] first_builder: bool,
9692 #[values(false, true)] second_builder: bool,
9693 ) {
9694 let config = LiveNodeConfig {
9695 trader_id: TraderId::from("PROBE-001"),
9696 ..Default::default()
9697 };
9698
9699 let mut node = if first_builder {
9700 LiveNodeBuilder::from_config(config)
9701 .unwrap()
9702 .build()
9703 .unwrap()
9704 } else {
9705 LiveNode::build("Original".to_string(), Some(config)).unwrap()
9706 };
9707
9708 let bus = msgbus::get_message_bus();
9709
9710 let ExecutionEvent::Account(before) = stub_account_event() else {
9711 unreachable!()
9712 };
9713
9714 get_exec_event_sender()
9715 .send(ExecutionEvent::Account(before.clone()))
9716 .unwrap();
9717 assert_eq!(node.drain_runner_pending(), 1);
9718 assert_eq!(
9719 node.kernel
9720 .cache
9721 .borrow()
9722 .account(&before.account_id)
9723 .unwrap()
9724 .last_event(),
9725 Some(before)
9726 );
9727
9728 let result = if second_builder {
9729 LiveNodeBuilder::from_config(LiveNodeConfig::default())
9730 .unwrap()
9731 .build()
9732 } else {
9733 LiveNode::build("Extra".to_string(), None)
9734 };
9735
9736 assert_eq!(
9737 result.unwrap_err().to_string(),
9738 "A LiveNode already exists or is being built on this thread; drop it before building another"
9739 );
9740
9741 let ExecutionEvent::Account(after) = stub_account_event() else {
9742 unreachable!()
9743 };
9744
9745 get_exec_event_sender()
9746 .send(ExecutionEvent::Account(after.clone()))
9747 .unwrap();
9748 assert_eq!(node.drain_runner_pending(), 1);
9749 assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
9750 assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
9751 assert_eq!(
9752 node.kernel
9753 .cache
9754 .borrow()
9755 .account(&after.account_id)
9756 .unwrap()
9757 .last_event(),
9758 Some(after)
9759 );
9760 }
9761
9762 #[rstest]
9763 fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
9764 let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
9765 .unwrap()
9766 .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
9767 let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
9768 assert!(
9769 error
9770 .to_string()
9771 .contains("already exists or is being built")
9772 );
9773 anyhow::bail!("Event store construction failed")
9774 })
9775 .build();
9776
9777 assert_eq!(
9778 result.unwrap_err().to_string(),
9779 "Event store construction failed"
9780 );
9781
9782 let node = LiveNode::build("Retry".to_string(), None).unwrap();
9783 assert_eq!(node.state(), NodeState::Idle);
9784 }
9785
9786 #[rstest]
9787 fn test_node_build_allows_separate_threads() {
9788 let node = LiveNode::build("First".to_string(), None).unwrap();
9789
9790 let other_state = std::thread::spawn(|| {
9791 let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
9792 other.state()
9793 })
9794 .join()
9795 .unwrap();
9796
9797 assert_eq!(node.state(), NodeState::Idle);
9798 assert_eq!(other_state, NodeState::Idle);
9799 }
9800
9801 #[rstest]
9802 fn test_node_build_releases_thread_after_drop() {
9803 let mut node = LiveNode::build("First".to_string(), None).unwrap();
9804 node.dispose();
9805 assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
9806 drop(node);
9807
9808 let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
9809 assert_eq!(replacement.state(), NodeState::Idle);
9810 }
9811
9812 #[rstest]
9813 fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
9814 let config = LiveNodeConfig {
9815 event_store: Some(EventStoreConfig::default()),
9816 ..Default::default()
9817 };
9818
9819 let result = if builder {
9820 LiveNodeBuilder::from_config(config).unwrap().build()
9821 } else {
9822 LiveNode::build("Failure".to_string(), Some(config))
9823 };
9824
9825 assert!(result.unwrap_err().to_string().contains("factory"));
9826
9827 let node = LiveNode::build("Retry".to_string(), None).unwrap();
9828 assert_eq!(node.state(), NodeState::Idle);
9829 }
9830
9831 #[cfg(feature = "python")]
9832 #[rstest]
9833 fn test_node_build_and_initial_state() {
9834 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9835 .unwrap()
9836 .with_name("TestNode")
9837 .build()
9838 .unwrap();
9839
9840 assert_eq!(node.state(), NodeState::Idle);
9841 assert!(!node.is_running());
9842 assert_eq!(node.environment(), Environment::Sandbox);
9843 assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
9844 }
9845
9846 #[cfg(feature = "python")]
9847 #[rstest]
9848 fn test_node_build_replaces_stale_runner_senders() {
9849 replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
9850 replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
9851
9852 let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9853 .unwrap()
9854 .with_name("FirstNode")
9855 .build()
9856 .unwrap();
9857
9858 assert_eq!(first.state(), NodeState::Idle);
9859 drop(first);
9860
9861 let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9862 .unwrap()
9863 .with_name("SecondNode")
9864 .build()
9865 .unwrap();
9866
9867 assert_eq!(second.state(), NodeState::Idle);
9868 assert!(!second.is_running());
9869 }
9870
9871 #[cfg(feature = "python")]
9872 #[rstest]
9873 fn test_node_handle_reflects_node_state() {
9874 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9875 .unwrap()
9876 .with_name("TestNode")
9877 .build()
9878 .unwrap();
9879
9880 let handle = node.handle();
9881
9882 assert_eq!(handle.state(), NodeState::Idle);
9883 assert!(!handle.is_running());
9884 }
9885
9886 #[rstest]
9887 fn test_pending_drain_data_returns_false_when_empty() {
9888 let mut pending = PendingEvents::default();
9889
9890 assert!(!pending.drain_data());
9891 }
9892
9893 #[rstest]
9894 fn test_pending_drain_data_returns_true_when_non_empty() {
9895 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
9896
9897 let mut pending = PendingEvents::default();
9898 pending.data_evts.push(
9899 DataEvent::Instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
9900 .into(),
9901 );
9902
9903 assert!(pending.drain_data());
9904 assert!(pending.data_evts.is_empty());
9905 }
9906
9907 fn stub_data_event() -> DataEvent {
9908 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
9909
9910 DataEvent::Instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
9911 }
9912
9913 fn stub_data_command() -> DataCommand {
9914 use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
9915 use nautilus_core::{UUID4, UnixNanos};
9916 use nautilus_model::identifiers::Venue;
9917
9918 DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
9919 None,
9920 Venue::from("TEST"),
9921 UUID4::new(),
9922 UnixNanos::default(),
9923 None,
9924 None,
9925 )))
9926 }
9927
9928 fn stub_system_command() -> SystemCommand {
9929 SystemCommand::ReconnectSocket(ReconnectSocket::new(
9930 TraderId::from("TRADER-001"),
9931 ClientId::from("POLYMARKET"),
9932 Ustr::from("polymarket-market-streams"),
9933 UnixNanos::default(),
9934 ))
9935 }
9936
9937 #[rstest]
9938 fn test_flush_pending_data_drains_events_and_commands() {
9939 let (evt_tx, mut evt_rx) =
9940 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
9941 let (cmd_tx, mut cmd_rx) =
9942 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
9943
9944 let mut pending = PendingEvents::default();
9945
9946 pending.data_evts.push((stub_data_event()).into());
9948 pending.data_cmds.push(stub_data_command().into());
9949
9950 evt_tx.send((stub_data_event()).into()).unwrap();
9952 cmd_tx.send(stub_data_command().into()).unwrap();
9953
9954 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
9955
9956 assert!(pending.data_evts.is_empty());
9957 assert!(pending.data_cmds.is_empty());
9958 assert!(evt_rx.try_recv().is_err());
9959 assert!(cmd_rx.try_recv().is_err());
9960 }
9961
9962 #[rstest]
9963 fn test_flush_pending_data_drains_mixed_sources() {
9964 let (evt_tx, mut evt_rx) =
9965 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
9966 let (cmd_tx, mut cmd_rx) =
9967 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
9968
9969 let mut pending = PendingEvents::default();
9970
9971 pending.data_evts.push((stub_data_event()).into());
9973 cmd_tx.send(stub_data_command().into()).unwrap();
9974
9975 evt_tx.send((stub_data_event()).into()).unwrap();
9977 evt_tx.send((stub_data_event()).into()).unwrap();
9978 cmd_tx.send(stub_data_command().into()).unwrap();
9979
9980 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
9981
9982 assert!(pending.data_evts.is_empty());
9983 assert!(pending.data_cmds.is_empty());
9984 assert!(evt_rx.try_recv().is_err());
9985 assert!(cmd_rx.try_recv().is_err());
9986 }
9987
9988 #[rstest]
9989 fn test_pending_system_events_stay_separate_from_data() {
9990 let mut pending = PendingEvents::default();
9991
9992 let change = SocketStateChange::new(
9993 ClientId::from("BINANCE"),
9994 Some(Venue::from("BINANCE")),
9995 ustr::Ustr::from("binance-futures-market-streams"),
9996 SocketState::Connected,
9997 );
9998
9999 pending
10000 .system_events
10001 .push(SystemEvent::SocketState(change).into());
10002 let system_events = pending
10003 .take_system_events()
10004 .into_iter()
10005 .map(|message| message.dispatch(|value| value))
10006 .collect::<Vec<_>>();
10007
10008 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10009 assert!(pending.is_empty());
10010 }
10011
10012 #[rstest]
10013 fn test_pending_system_commands_stay_separate_from_data() {
10014 let mut pending = PendingEvents::default();
10015 let command = stub_system_command();
10016
10017 pending.system_commands.push(command.into());
10018 let system_commands = pending
10019 .take_system_commands()
10020 .into_iter()
10021 .map(|message| message.dispatch(|value| value))
10022 .collect::<Vec<_>>();
10023
10024 assert_eq!(system_commands, vec![command]);
10025 assert!(pending.is_empty());
10026 }
10027
10028 fn stub_time_event_handler() -> TimeEventMessage {
10029 use std::rc::Rc;
10030
10031 use nautilus_common::{
10032 runner::TimeEventMessage,
10033 timer::{TimeEvent, TimeEventCallback},
10034 };
10035 use nautilus_core::{UUID4, UnixNanos};
10036 use ustr::Ustr;
10037
10038 TimeEventMessage::new(
10039 TimeEvent::new(
10040 Ustr::from("test-timer"),
10041 UUID4::new(),
10042 UnixNanos::default(),
10043 UnixNanos::default(),
10044 ),
10045 TimeEventCallback::RustLocal(Rc::new(|_| {})),
10046 )
10047 }
10048
10049 fn stub_trading_command_message() -> TradingCommandMessage {
10050 use nautilus_common::messages::execution::query::QueryAccount;
10051 use nautilus_core::{UUID4, UnixNanos};
10052 use nautilus_model::identifiers::AccountId;
10053
10054 TradingCommandMessage::new(
10055 MessagingSwitchboard::exec_engine_execute(),
10056 TradingCommand::QueryAccount(QueryAccount::new(
10057 TraderId::from("TESTER-001"),
10058 None,
10059 AccountId::from("TEST-001"),
10060 UUID4::new(),
10061 UnixNanos::default(),
10062 None,
10063 None, )),
10065 )
10066 }
10067
10068 fn stub_exec_event() -> ExecutionEvent {
10069 use nautilus_model::{
10070 enums::{LiquiditySide, OrderSide},
10071 identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
10072 reports::FillReport,
10073 types::{Money, Price, Quantity},
10074 };
10075
10076 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
10077 AccountId::from("TEST-001"),
10078 InstrumentId::from("TEST.VENUE"),
10079 VenueOrderId::from("V-001"),
10080 TradeId::from("T-001"),
10081 OrderSide::Buy,
10082 Quantity::from("1.0"),
10083 Price::from("100.0"),
10084 Money::from("0.01 USD"),
10085 LiquiditySide::Maker,
10086 None,
10087 None,
10088 nautilus_core::UnixNanos::default(),
10089 nautilus_core::UnixNanos::default(),
10090 None,
10091 ))))
10092 }
10093
10094 #[rstest]
10095 fn test_flush_all_pending_drains_buffered_channels() {
10096 let (time_tx, mut time_rx) =
10097 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10098 let (system_evt_tx, mut system_evt_rx) =
10099 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10100 let (system_cmd_tx, mut system_cmd_rx) =
10101 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10102 let (data_evt_tx, mut data_evt_rx) =
10103 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10104 let (data_cmd_tx, mut data_cmd_rx) =
10105 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10106 let (exec_evt_tx, mut exec_evt_rx) =
10107 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10108 let (exec_cmd_tx, mut exec_cmd_rx) =
10109 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10110
10111 let mut pending = PendingEvents::default();
10112
10113 pending.data_evts.push((stub_data_event()).into());
10115 pending.data_cmds.push(stub_data_command().into());
10116
10117 time_tx.send((stub_time_event_handler()).into()).unwrap();
10119
10120 let change = SocketStateChange::new(
10121 ClientId::from("BINANCE"),
10122 Some(Venue::from("BINANCE")),
10123 Ustr::from("binance-futures-market-streams"),
10124 SocketState::Connected,
10125 );
10126 system_evt_tx
10127 .send((SystemEvent::SocketState(change)).into())
10128 .unwrap();
10129 system_cmd_tx.send((stub_system_command()).into()).unwrap();
10130 data_evt_tx.send((stub_data_event()).into()).unwrap();
10131 data_cmd_tx.send(stub_data_command().into()).unwrap();
10132 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10133 exec_cmd_tx
10134 .send(stub_trading_command_message().into())
10135 .unwrap();
10136
10137 flush_all_pending(
10138 &mut pending,
10139 &mut time_rx,
10140 &mut system_evt_rx,
10141 &mut system_cmd_rx,
10142 &mut exec_evt_rx,
10143 &mut exec_cmd_rx,
10144 &mut data_evt_rx,
10145 &mut data_cmd_rx,
10146 );
10147
10148 let system_events = pending
10149 .take_system_events()
10150 .into_iter()
10151 .map(|message| message.dispatch(|value| value))
10152 .collect::<Vec<_>>();
10153 let system_commands = pending
10154 .take_system_commands()
10155 .into_iter()
10156 .map(|message| message.dispatch(|value| value))
10157 .collect::<Vec<_>>();
10158 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10159 assert_eq!(system_commands, vec![stub_system_command()]);
10160 assert!(pending.data_evts.is_empty());
10161 assert!(pending.data_cmds.is_empty());
10162 assert!(pending.exec_reports.is_empty());
10163 assert!(pending.exec_cmds.is_empty());
10164 assert!(pending.order_evts.is_empty());
10165 assert!(time_rx.try_recv().is_err());
10166 assert!(system_evt_rx.try_recv().is_err());
10167 assert!(system_cmd_rx.try_recv().is_err());
10168 assert!(data_evt_rx.try_recv().is_err());
10169 assert!(data_cmd_rx.try_recv().is_err());
10170 assert!(exec_evt_rx.try_recv().is_err());
10171 assert!(exec_cmd_rx.try_recv().is_err());
10172 }
10173
10174 fn stub_order_event() -> ExecutionEvent {
10175 use nautilus_model::events::order::spec::OrderSubmittedSpec;
10176
10177 ExecutionEvent::Order(OrderEventAny::Submitted(
10178 OrderSubmittedSpec::builder().build(),
10179 ))
10180 }
10181
10182 fn stub_account_event() -> ExecutionEvent {
10183 use nautilus_core::{UUID4, UnixNanos};
10184 use nautilus_model::{
10185 enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
10186 };
10187
10188 ExecutionEvent::Account(AccountState::new(
10189 AccountId::from("TEST-001"),
10190 AccountType::Cash,
10191 vec![],
10192 vec![],
10193 true,
10194 UUID4::new(),
10195 UnixNanos::default(),
10196 UnixNanos::default(),
10197 None,
10198 ))
10199 }
10200
10201 #[rstest]
10202 fn test_flush_all_pending_routes_order_event_to_order_evts() {
10203 let (_time_tx, mut time_rx) =
10204 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10205 let (_system_evt_tx, mut system_evt_rx) =
10206 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10207 let (_system_cmd_tx, mut system_cmd_rx) =
10208 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10209 let (_data_evt_tx, mut data_evt_rx) =
10210 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10211 let (_data_cmd_tx, mut data_cmd_rx) =
10212 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10213 let (exec_evt_tx, mut exec_evt_rx) =
10214 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10215 let (_exec_cmd_tx, mut exec_cmd_rx) =
10216 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10217
10218 let mut pending = PendingEvents::default();
10219
10220 exec_evt_tx.send((stub_order_event()).into()).unwrap();
10221 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10222
10223 flush_all_pending(
10224 &mut pending,
10225 &mut time_rx,
10226 &mut system_evt_rx,
10227 &mut system_cmd_rx,
10228 &mut exec_evt_rx,
10229 &mut exec_cmd_rx,
10230 &mut data_evt_rx,
10231 &mut data_cmd_rx,
10232 );
10233
10234 assert!(pending.order_evts.is_empty());
10236 assert!(pending.exec_reports.is_empty());
10237 assert!(exec_evt_rx.try_recv().is_err());
10238 }
10239
10240 #[rstest]
10241 fn test_flush_all_pending_routes_account_event_immediately() {
10242 let (_time_tx, mut time_rx) =
10243 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10244 let (_system_evt_tx, mut system_evt_rx) =
10245 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10246 let (_system_cmd_tx, mut system_cmd_rx) =
10247 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10248 let (_data_evt_tx, mut data_evt_rx) =
10249 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10250 let (_data_cmd_tx, mut data_cmd_rx) =
10251 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10252 let (exec_evt_tx, mut exec_evt_rx) =
10253 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10254 let (_exec_cmd_tx, mut exec_cmd_rx) =
10255 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10256
10257 let mut pending = PendingEvents::default();
10258
10259 exec_evt_tx.send((stub_account_event()).into()).unwrap();
10260
10261 flush_all_pending(
10262 &mut pending,
10263 &mut time_rx,
10264 &mut system_evt_rx,
10265 &mut system_cmd_rx,
10266 &mut exec_evt_rx,
10267 &mut exec_cmd_rx,
10268 &mut data_evt_rx,
10269 &mut data_cmd_rx,
10270 );
10271
10272 assert!(pending.exec_reports.is_empty());
10274 assert!(pending.order_evts.is_empty());
10275 assert!(pending.exec_cmds.is_empty());
10276 assert!(exec_evt_rx.try_recv().is_err());
10277 }
10278
10279 #[rstest]
10280 fn test_pending_is_empty_when_default() {
10281 let pending = PendingEvents::default();
10282
10283 assert!(pending.is_empty());
10284 }
10285
10286 #[rstest]
10287 fn test_pending_is_empty_false_with_data_evt() {
10288 let mut pending = PendingEvents::default();
10289 pending.data_evts.push((stub_data_event()).into());
10290
10291 assert!(!pending.is_empty());
10292 }
10293
10294 #[rstest]
10295 fn test_pending_is_empty_false_with_data_cmd() {
10296 let mut pending = PendingEvents::default();
10297 pending.data_cmds.push(stub_data_command().into());
10298
10299 assert!(!pending.is_empty());
10300 }
10301
10302 #[rstest]
10303 fn test_pending_is_empty_false_with_exec_cmd() {
10304 let mut pending = PendingEvents::default();
10305 pending
10306 .exec_cmds
10307 .push(stub_trading_command_message().into());
10308
10309 assert!(!pending.is_empty());
10310 }
10311
10312 #[rstest]
10313 fn test_pending_drain_preserves_trading_command_target() {
10314 std::thread::spawn(|| {
10315 msgbus::get_message_bus().borrow_mut().dispose();
10316 let risk_commands = Rc::new(RefCell::new(Vec::new()));
10317 let exec_commands = Rc::new(RefCell::new(Vec::new()));
10318
10319 let risk_commands_handler = risk_commands.clone();
10320 msgbus::register_trading_command_endpoint(
10321 MessagingSwitchboard::risk_engine_execute(),
10322 TypedIntoHandler::from(move |command: TradingCommand| {
10323 risk_commands_handler.borrow_mut().push(command);
10324 }),
10325 );
10326
10327 let exec_commands_handler = exec_commands.clone();
10328 msgbus::register_trading_command_endpoint(
10329 MessagingSwitchboard::exec_engine_execute(),
10330 TypedIntoHandler::from(move |command: TradingCommand| {
10331 exec_commands_handler.borrow_mut().push(command);
10332 }),
10333 );
10334
10335 let mut pending = PendingEvents::default();
10336 pending.exec_cmds.push(
10337 TradingCommandMessage::new(
10338 MessagingSwitchboard::risk_engine_execute(),
10339 TradingCommand::QueryAccount(QueryAccount::new(
10340 TraderId::from("TESTER-001"),
10341 None,
10342 AccountId::from("TEST-001"),
10343 UUID4::new(),
10344 UnixNanos::default(),
10345 None,
10346 None,
10347 )),
10348 )
10349 .into(),
10350 );
10351
10352 pending.drain();
10353
10354 assert!(pending.is_empty());
10355 assert_eq!(risk_commands.borrow().len(), 1);
10356 assert!(matches!(
10357 &risk_commands.borrow()[0],
10358 TradingCommand::QueryAccount(_)
10359 ));
10360 assert_eq!(exec_commands.borrow().as_slice(), &[]);
10361 })
10362 .join()
10363 .unwrap();
10364 }
10365
10366 #[rstest]
10367 fn test_pending_is_empty_false_with_exec_report() {
10368 let mut pending = PendingEvents::default();
10369
10370 if let ExecutionEvent::Report(report) = stub_exec_event() {
10371 pending.exec_reports.push((report).into());
10372 }
10373
10374 assert!(!pending.is_empty());
10375 }
10376
10377 #[rstest]
10378 fn test_pending_is_empty_false_with_order_evt() {
10379 let mut pending = PendingEvents::default();
10380
10381 if let ExecutionEvent::Order(order_evt) = stub_order_event() {
10382 pending.order_evts.push((order_evt).into());
10383 }
10384
10385 assert!(!pending.is_empty());
10386 }
10387
10388 fn stub_submitted_batch_event() -> ExecutionEvent {
10389 use nautilus_model::{
10390 events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
10391 identifiers::ClientOrderId,
10392 };
10393
10394 let events = vec![
10395 OrderSubmittedSpec::builder()
10396 .client_order_id(ClientOrderId::from("O-001"))
10397 .build(),
10398 OrderSubmittedSpec::builder()
10399 .client_order_id(ClientOrderId::from("O-002"))
10400 .build(),
10401 ];
10402
10403 ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
10404 }
10405
10406 fn stub_canceled_batch_event() -> ExecutionEvent {
10407 use nautilus_model::{
10408 events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
10409 identifiers::ClientOrderId,
10410 };
10411
10412 let events = vec![
10413 OrderCanceledSpec::builder()
10414 .client_order_id(ClientOrderId::from("O-001"))
10415 .build(),
10416 OrderCanceledSpec::builder()
10417 .client_order_id(ClientOrderId::from("O-002"))
10418 .build(),
10419 ];
10420
10421 ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
10422 }
10423
10424 #[rstest]
10425 fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
10426 let (_time_tx, mut time_rx) =
10427 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10428 let (_system_evt_tx, mut system_evt_rx) =
10429 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10430 let (_system_cmd_tx, mut system_cmd_rx) =
10431 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10432 let (_data_evt_tx, mut data_evt_rx) =
10433 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10434 let (_data_cmd_tx, mut data_cmd_rx) =
10435 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10436 let (exec_evt_tx, mut exec_evt_rx) =
10437 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10438 let (_exec_cmd_tx, mut exec_cmd_rx) =
10439 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10440
10441 let mut pending = PendingEvents::default();
10442
10443 exec_evt_tx
10444 .send((stub_submitted_batch_event()).into())
10445 .unwrap();
10446
10447 flush_all_pending(
10448 &mut pending,
10449 &mut time_rx,
10450 &mut system_evt_rx,
10451 &mut system_cmd_rx,
10452 &mut exec_evt_rx,
10453 &mut exec_cmd_rx,
10454 &mut data_evt_rx,
10455 &mut data_cmd_rx,
10456 );
10457
10458 assert!(pending.order_evts.is_empty());
10460 assert!(exec_evt_rx.try_recv().is_err());
10461 }
10462
10463 #[rstest]
10464 fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
10465 let (_time_tx, mut time_rx) =
10466 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10467 let (_system_evt_tx, mut system_evt_rx) =
10468 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10469 let (_system_cmd_tx, mut system_cmd_rx) =
10470 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10471 let (_data_evt_tx, mut data_evt_rx) =
10472 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10473 let (_data_cmd_tx, mut data_cmd_rx) =
10474 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10475 let (exec_evt_tx, mut exec_evt_rx) =
10476 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10477 let (_exec_cmd_tx, mut exec_cmd_rx) =
10478 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10479
10480 let mut pending = PendingEvents::default();
10481
10482 exec_evt_tx
10483 .send((stub_canceled_batch_event()).into())
10484 .unwrap();
10485
10486 flush_all_pending(
10487 &mut pending,
10488 &mut time_rx,
10489 &mut system_evt_rx,
10490 &mut system_cmd_rx,
10491 &mut exec_evt_rx,
10492 &mut exec_cmd_rx,
10493 &mut data_evt_rx,
10494 &mut data_cmd_rx,
10495 );
10496
10497 assert!(pending.order_evts.is_empty());
10499 assert!(exec_evt_rx.try_recv().is_err());
10500 }
10501
10502 #[rstest]
10503 fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
10504 use nautilus_model::identifiers::ClientOrderId;
10505
10506 let (exec_evt_tx, mut exec_evt_rx) =
10507 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10508
10509 exec_evt_tx
10510 .send((stub_canceled_batch_event()).into())
10511 .unwrap();
10512
10513 let mut pending = PendingEvents::default();
10514
10515 while let Ok(evt) = exec_evt_rx.try_recv() {
10516 pending.push_exec_event(evt);
10517 }
10518
10519 let events: Vec<_> = pending
10520 .order_evts
10521 .drain(..)
10522 .map(|event| event.dispatch(|event| event))
10523 .collect();
10524 assert_eq!(events.len(), 2);
10525 assert!(
10526 matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
10527 );
10528 assert!(
10529 matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
10530 );
10531 }
10532
10533 #[rstest]
10534 #[cfg_attr(
10535 not(all(feature = "simulation", madsim)),
10536 tokio::test(start_paused = true)
10537 )]
10538 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10539 async fn test_submission_confirmation_prevents_duplicate_tracking(
10540 #[values(false, true)] report_confirmation: bool,
10541 #[values(false, true)] exhausted: bool,
10542 ) {
10543 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
10544 let client_order_id = order.client_order_id();
10545 let account_id = AccountId::from("TEST-001");
10546 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10547 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10548 OrderPendingUpdateSpec::builder()
10549 .trader_id(order.trader_id())
10550 .strategy_id(order.strategy_id())
10551 .instrument_id(order.instrument_id())
10552 .client_order_id(client_order_id)
10553 .account_id(account_id)
10554 .build(),
10555 )));
10556 advance_clock(Duration::from_millis(101)).await;
10557 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10558
10559 if exhausted {
10560 advance_clock(Duration::from_millis(101)).await;
10561 let result = node.exec_manager.check_inflight_orders();
10562 assert!(result.events.is_empty());
10563 assert!(result.queries.is_empty());
10564 assert_eq!(
10565 node.exec_manager
10566 .take_submission_recovery_exhaustions()
10567 .len(),
10568 1
10569 );
10570 }
10571
10572 if report_confirmation {
10573 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
10574 OrderStatusReport::new(
10575 account_id,
10576 order.instrument_id(),
10577 Some(client_order_id),
10578 venue_order_id,
10579 Some(order.order_side()),
10580 order.order_type(),
10581 order.time_in_force(),
10582 OrderStatus::Submitted,
10583 order.quantity(),
10584 Quantity::zero(3),
10585 UnixNanos::default(),
10586 UnixNanos::default(),
10587 UnixNanos::default(),
10588 None,
10589 )
10590 .with_price(order.price().unwrap()),
10591 ))));
10592 node.exec_manager
10593 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
10594 advance_clock(Duration::from_millis(101)).await;
10595 let result = node.exec_manager.check_inflight_orders();
10596 assert!(result.queries.is_empty());
10597 assert!(result.events.is_empty());
10598 }
10599
10600 let mut update = OrderUpdatedSpec::builder()
10601 .trader_id(order.trader_id())
10602 .strategy_id(order.strategy_id())
10603 .instrument_id(order.instrument_id())
10604 .client_order_id(client_order_id)
10605 .account_id(account_id)
10606 .quantity(order.quantity())
10607 .price(order.price().unwrap())
10608 .build();
10609
10610 if !report_confirmation {
10611 update.venue_order_id = Some(venue_order_id);
10612 }
10613 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
10614 assert_eq!(
10615 node.kernel
10616 .cache
10617 .borrow()
10618 .order(&client_order_id)
10619 .unwrap()
10620 .status(),
10621 OrderStatus::Submitted
10622 );
10623
10624 for _ in 0..3 {
10625 node.exec_manager
10626 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
10627 advance_clock(Duration::from_millis(101)).await;
10628 let result = node.exec_manager.check_inflight_orders();
10629 assert!(result.queries.is_empty());
10630 assert!(result.events.is_empty());
10631 assert!(
10632 node.exec_manager
10633 .take_submission_recovery_exhaustions()
10634 .is_empty()
10635 );
10636 }
10637 }
10638
10639 #[rstest]
10640 #[cfg_attr(
10641 not(all(feature = "simulation", madsim)),
10642 tokio::test(start_paused = true)
10643 )]
10644 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10645 async fn test_submission_report_confirmation_preserves_later_command_budget(
10646 #[values(false, true)] cancel: bool,
10647 #[values(false, true)] later_fill: bool,
10648 #[values(false, true)] repeated_report: bool,
10649 #[values(
10650 SubmissionRecoveryPolicy::ResolveLocally,
10651 SubmissionRecoveryPolicy::RetainUnresolved
10652 )]
10653 policy: SubmissionRecoveryPolicy,
10654 ) {
10655 let (mut node, order, instrument) = venue_evidence_node(policy);
10656 let client_order_id = order.client_order_id();
10657 let account_id = AccountId::from("TEST-001");
10658 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10659 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10660 OrderPendingUpdateSpec::builder()
10661 .trader_id(order.trader_id())
10662 .strategy_id(order.strategy_id())
10663 .instrument_id(order.instrument_id())
10664 .client_order_id(client_order_id)
10665 .account_id(account_id)
10666 .build(),
10667 )));
10668 node.exec_manager.register_inflight(client_order_id);
10669 advance_clock(Duration::from_millis(101)).await;
10670 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10671 let report = ExecutionReport::Order(Box::new(
10672 OrderStatusReport::new(
10673 account_id,
10674 order.instrument_id(),
10675 Some(client_order_id),
10676 venue_order_id,
10677 Some(order.order_side()),
10678 order.order_type(),
10679 order.time_in_force(),
10680 OrderStatus::Submitted,
10681 order.quantity(),
10682 Quantity::zero(3),
10683 UnixNanos::default(),
10684 UnixNanos::default(),
10685 UnixNanos::default(),
10686 None,
10687 )
10688 .with_price(order.price().unwrap()),
10689 ));
10690 node.process_exec_event(ExecutionEvent::Report(report.clone()));
10691 advance_clock(Duration::from_millis(101)).await;
10692 let result = node.exec_manager.check_inflight_orders();
10693 assert!(result.events.is_empty());
10694 assert!(result.queries.is_empty());
10695
10696 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
10697 OrderUpdatedSpec::builder()
10698 .trader_id(order.trader_id())
10699 .strategy_id(order.strategy_id())
10700 .instrument_id(order.instrument_id())
10701 .client_order_id(client_order_id)
10702 .account_id(account_id)
10703 .quantity(order.quantity())
10704 .price(order.price().unwrap())
10705 .build(),
10706 )));
10707 let pending = if cancel {
10708 OrderEventAny::PendingCancel(
10709 OrderPendingCancelSpec::builder()
10710 .trader_id(order.trader_id())
10711 .strategy_id(order.strategy_id())
10712 .instrument_id(order.instrument_id())
10713 .client_order_id(client_order_id)
10714 .account_id(account_id)
10715 .build(),
10716 )
10717 } else {
10718 OrderEventAny::PendingUpdate(
10719 OrderPendingUpdateSpec::builder()
10720 .trader_id(order.trader_id())
10721 .strategy_id(order.strategy_id())
10722 .instrument_id(order.instrument_id())
10723 .client_order_id(client_order_id)
10724 .account_id(account_id)
10725 .build(),
10726 )
10727 };
10728 node.process_exec_event(ExecutionEvent::Order(pending));
10729 node.exec_manager.register_inflight(client_order_id);
10730
10731 if repeated_report {
10732 node.process_exec_event(ExecutionEvent::Report(report));
10733 }
10734 advance_clock(Duration::from_millis(50)).await;
10735
10736 if later_fill {
10737 let cached = node
10738 .kernel
10739 .cache
10740 .borrow()
10741 .order_owned(&client_order_id)
10742 .unwrap();
10743 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
10744 .last_qty(Quantity::from("0.400"))
10745 .last_px(order.price().unwrap())
10746 .commission(Money::from("0 USDT"))
10747 .without_position_id()
10748 .build()
10749 else {
10750 unreachable!()
10751 };
10752 fill.venue_order_id = venue_order_id;
10753 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
10754 }
10755 let early = node.exec_manager.check_inflight_orders();
10756 assert!(early.events.is_empty());
10757 assert!(early.queries.is_empty());
10758 advance_clock(Duration::from_millis(51)).await;
10759 let first = node.exec_manager.check_inflight_orders();
10760 assert_eq!(first.queries.len(), usize::from(!repeated_report));
10761 assert!(first.events.is_empty());
10762 advance_clock(Duration::from_millis(101)).await;
10763 let second = node.exec_manager.check_inflight_orders();
10764 assert!(second.queries.is_empty());
10765 if repeated_report {
10766 assert!(second.events.is_empty());
10767 } else {
10768 assert!(
10769 matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
10770 );
10771 }
10772 assert!(
10773 node.exec_manager
10774 .take_submission_recovery_exhaustions()
10775 .is_empty()
10776 );
10777 }
10778
10779 #[rstest]
10780 #[cfg_attr(
10781 not(all(feature = "simulation", madsim)),
10782 tokio::test(start_paused = true)
10783 )]
10784 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10785 async fn test_confirmed_submission_command_dispatch_starts_own_budget(
10786 #[values(false, true)] report_confirmation: bool,
10787 #[values(false, true)] cancel: bool,
10788 #[values(false, true)] batch: bool,
10789 #[values(false, true)] pending_before_dispatch: bool,
10790 #[values(false, true)] missing_order_check: bool,
10791 #[values(
10792 SubmissionRecoveryPolicy::ResolveLocally,
10793 SubmissionRecoveryPolicy::RetainUnresolved
10794 )]
10795 policy: SubmissionRecoveryPolicy,
10796 ) {
10797 let (mut node, order, instrument) = venue_evidence_node_with_config(
10798 crate::config::LiveExecutionEngineConfig {
10799 reconciliation: true,
10800 inflight_check_threshold_ms: 100,
10801 inflight_check_retries: 2,
10802 open_check_threshold_ms: 5_000,
10803 open_check_open_only: !missing_order_check,
10804 open_check_lookback_mins: None,
10805 open_check_missing_retries: 5,
10806 single_order_query_delay_ms: 0,
10807 submission_recovery_policy: policy,
10808 ..Default::default()
10809 },
10810 OrderType::Limit,
10811 true,
10812 );
10813 let client_order_id = order.client_order_id();
10814 let client_id = ClientId::from("TEST-EVIDENCE");
10815 let account_id = AccountId::from("TEST-001");
10816 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10817 node.kernel
10818 .exec_engine
10819 .borrow_mut()
10820 .register_client(Box::new(StubExecutionClient::new(
10821 client_id,
10822 account_id,
10823 instrument.id().venue,
10824 OmsType::Netting,
10825 None,
10826 )))
10827 .unwrap();
10828
10829 advance_clock(Duration::from_millis(101)).await;
10831 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10832
10833 if report_confirmation {
10834 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10835 OrderPendingUpdateSpec::builder()
10836 .trader_id(order.trader_id())
10837 .strategy_id(order.strategy_id())
10838 .instrument_id(order.instrument_id())
10839 .client_order_id(client_order_id)
10840 .account_id(account_id)
10841 .build(),
10842 )));
10843 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
10844 OrderStatusReport::new(
10845 account_id,
10846 order.instrument_id(),
10847 Some(client_order_id),
10848 venue_order_id,
10849 Some(order.order_side()),
10850 order.order_type(),
10851 order.time_in_force(),
10852 OrderStatus::Submitted,
10853 order.quantity(),
10854 Quantity::zero(3),
10855 UnixNanos::default(),
10856 UnixNanos::default(),
10857 UnixNanos::default(),
10858 None,
10859 )
10860 .with_price(order.price().unwrap()),
10861 ))));
10862 }
10863
10864 let mut update = OrderUpdatedSpec::builder()
10865 .trader_id(order.trader_id())
10866 .strategy_id(order.strategy_id())
10867 .instrument_id(order.instrument_id())
10868 .client_order_id(client_order_id)
10869 .account_id(account_id)
10870 .quantity(order.quantity())
10871 .price(order.price().unwrap())
10872 .build();
10873
10874 if !report_confirmation {
10875 update.venue_order_id = Some(venue_order_id);
10876 }
10877 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
10878 assert_eq!(
10879 node.kernel
10880 .cache
10881 .borrow()
10882 .order(&client_order_id)
10883 .unwrap()
10884 .status(),
10885 OrderStatus::Submitted,
10886 );
10887 advance_clock(Duration::from_millis(101)).await;
10888 let confirmed = node.exec_manager.check_inflight_orders();
10889 assert!(confirmed.queries.is_empty());
10890 assert!(confirmed.events.is_empty());
10891
10892 let pending = if cancel {
10893 OrderEventAny::PendingCancel(
10894 OrderPendingCancelSpec::builder()
10895 .trader_id(order.trader_id())
10896 .strategy_id(order.strategy_id())
10897 .instrument_id(order.instrument_id())
10898 .client_order_id(client_order_id)
10899 .account_id(account_id)
10900 .build(),
10901 )
10902 } else {
10903 OrderEventAny::PendingUpdate(
10904 OrderPendingUpdateSpec::builder()
10905 .trader_id(order.trader_id())
10906 .strategy_id(order.strategy_id())
10907 .instrument_id(order.instrument_id())
10908 .client_order_id(client_order_id)
10909 .account_id(account_id)
10910 .build(),
10911 )
10912 };
10913 let command = venue_evidence_command(&order, cancel, batch);
10914
10915 if pending_before_dispatch {
10916 node.process_exec_event(ExecutionEvent::Order(pending.clone()));
10917 }
10918 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
10919 MessagingSwitchboard::exec_engine_execute(),
10920 command,
10921 )));
10922 advance_clock(Duration::from_millis(50)).await;
10923
10924 if missing_order_check {
10925 let engine = node.kernel.exec_engine.borrow();
10926 let clients = engine.get_all_clients();
10927 let check = node
10928 .exec_manager
10929 .prepare_open_order_report_check(UUID4::new(), &clients);
10930 assert_eq!(check.filtered_orders.len(), 1);
10931 let missing = node.exec_manager.reconcile_open_order_reports(
10932 &check,
10933 Vec::new(),
10934 &IndexSet::from([client_id]),
10935 &IndexSet::new(),
10936 &clients,
10937 );
10938 assert!(missing.events.is_empty());
10939 assert!(missing.targeted_queries.is_empty());
10940 assert_eq!(
10941 node.exec_manager.recon_check_retry_count(&client_order_id),
10942 1
10943 );
10944 }
10945 let early = node.exec_manager.check_inflight_orders();
10946 assert!(early.queries.is_empty());
10947 assert!(early.events.is_empty());
10948
10949 if !pending_before_dispatch {
10950 assert_eq!(
10953 node.kernel
10954 .cache
10955 .borrow()
10956 .order(&client_order_id)
10957 .unwrap()
10958 .status(),
10959 OrderStatus::Submitted,
10960 );
10961 node.process_exec_event(ExecutionEvent::Order(pending));
10962 }
10963
10964 if policy == SubmissionRecoveryPolicy::RetainUnresolved {
10965 node.exec_manager
10966 .register_submission(order.init_event(), Some(client_id));
10967 }
10968 advance_clock(Duration::from_millis(51)).await;
10969 let first = node.exec_manager.check_inflight_orders();
10970 assert_eq!(first.queries.len(), 1);
10971 assert!(
10972 matches!(&first.queries[0], TradingCommand::QueryOrder(query)
10973 if query.client_order_id == client_order_id)
10974 );
10975 assert!(first.events.is_empty());
10976 advance_clock(Duration::from_millis(101)).await;
10977 let exhausted = node.exec_manager.check_inflight_orders();
10978 assert!(exhausted.queries.is_empty());
10979 assert!(
10980 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
10981 if event.client_order_id == client_order_id)
10982 );
10983 assert!(
10984 node.exec_manager
10985 .take_submission_recovery_exhaustions()
10986 .is_empty()
10987 );
10988 node.process_reconciliation_events(&exhausted.events);
10989 assert_eq!(
10990 node.kernel
10991 .cache
10992 .borrow()
10993 .order(&client_order_id)
10994 .unwrap()
10995 .status(),
10996 OrderStatus::Canceled,
10997 );
10998 }
10999
11000 #[rstest]
11001 #[cfg_attr(
11002 not(all(feature = "simulation", madsim)),
11003 tokio::test(start_paused = true)
11004 )]
11005 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11006 async fn test_submission_command_report_cleanup_matches_default(
11007 #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11008 #[values(
11009 SubmissionRecoveryPolicy::ResolveLocally,
11010 SubmissionRecoveryPolicy::RetainUnresolved
11011 )]
11012 policy: SubmissionRecoveryPolicy,
11013 ) {
11014 let (mut node, order, instrument) = venue_evidence_node(policy);
11015 let client_order_id = order.client_order_id();
11016 let client_id = ClientId::from("TEST-EVIDENCE");
11017 let account_id = AccountId::from("TEST-001");
11018 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11019 node.kernel
11020 .exec_engine
11021 .borrow_mut()
11022 .register_client(Box::new(StubExecutionClient::new(
11023 client_id,
11024 account_id,
11025 instrument.id().venue,
11026 OmsType::Netting,
11027 None,
11028 )))
11029 .unwrap();
11030 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11031 &order,
11032 account_id,
11033 venue_order_id,
11034 )));
11035 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11036 MessagingSwitchboard::exec_engine_execute(),
11037 venue_evidence_command(&order, true, false),
11038 )));
11039 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11040 OrderPendingCancelSpec::builder()
11041 .trader_id(order.trader_id())
11042 .strategy_id(order.strategy_id())
11043 .instrument_id(order.instrument_id())
11044 .client_order_id(client_order_id)
11045 .account_id(account_id)
11046 .venue_order_id(venue_order_id)
11047 .build(),
11048 )));
11049 advance_clock(Duration::from_millis(101)).await;
11050 let first = node.exec_manager.check_inflight_orders();
11051 assert_eq!(first.queries.len(), 1);
11052 assert!(first.events.is_empty());
11053
11054 let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11055 Quantity::from("0.400")
11056 } else {
11057 Quantity::zero(3)
11058 };
11059
11060 let report = OrderStatusReport::new(
11061 account_id,
11062 order.instrument_id(),
11063 Some(client_order_id),
11064 venue_order_id,
11065 Some(order.order_side()),
11066 order.order_type(),
11067 order.time_in_force(),
11068 report_status,
11069 order.quantity(),
11070 filled_qty,
11071 UnixNanos::default(),
11072 UnixNanos::default(),
11073 UnixNanos::default(),
11074 None,
11075 )
11076 .with_price(order.price().unwrap())
11077 .with_avg_px(dec!(100.00));
11078 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11079 report,
11080 ))));
11081 advance_clock(Duration::from_millis(101)).await;
11082 let exhausted = node.exec_manager.check_inflight_orders();
11083
11084 assert!(exhausted.queries.is_empty());
11085 assert!(exhausted.events.is_empty());
11086 assert!(
11087 node.exec_manager
11088 .take_submission_recovery_exhaustions()
11089 .is_empty()
11090 );
11091 let cache = node.kernel.cache.borrow();
11092 let cached = cache.order_ref(&client_order_id).unwrap();
11093 assert_eq!(cached.status(), OrderStatus::PendingCancel);
11094 assert_eq!(cached.filled_qty(), filled_qty);
11095 }
11096
11097 #[rstest]
11098 #[case::accepted_modify(false, "accepted")]
11099 #[case::accepted_cancel(true, "accepted")]
11100 #[case::accepted_batch_modify(false, "accepted_batch")]
11101 #[case::accepted_batch_cancel(true, "accepted_batch")]
11102 #[case::updated_cancel(true, "updated")]
11103 #[case::triggered_cancel(true, "triggered")]
11104 #[cfg_attr(
11105 not(all(feature = "simulation", madsim)),
11106 tokio::test(start_paused = true)
11107 )]
11108 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11109 async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
11110 #[case] cancel: bool,
11111 #[case] evidence: &str,
11112 #[values(false, true)] batch: bool,
11113 #[values(false, true)] report_first: bool,
11114 #[values(
11115 SubmissionRecoveryPolicy::ResolveLocally,
11116 SubmissionRecoveryPolicy::RetainUnresolved
11117 )]
11118 policy: SubmissionRecoveryPolicy,
11119 ) {
11120 use nautilus_model::events::order::spec::OrderTriggeredSpec;
11121
11122 let triggered = evidence == "triggered";
11123
11124 let order_type = if triggered {
11125 OrderType::StopLimit
11126 } else {
11127 OrderType::Limit
11128 };
11129 let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
11130 let client_order_id = order.client_order_id();
11131 let account_id = AccountId::from("TEST-001");
11132 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11133 node.kernel
11134 .exec_engine
11135 .borrow_mut()
11136 .register_client(Box::new(StubExecutionClient::new(
11137 ClientId::from("TEST-EVIDENCE"),
11138 account_id,
11139 instrument.id().venue,
11140 OmsType::Netting,
11141 None,
11142 )))
11143 .unwrap();
11144 advance_clock(Duration::from_millis(101)).await;
11145 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11146
11147 if triggered || report_first {
11148 let previous_pending = if cancel {
11150 OrderEventAny::PendingUpdate(
11151 OrderPendingUpdateSpec::builder()
11152 .trader_id(order.trader_id())
11153 .strategy_id(order.strategy_id())
11154 .instrument_id(order.instrument_id())
11155 .client_order_id(client_order_id)
11156 .account_id(account_id)
11157 .build(),
11158 )
11159 } else {
11160 OrderEventAny::PendingCancel(
11161 OrderPendingCancelSpec::builder()
11162 .trader_id(order.trader_id())
11163 .strategy_id(order.strategy_id())
11164 .instrument_id(order.instrument_id())
11165 .client_order_id(client_order_id)
11166 .account_id(account_id)
11167 .build(),
11168 )
11169 };
11170 node.process_exec_event(ExecutionEvent::Order(previous_pending));
11171 }
11172 advance_clock(Duration::from_millis(10)).await;
11173 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11174 MessagingSwitchboard::exec_engine_execute(),
11175 venue_evidence_command(&order, cancel, batch),
11176 )));
11177
11178 if report_first {
11179 let report = OrderStatusReport::new(
11180 account_id,
11181 instrument.id(),
11182 Some(client_order_id),
11183 venue_order_id,
11184 Some(order.order_side()),
11185 order.order_type(),
11186 order.time_in_force(),
11187 OrderStatus::Submitted,
11188 order.quantity(),
11189 Quantity::zero(3),
11190 UnixNanos::default(),
11191 UnixNanos::default(),
11192 UnixNanos::default(),
11193 None,
11194 )
11195 .with_price(order.price().unwrap());
11196 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11197 report,
11198 ))));
11199 assert!(
11200 node.kernel
11201 .cache
11202 .borrow()
11203 .order(&client_order_id)
11204 .unwrap()
11205 .venue_order_id()
11206 .is_none()
11207 );
11208 }
11209 advance_clock(Duration::from_millis(40)).await;
11210 let acknowledgement = if triggered {
11211 OrderEventAny::Triggered(
11212 OrderTriggeredSpec::builder()
11213 .trader_id(order.trader_id())
11214 .strategy_id(order.strategy_id())
11215 .instrument_id(order.instrument_id())
11216 .client_order_id(client_order_id)
11217 .account_id(account_id)
11218 .venue_order_id(venue_order_id)
11219 .build(),
11220 )
11221 } else if evidence == "updated" {
11222 OrderEventAny::Updated(
11223 OrderUpdatedSpec::builder()
11224 .trader_id(order.trader_id())
11225 .strategy_id(order.strategy_id())
11226 .instrument_id(order.instrument_id())
11227 .client_order_id(client_order_id)
11228 .account_id(account_id)
11229 .venue_order_id(venue_order_id)
11230 .quantity(order.quantity())
11231 .price(order.price().unwrap())
11232 .build(),
11233 )
11234 } else {
11235 TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
11236 };
11237
11238 if evidence == "accepted_batch" {
11239 let OrderEventAny::Accepted(accepted) = acknowledgement else {
11240 unreachable!()
11241 };
11242 node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
11243 vec![accepted],
11244 )));
11245 } else {
11246 node.process_exec_event(ExecutionEvent::Order(acknowledgement));
11247 }
11248 assert_eq!(
11249 node.kernel
11250 .cache
11251 .borrow()
11252 .order(&client_order_id)
11253 .unwrap()
11254 .status(),
11255 match evidence {
11256 "triggered" => OrderStatus::Triggered,
11257 "updated" => OrderStatus::Submitted,
11258 _ => OrderStatus::Accepted,
11259 }
11260 );
11261 advance_clock(Duration::from_millis(20)).await;
11262 let pending = if cancel {
11263 OrderEventAny::PendingCancel(
11264 OrderPendingCancelSpec::builder()
11265 .trader_id(order.trader_id())
11266 .strategy_id(order.strategy_id())
11267 .instrument_id(order.instrument_id())
11268 .client_order_id(client_order_id)
11269 .account_id(account_id)
11270 .build(),
11271 )
11272 } else {
11273 OrderEventAny::PendingUpdate(
11274 OrderPendingUpdateSpec::builder()
11275 .trader_id(order.trader_id())
11276 .strategy_id(order.strategy_id())
11277 .instrument_id(order.instrument_id())
11278 .client_order_id(client_order_id)
11279 .account_id(account_id)
11280 .build(),
11281 )
11282 };
11283 node.process_exec_event(ExecutionEvent::Order(pending));
11284 let early = node.exec_manager.check_inflight_orders();
11285 assert!(early.queries.is_empty());
11286 assert!(early.events.is_empty());
11287 advance_clock(Duration::from_millis(41)).await;
11288 let first = node.exec_manager.check_inflight_orders();
11289 let recover_command =
11291 triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
11292 assert_eq!(first.queries.len(), usize::from(recover_command));
11293 assert!(first.events.is_empty());
11294 advance_clock(Duration::from_millis(101)).await;
11295 let exhausted = node.exec_manager.check_inflight_orders();
11296 assert!(exhausted.queries.is_empty());
11297 assert!(
11298 node.exec_manager
11299 .take_submission_recovery_exhaustions()
11300 .is_empty()
11301 );
11302
11303 if recover_command {
11304 assert!(
11305 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11306 if event.client_order_id == client_order_id)
11307 );
11308 } else {
11309 assert!(exhausted.events.is_empty());
11310 }
11311 }
11312
11313 #[rstest]
11314 #[case::submission_only(None, false)]
11315 #[case::modify(Some(false), false)]
11316 #[case::cancel(Some(true), false)]
11317 #[case::batch_modify(Some(false), true)]
11318 #[case::batch_cancel(Some(true), true)]
11319 #[cfg_attr(
11320 not(all(feature = "simulation", madsim)),
11321 tokio::test(start_paused = true)
11322 )]
11323 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11324 async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
11325 #[case] cancel: Option<bool>,
11326 #[case] batch: bool,
11327 #[values(false, true)] missing_order_check: bool,
11328 #[values(
11329 SubmissionRecoveryPolicy::ResolveLocally,
11330 SubmissionRecoveryPolicy::RetainUnresolved
11331 )]
11332 policy: SubmissionRecoveryPolicy,
11333 ) {
11334 let (mut node, order, instrument) = venue_evidence_node_with_config(
11335 crate::config::LiveExecutionEngineConfig {
11336 reconciliation: true,
11337 inflight_check_threshold_ms: 100,
11338 inflight_check_retries: 2,
11339 open_check_threshold_ms: 1,
11340 open_check_open_only: false,
11341 open_check_lookback_mins: None,
11342 open_check_missing_retries: 5,
11343 single_order_query_delay_ms: 0,
11344 submission_recovery_policy: policy,
11345 ..Default::default()
11346 },
11347 OrderType::Limit,
11348 false,
11349 );
11350 let client_order_id = order.client_order_id();
11351 let client_id = ClientId::from("TEST-EVIDENCE");
11352 let account_id = AccountId::from("TEST-001");
11353 node.kernel
11354 .exec_engine
11355 .borrow_mut()
11356 .register_client(Box::new(StubExecutionClient::new(
11357 client_id,
11358 account_id,
11359 instrument.id().venue,
11360 OmsType::Netting,
11361 None,
11362 )))
11363 .unwrap();
11364 let queries = Rc::new(RefCell::new(Vec::new()));
11365 let client = OrderReportClient {
11366 client_id,
11367 report: OrderStatusReport::new(
11368 account_id,
11369 instrument.id(),
11370 Some(client_order_id),
11371 VenueOrderId::from("V-VENUE-EVIDENCE"),
11372 Some(order.order_side()),
11373 order.order_type(),
11374 order.time_in_force(),
11375 OrderStatus::Submitted,
11376 order.quantity(),
11377 Quantity::zero(3),
11378 UnixNanos::default(),
11379 UnixNanos::default(),
11380 UnixNanos::default(),
11381 None,
11382 )
11383 .with_price(order.price().unwrap()),
11384 targeted: false,
11385 queries: queries.clone(),
11386 };
11387 advance_clock(Duration::from_millis(2)).await;
11388 let events = node.exec_manager.check_open_orders(&[&client]).await;
11389 assert!(events.is_empty());
11390 assert_eq!(*queries.borrow(), vec!["bulk"]);
11391 assert_eq!(
11392 node.kernel
11393 .cache
11394 .borrow()
11395 .order(&client_order_id)
11396 .unwrap()
11397 .status(),
11398 OrderStatus::Submitted
11399 );
11400 advance_clock(Duration::from_millis(2)).await;
11401
11402 if missing_order_check {
11403 let engine = node.kernel.exec_engine.borrow();
11404 let clients = engine.get_all_clients();
11405 let check = node
11406 .exec_manager
11407 .prepare_open_order_report_check(UUID4::new(), &clients);
11408 assert_eq!(check.filtered_orders.len(), 1);
11409 let missing = node.exec_manager.reconcile_open_order_reports(
11410 &check,
11411 Vec::new(),
11412 &IndexSet::from([client_id]),
11413 &IndexSet::new(),
11414 &clients,
11415 );
11416 assert!(missing.events.is_empty());
11417 assert!(missing.targeted_queries.is_empty());
11418 assert_eq!(
11419 node.exec_manager.recon_check_retry_count(&client_order_id),
11420 1
11421 );
11422 }
11423 node.exec_manager
11424 .register_submission(order.init_event(), Some(client_id));
11425 advance_clock(Duration::from_millis(101)).await;
11426 let duplicate = node.exec_manager.check_inflight_orders();
11427 assert!(duplicate.events.is_empty());
11428 assert_eq!(
11429 duplicate.queries.len(),
11430 usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
11431 );
11432
11433 if let Some(cancel) = cancel {
11434 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11435 MessagingSwitchboard::exec_engine_execute(),
11436 venue_evidence_command(&order, cancel, batch),
11437 )));
11438 let pending = if cancel {
11439 OrderEventAny::PendingCancel(
11440 OrderPendingCancelSpec::builder()
11441 .trader_id(order.trader_id())
11442 .strategy_id(order.strategy_id())
11443 .instrument_id(order.instrument_id())
11444 .client_order_id(client_order_id)
11445 .account_id(account_id)
11446 .build(),
11447 )
11448 } else {
11449 OrderEventAny::PendingUpdate(
11450 OrderPendingUpdateSpec::builder()
11451 .trader_id(order.trader_id())
11452 .strategy_id(order.strategy_id())
11453 .instrument_id(order.instrument_id())
11454 .client_order_id(client_order_id)
11455 .account_id(account_id)
11456 .build(),
11457 )
11458 };
11459 node.process_exec_event(ExecutionEvent::Order(pending));
11460 }
11461 advance_clock(Duration::from_millis(101)).await;
11462 let first = node.exec_manager.check_inflight_orders();
11463 assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
11464
11465 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11466 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11467 if event.client_order_id == client_order_id));
11468 } else {
11469 assert!(first.events.is_empty());
11470 }
11471 node.process_reconciliation_events(&first.events);
11472 advance_clock(Duration::from_millis(101)).await;
11473 let exhausted = node.exec_manager.check_inflight_orders();
11474 assert!(exhausted.queries.is_empty());
11475
11476 if cancel.is_some() {
11477 assert!(
11478 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11479 if event.client_order_id == client_order_id)
11480 );
11481 } else {
11482 assert!(exhausted.events.is_empty());
11483 }
11484 assert!(
11485 node.exec_manager
11486 .take_submission_recovery_exhaustions()
11487 .is_empty()
11488 );
11489 }
11490
11491 #[rstest]
11492 #[case::submission_only(None, false)]
11493 #[case::modify(Some(false), false)]
11494 #[case::cancel(Some(true), false)]
11495 #[case::batch_modify(Some(false), true)]
11496 #[case::batch_cancel(Some(true), true)]
11497 #[cfg_attr(
11498 not(all(feature = "simulation", madsim)),
11499 tokio::test(start_paused = true)
11500 )]
11501 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11502 async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
11503 #[case] cancel: Option<bool>,
11504 #[case] batch: bool,
11505 #[values(false, true)] targeted: bool,
11506 #[values(
11507 SubmissionRecoveryPolicy::ResolveLocally,
11508 SubmissionRecoveryPolicy::RetainUnresolved
11509 )]
11510 policy: SubmissionRecoveryPolicy,
11511 ) {
11512 let (mut node, order, instrument) = venue_evidence_node_with_config(
11513 crate::config::LiveExecutionEngineConfig {
11514 reconciliation: true,
11515 inflight_check_threshold_ms: 100,
11516 inflight_check_retries: 2,
11517 open_check_threshold_ms: 1,
11518 open_check_open_only: false,
11519 open_check_lookback_mins: None,
11520 open_check_missing_retries: 1,
11521 single_order_query_delay_ms: 0,
11522 submission_recovery_policy: policy,
11523 ..Default::default()
11524 },
11525 OrderType::Limit,
11526 true,
11527 );
11528 let client_order_id = order.client_order_id();
11529 let client_id = ClientId::from("TEST-EVIDENCE");
11530 let account_id = AccountId::from("TEST-001");
11531 node.kernel
11532 .exec_engine
11533 .borrow_mut()
11534 .register_client(Box::new(StubExecutionClient::new(
11535 client_id,
11536 account_id,
11537 instrument.id().venue,
11538 OmsType::Netting,
11539 None,
11540 )))
11541 .unwrap();
11542 advance_clock(Duration::from_millis(101)).await;
11543 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11544 advance_clock(Duration::from_millis(10)).await;
11545
11546 if let Some(cancel) = cancel {
11547 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11548 MessagingSwitchboard::exec_engine_execute(),
11549 venue_evidence_command(&order, cancel, batch),
11550 )));
11551 let pending = if cancel {
11552 OrderEventAny::PendingCancel(
11553 OrderPendingCancelSpec::builder()
11554 .trader_id(order.trader_id())
11555 .strategy_id(order.strategy_id())
11556 .instrument_id(order.instrument_id())
11557 .client_order_id(client_order_id)
11558 .account_id(account_id)
11559 .build(),
11560 )
11561 } else {
11562 OrderEventAny::PendingUpdate(
11563 OrderPendingUpdateSpec::builder()
11564 .trader_id(order.trader_id())
11565 .strategy_id(order.strategy_id())
11566 .instrument_id(order.instrument_id())
11567 .client_order_id(client_order_id)
11568 .account_id(account_id)
11569 .build(),
11570 )
11571 };
11572 node.process_exec_event(ExecutionEvent::Order(pending));
11573 }
11574 let queries = Rc::new(RefCell::new(Vec::new()));
11575 let client = OrderReportClient {
11576 client_id,
11577 report: OrderStatusReport::new(
11578 account_id,
11579 order.instrument_id(),
11580 Some(client_order_id),
11581 VenueOrderId::from("V-VENUE-EVIDENCE"),
11582 Some(order.order_side()),
11583 order.order_type(),
11584 order.time_in_force(),
11585 OrderStatus::Submitted,
11586 order.quantity(),
11587 Quantity::zero(3),
11588 UnixNanos::default(),
11589 UnixNanos::default(),
11590 UnixNanos::default(),
11591 None,
11592 )
11593 .with_price(order.price().unwrap()),
11594 targeted,
11595 queries: queries.clone(),
11596 };
11597 advance_clock(Duration::from_millis(40)).await;
11598 let events = node.exec_manager.check_open_orders(&[&client]).await;
11599 assert!(events.is_empty());
11600 assert_eq!(
11601 *queries.borrow(),
11602 if targeted {
11603 vec!["bulk", "targeted"]
11604 } else {
11605 vec!["bulk"]
11606 }
11607 );
11608 node.process_reconciliation_events(&events);
11609 advance_clock(Duration::from_millis(61)).await;
11610 let first = node.exec_manager.check_inflight_orders();
11611 let recover_command =
11612 cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
11613 assert_eq!(first.queries.len(), usize::from(recover_command));
11614
11615 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11616 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11617 if event.client_order_id == client_order_id));
11618 } else {
11619 assert!(first.events.is_empty());
11620 }
11621 node.process_reconciliation_events(&first.events);
11622 advance_clock(Duration::from_millis(101)).await;
11623 let exhausted = node.exec_manager.check_inflight_orders();
11624 assert!(exhausted.queries.is_empty());
11625
11626 if recover_command {
11627 assert!(
11628 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11629 if event.client_order_id == client_order_id)
11630 );
11631 } else {
11632 assert!(exhausted.events.is_empty());
11633 }
11634 assert!(
11635 node.exec_manager
11636 .take_submission_recovery_exhaustions()
11637 .is_empty()
11638 );
11639 }
11640
11641 #[rstest]
11642 #[case::local_update_before_pending("updated")]
11643 #[case::modify_rejected_after_pending("modify_rejected")]
11644 #[case::cancel_rejected_after_pending("cancel_rejected")]
11645 #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
11646 #[cfg_attr(
11647 not(all(feature = "simulation", madsim)),
11648 tokio::test(start_paused = true)
11649 )]
11650 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11651 async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
11652 #[case] completion: &str,
11653 #[values(
11654 SubmissionRecoveryPolicy::ResolveLocally,
11655 SubmissionRecoveryPolicy::RetainUnresolved
11656 )]
11657 policy: SubmissionRecoveryPolicy,
11658 ) {
11659 use nautilus_model::events::order::spec::{
11660 OrderCancelRejectedSpec, OrderModifyRejectedSpec,
11661 };
11662
11663 let (mut node, order, instrument) = venue_evidence_node(policy);
11664 let client_order_id = order.client_order_id();
11665 let client_id = ClientId::from("TEST-EVIDENCE");
11666 let account_id = AccountId::from("TEST-001");
11667 let cancel = completion == "cancel_rejected";
11668 let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
11669 node.kernel
11670 .exec_engine
11671 .borrow_mut()
11672 .register_client(Box::new(StubExecutionClient::new(
11673 client_id,
11674 account_id,
11675 instrument.id().venue,
11676 OmsType::Netting,
11677 None,
11678 )))
11679 .unwrap();
11680 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11681 MessagingSwitchboard::exec_engine_execute(),
11682 venue_evidence_command(&order, cancel, false),
11683 )));
11684 advance_clock(Duration::from_millis(10)).await;
11685
11686 if completion != "updated" {
11687 let pending = if cancel {
11688 OrderEventAny::PendingCancel(
11689 OrderPendingCancelSpec::builder()
11690 .trader_id(order.trader_id())
11691 .strategy_id(order.strategy_id())
11692 .instrument_id(order.instrument_id())
11693 .client_order_id(client_order_id)
11694 .account_id(account_id)
11695 .build(),
11696 )
11697 } else {
11698 OrderEventAny::PendingUpdate(
11699 OrderPendingUpdateSpec::builder()
11700 .trader_id(order.trader_id())
11701 .strategy_id(order.strategy_id())
11702 .instrument_id(order.instrument_id())
11703 .client_order_id(client_order_id)
11704 .account_id(account_id)
11705 .build(),
11706 )
11707 };
11708 node.process_exec_event(ExecutionEvent::Order(pending));
11709 }
11710
11711 if later_cancel_pending {
11712 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11714 OrderPendingCancelSpec::builder()
11715 .trader_id(order.trader_id())
11716 .strategy_id(order.strategy_id())
11717 .instrument_id(order.instrument_id())
11718 .client_order_id(client_order_id)
11719 .account_id(account_id)
11720 .build(),
11721 )));
11722 }
11723 advance_clock(Duration::from_millis(10)).await;
11724 let completed = match completion {
11725 "updated" => OrderEventAny::Updated(
11726 OrderUpdatedSpec::builder()
11727 .trader_id(order.trader_id())
11728 .strategy_id(order.strategy_id())
11729 .instrument_id(order.instrument_id())
11730 .client_order_id(client_order_id)
11731 .account_id(account_id)
11732 .quantity(order.quantity())
11733 .price(Price::from("101.00"))
11734 .build(),
11735 ),
11736 "modify_rejected" | "modify_rejected_after_cancel_pending" => {
11737 OrderEventAny::ModifyRejected(
11738 OrderModifyRejectedSpec::builder()
11739 .trader_id(order.trader_id())
11740 .strategy_id(order.strategy_id())
11741 .instrument_id(order.instrument_id())
11742 .client_order_id(client_order_id)
11743 .account_id(account_id)
11744 .build(),
11745 )
11746 }
11747 "cancel_rejected" => OrderEventAny::CancelRejected(
11748 OrderCancelRejectedSpec::builder()
11749 .trader_id(order.trader_id())
11750 .strategy_id(order.strategy_id())
11751 .instrument_id(order.instrument_id())
11752 .client_order_id(client_order_id)
11753 .account_id(account_id)
11754 .build(),
11755 ),
11756 _ => unreachable!(),
11757 };
11758 node.process_exec_event(ExecutionEvent::Order(completed));
11759 assert_eq!(
11760 node.kernel
11761 .cache
11762 .borrow()
11763 .order(&client_order_id)
11764 .unwrap()
11765 .status(),
11766 if later_cancel_pending {
11767 OrderStatus::PendingCancel
11768 } else {
11769 OrderStatus::Submitted
11770 }
11771 );
11772
11773 advance_clock(Duration::from_millis(10)).await;
11775
11776 if !later_cancel_pending {
11777 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11778 OrderPendingUpdateSpec::builder()
11779 .trader_id(order.trader_id())
11780 .strategy_id(order.strategy_id())
11781 .instrument_id(order.instrument_id())
11782 .client_order_id(client_order_id)
11783 .account_id(account_id)
11784 .build(),
11785 )));
11786 }
11787 advance_clock(Duration::from_millis(10)).await;
11788 let cached = node
11789 .kernel
11790 .cache
11791 .borrow()
11792 .order_owned(&client_order_id)
11793 .unwrap();
11794 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11795 .last_qty(Quantity::from("0.400"))
11796 .last_px(order.price().unwrap())
11797 .commission(Money::from("0 USDT"))
11798 .without_position_id()
11799 .build()
11800 else {
11801 unreachable!()
11802 };
11803 fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11804 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11805
11806 for _ in 0..3 {
11807 advance_clock(Duration::from_millis(101)).await;
11808 let result = node.exec_manager.check_inflight_orders();
11809 assert!(result.queries.is_empty());
11810 assert!(result.events.is_empty());
11811 assert!(
11812 node.exec_manager
11813 .take_submission_recovery_exhaustions()
11814 .is_empty()
11815 );
11816 }
11817 assert_eq!(
11818 node.kernel
11819 .cache
11820 .borrow()
11821 .order(&client_order_id)
11822 .unwrap()
11823 .status(),
11824 if later_cancel_pending {
11825 OrderStatus::PendingCancel
11826 } else {
11827 OrderStatus::PendingUpdate
11828 }
11829 );
11830 }
11831
11832 #[rstest]
11833 #[case::wrong_account("account")]
11834 #[case::wrong_instrument("instrument")]
11835 #[case::conflicting_report_venue("report_venue")]
11836 #[case::unapplied_venue_update("update_venue")]
11837 #[case::local_update("local_update")]
11838 #[cfg_attr(
11839 not(all(feature = "simulation", madsim)),
11840 tokio::test(start_paused = true)
11841 )]
11842 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11843 async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
11844 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11845 let client_order_id = order.client_order_id();
11846 let account_id = AccountId::from("TEST-001");
11847 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11848
11849 if invalid.ends_with("venue") {
11850 insert_accepted_limit_order_in_node(
11851 &node,
11852 account_id,
11853 ClientId::from("TEST-EVIDENCE"),
11854 order.instrument_id(),
11855 ClientOrderId::from("O-OTHER"),
11856 venue_order_id,
11857 );
11858 }
11859 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11860 OrderPendingCancelSpec::builder()
11861 .trader_id(order.trader_id())
11862 .strategy_id(order.strategy_id())
11863 .instrument_id(order.instrument_id())
11864 .client_order_id(client_order_id)
11865 .account_id(account_id)
11866 .build(),
11867 )));
11868 advance_clock(Duration::from_millis(101)).await;
11869 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11870
11871 if invalid == "update_venue" || invalid == "local_update" {
11872 let mut updated = OrderUpdatedSpec::builder()
11873 .trader_id(order.trader_id())
11874 .strategy_id(order.strategy_id())
11875 .instrument_id(order.instrument_id())
11876 .client_order_id(client_order_id)
11877 .account_id(account_id)
11878 .quantity(order.quantity())
11879 .price(order.price().unwrap())
11880 .build();
11881
11882 if invalid == "update_venue" {
11883 updated.venue_order_id = Some(venue_order_id);
11884 }
11885 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
11886 } else {
11887 let report = OrderStatusReport::new(
11888 if invalid == "account" {
11889 AccountId::from("OTHER-001")
11890 } else {
11891 account_id
11892 },
11893 if invalid == "instrument" {
11894 InstrumentId::from("OTHER.BINANCE")
11895 } else {
11896 order.instrument_id()
11897 },
11898 Some(client_order_id),
11899 venue_order_id,
11900 Some(order.order_side()),
11901 order.order_type(),
11902 order.time_in_force(),
11903 OrderStatus::Submitted,
11904 order.quantity(),
11905 Quantity::zero(3),
11906 UnixNanos::default(),
11907 UnixNanos::default(),
11908 UnixNanos::default(),
11909 None,
11910 )
11911 .with_price(order.price().unwrap());
11912 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11913 report,
11914 ))));
11915 }
11916 assert!(
11917 node.kernel
11918 .cache
11919 .borrow()
11920 .order(&client_order_id)
11921 .unwrap()
11922 .venue_order_id()
11923 .is_none()
11924 );
11925 advance_clock(Duration::from_millis(101)).await;
11926 let result = node.exec_manager.check_inflight_orders();
11927 assert!(result.events.is_empty());
11928 assert!(result.queries.is_empty());
11929 let cached = node
11930 .kernel
11931 .cache
11932 .borrow()
11933 .order_owned(&client_order_id)
11934 .unwrap();
11935 assert_eq!(
11936 cached.status(),
11937 if invalid == "local_update" {
11938 OrderStatus::Submitted
11939 } else {
11940 OrderStatus::PendingCancel
11941 }
11942 );
11943 let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
11944 assert_eq!(diagnostics.len(), 1);
11945 assert_eq!(diagnostics[0].client_order_id, client_order_id);
11946 }
11947
11948 fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
11949 use nautilus_common::messages::execution::{
11950 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
11951 };
11952
11953 let client_id = ClientId::from("TEST-EVIDENCE");
11954 let client_order_id = order.client_order_id();
11955 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11956
11957 if cancel {
11958 let cancel = CancelOrder::new(
11959 order.trader_id(),
11960 Some(client_id),
11961 order.strategy_id(),
11962 order.instrument_id(),
11963 client_order_id,
11964 Some(venue_order_id),
11965 UUID4::new(),
11966 UnixNanos::default(),
11967 None,
11968 None,
11969 );
11970
11971 if batch {
11972 TradingCommand::CancelOrders(BatchCancelOrders::new(
11973 order.trader_id(),
11974 Some(client_id),
11975 order.strategy_id(),
11976 order.instrument_id(),
11977 vec![cancel],
11978 UUID4::new(),
11979 UnixNanos::default(),
11980 None,
11981 None,
11982 ))
11983 } else {
11984 TradingCommand::CancelOrder(cancel)
11985 }
11986 } else {
11987 let modify = ModifyOrder::new(
11988 order.trader_id(),
11989 Some(client_id),
11990 order.strategy_id(),
11991 order.instrument_id(),
11992 client_order_id,
11993 Some(venue_order_id),
11994 None,
11995 Some(Price::from("101.00")),
11996 None,
11997 UUID4::new(),
11998 UnixNanos::default(),
11999 None,
12000 None,
12001 );
12002
12003 if batch {
12004 TradingCommand::ModifyOrders(BatchModifyOrders::new(
12005 order.trader_id(),
12006 Some(client_id),
12007 order.strategy_id(),
12008 order.instrument_id(),
12009 vec![modify],
12010 UUID4::new(),
12011 UnixNanos::default(),
12012 None,
12013 None,
12014 ))
12015 } else {
12016 TradingCommand::ModifyOrder(modify)
12017 }
12018 }
12019 }
12020
12021 fn venue_evidence_node(
12022 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12023 ) -> (LiveNode, OrderAny, InstrumentAny) {
12024 venue_evidence_node_with_order_type(policy, OrderType::Limit)
12025 }
12026
12027 fn venue_evidence_node_with_order_type(
12028 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12029 order_type: OrderType,
12030 ) -> (LiveNode, OrderAny, InstrumentAny) {
12031 venue_evidence_node_with_config(
12032 crate::config::LiveExecutionEngineConfig {
12033 reconciliation: true,
12034 inflight_check_threshold_ms: 100,
12035 inflight_check_retries: 2,
12036 open_check_threshold_ms: 5_000,
12037 single_order_query_delay_ms: 0,
12038 submission_recovery_policy: policy,
12039 ..Default::default()
12040 },
12041 order_type,
12042 true,
12043 )
12044 }
12045
12046 fn venue_evidence_node_with_config(
12047 exec_engine: crate::config::LiveExecutionEngineConfig,
12048 order_type: OrderType,
12049 register_submission: bool,
12050 ) -> (LiveNode, OrderAny, InstrumentAny) {
12051 let config = LiveNodeConfig {
12052 exec_engine,
12053 ..Default::default()
12054 };
12055 let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12056 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12057 let account_id = AccountId::from("TEST-001");
12058 let client_id = ClientId::from("TEST-EVIDENCE");
12059 let account = AccountAny::Margin(MarginAccount::new(
12060 AccountState::new(
12061 account_id,
12062 AccountType::Margin,
12063 vec![AccountBalance::new(
12064 Money::from("1000000 USDT"),
12065 Money::from("0 USDT"),
12066 Money::from("1000000 USDT"),
12067 )],
12068 Vec::new(),
12069 true,
12070 UUID4::new(),
12071 UnixNanos::default(),
12072 UnixNanos::default(),
12073 Some(Currency::USDT()),
12074 ),
12075 true,
12076 ));
12077 node.kernel.cache.borrow_mut().add_account(account).unwrap();
12078 node.kernel
12079 .cache
12080 .borrow_mut()
12081 .add_instrument(instrument.clone())
12082 .unwrap();
12083 let mut builder = OrderTestBuilder::new(order_type);
12084 builder
12085 .trader_id(node.trader_id())
12086 .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
12087 .instrument_id(instrument.id())
12088 .quantity(Quantity::from("1.000"))
12089 .price(Price::from("100.00"));
12090
12091 if order_type == OrderType::StopLimit {
12092 builder.trigger_price(Price::from("110.00"));
12093 }
12094 let order = builder.build();
12095 let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
12096 &order,
12097 node.trader_id(),
12098 Some(client_id),
12099 None,
12100 UUID4::new(),
12101 UnixNanos::default(),
12102 ));
12103
12104 if register_submission {
12105 node.observe_exec_command_before_dispatch(&command);
12106 }
12107 node.kernel
12108 .cache
12109 .borrow_mut()
12110 .add_order(order.clone(), None, Some(client_id), false)
12111 .unwrap();
12112 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
12113 &order, account_id,
12114 )));
12115 (node, order, instrument)
12116 }
12117
12118 #[rstest]
12119 #[case::partial("0.400", OrderStatus::PartiallyFilled)]
12120 #[case::full("1.000", OrderStatus::Filled)]
12121 #[tokio::test(start_paused = true)]
12122 async fn test_retained_submission_resolves_late_fill(
12123 #[case] quantity: &str,
12124 #[case] expected_status: OrderStatus,
12125 ) {
12126 let (mut node, order, instrument) =
12127 venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12128 let client_order_id = order.client_order_id();
12129
12130 for _ in 0..2 {
12131 advance_clock(Duration::from_millis(101)).await;
12132 assert!(node.exec_manager.check_inflight_orders().events.is_empty());
12133 }
12134
12135 assert_eq!(
12136 node.exec_manager
12137 .take_submission_recovery_exhaustions()
12138 .len(),
12139 1
12140 );
12141 let fill = TestOrderEventStubs::filled(
12142 &order,
12143 &instrument,
12144 Some(TradeId::from("T-LATE-RETAINED")),
12145 None,
12146 Some(Price::from("100.00")),
12147 Some(Quantity::from(quantity)),
12148 Some(LiquiditySide::Taker),
12149 None,
12150 None,
12151 Some(AccountId::from("TEST-001")),
12152 );
12153 node.process_exec_event(ExecutionEvent::Order(fill));
12154 advance_clock(Duration::from_millis(101)).await;
12155 let result = node.exec_manager.check_inflight_orders();
12156 let cache = node.kernel.cache.borrow();
12157 let cached = cache.order_ref(&client_order_id).unwrap();
12158 assert_eq!(cached.status(), expected_status);
12159 assert_eq!(cached.filled_qty(), Quantity::from(quantity));
12160 assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
12161 assert!(result.events.is_empty());
12162 assert!(result.queries.is_empty());
12163 assert!(
12164 node.exec_manager
12165 .take_submission_recovery_exhaustions()
12166 .is_empty()
12167 );
12168 }
12169
12170 #[rstest]
12171 #[case::submitted_report("order")]
12172 #[case::submitted_report_with_fills("order_with_fills")]
12173 #[case::venue_update("updated")]
12174 #[cfg_attr(
12175 not(all(feature = "simulation", madsim)),
12176 tokio::test(start_paused = true)
12177 )]
12178 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12179 async fn test_venue_evidence_stops_submission_timeout(
12180 #[case] evidence: &str,
12181 #[values(
12182 OrderStatus::Submitted,
12183 OrderStatus::PendingCancel,
12184 OrderStatus::PendingUpdate
12185 )]
12186 status: OrderStatus,
12187 #[values(
12188 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12189 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12190 )]
12191 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12192 ) {
12193 let (mut node, order, _) = venue_evidence_node(policy);
12194 let client_order_id = order.client_order_id();
12195 let account_id = AccountId::from("TEST-001");
12196 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12197 let pending = match status {
12198 OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
12199 OrderPendingCancelSpec::builder()
12200 .trader_id(order.trader_id())
12201 .strategy_id(order.strategy_id())
12202 .instrument_id(order.instrument_id())
12203 .client_order_id(client_order_id)
12204 .account_id(account_id)
12205 .build(),
12206 )),
12207 OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
12208 OrderPendingUpdateSpec::builder()
12209 .trader_id(order.trader_id())
12210 .strategy_id(order.strategy_id())
12211 .instrument_id(order.instrument_id())
12212 .client_order_id(client_order_id)
12213 .account_id(account_id)
12214 .build(),
12215 )),
12216 _ => None,
12217 };
12218
12219 if let Some(pending) = pending {
12220 node.process_exec_event(ExecutionEvent::Order(pending));
12221 }
12222 advance_clock(Duration::from_millis(101)).await;
12223 let first = node.exec_manager.check_inflight_orders();
12224 assert_eq!(first.queries.len(), 1);
12225 assert!(first.events.is_empty());
12226 advance_clock(Duration::from_millis(94)).await;
12227
12228 let event = if evidence == "updated" {
12229 ExecutionEvent::Order(OrderEventAny::Updated(
12230 OrderUpdatedSpec::builder()
12231 .trader_id(order.trader_id())
12232 .strategy_id(order.strategy_id())
12233 .instrument_id(order.instrument_id())
12234 .client_order_id(client_order_id)
12235 .account_id(account_id)
12236 .venue_order_id(venue_order_id)
12237 .quantity(order.quantity())
12238 .price(Price::from("101.00"))
12239 .build(),
12240 ))
12241 } else {
12242 let report = OrderStatusReport::new(
12243 account_id,
12244 order.instrument_id(),
12245 Some(client_order_id),
12246 venue_order_id,
12247 Some(order.order_side()),
12248 order.order_type(),
12249 order.time_in_force(),
12250 OrderStatus::Submitted,
12251 order.quantity(),
12252 Quantity::zero(3),
12253 UnixNanos::default(),
12254 UnixNanos::default(),
12255 UnixNanos::default(),
12256 None,
12257 )
12258 .with_price(Price::from("100.00"));
12259 ExecutionEvent::Report(if evidence == "order" {
12260 ExecutionReport::Order(Box::new(report))
12261 } else {
12262 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
12263 })
12264 };
12265 node.process_exec_event(event);
12266 let expected_status = if evidence == "updated" {
12267 OrderStatus::Submitted
12268 } else if status == OrderStatus::Submitted {
12269 OrderStatus::Accepted
12270 } else {
12271 status
12272 };
12273 assert_eq!(
12274 node.kernel
12275 .cache
12276 .borrow()
12277 .order(&client_order_id)
12278 .unwrap()
12279 .status(),
12280 expected_status,
12281 );
12282
12283 for _ in 0..3 {
12284 advance_clock(Duration::from_millis(101)).await;
12285 let result = node.exec_manager.check_inflight_orders();
12286 assert!(result.queries.is_empty());
12287 assert!(result.events.is_empty());
12288 }
12289 assert_eq!(
12290 node.exec_manager.recon_check_retry_count(&client_order_id),
12291 0
12292 );
12293 assert!(node.exec_manager.check_open_order_queries().is_empty());
12294 advance_clock(Duration::from_secs(5)).await;
12295 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12296 }
12297
12298 #[rstest]
12299 #[case::fill("fill", false)]
12300 #[case::terminal_order("order", true)]
12301 #[case::terminal_order_with_fills("order_with_fills", true)]
12302 #[case::working_order_for_closed_cache("order", false)]
12303 #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
12304 #[cfg_attr(
12305 not(all(feature = "simulation", madsim)),
12306 tokio::test(start_paused = true)
12307 )]
12308 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12309 async fn test_report_cleanup_preserves_original_settling_predicate(
12310 #[case] report_kind: &str,
12311 #[case] terminal_report: bool,
12312 #[values(false, true)] explicit_client_id: bool,
12313 #[values(
12314 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12315 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12316 )]
12317 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12318 ) {
12319 let (mut node, order, instrument) = venue_evidence_node(policy);
12320 let client_order_id = order.client_order_id();
12321 let account_id = AccountId::from("TEST-001");
12322 let client_id = ClientId::from("TEST-EVIDENCE");
12323 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12324 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
12325 &order,
12326 account_id,
12327 venue_order_id,
12328 )));
12329 let accepted = node
12330 .kernel
12331 .cache
12332 .borrow()
12333 .order_owned(&client_order_id)
12334 .unwrap();
12335 let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
12336 .last_qty(accepted.quantity())
12337 .last_px(Price::from("100.00"))
12338 .commission(Money::from("0 USDT"))
12339 .without_position_id()
12340 .build()
12341 else {
12342 unreachable!()
12343 };
12344 let mut fill_report = FillReport::new(
12345 account_id,
12346 instrument.id(),
12347 venue_order_id,
12348 fill.trade_id,
12349 fill.order_side,
12350 fill.last_qty,
12351 fill.last_px,
12352 fill.commission.unwrap(),
12353 fill.liquidity_side,
12354 Some(client_order_id),
12355 None,
12356 fill.ts_event,
12357 fill.ts_init,
12358 None,
12359 );
12360
12361 if !explicit_client_id {
12362 fill_report.client_order_id = None;
12363 }
12364 let event = if report_kind == "fill" {
12365 ExecutionReport::Fill(Box::new(fill_report))
12366 } else {
12367 if !terminal_report {
12368 node.kernel
12369 .cache
12370 .borrow_mut()
12371 .update_order(&OrderEventAny::Filled(fill))
12372 .unwrap();
12373 }
12374 let report = OrderStatusReport::new(
12375 account_id,
12376 instrument.id(),
12377 explicit_client_id.then_some(client_order_id),
12378 venue_order_id,
12379 Some(order.order_side()),
12380 order.order_type(),
12381 order.time_in_force(),
12382 if terminal_report {
12383 OrderStatus::Filled
12384 } else {
12385 OrderStatus::Accepted
12386 },
12387 order.quantity(),
12388 order.quantity(),
12389 UnixNanos::default(),
12390 UnixNanos::default(),
12391 UnixNanos::default(),
12392 None,
12393 )
12394 .with_price(Price::from("100.00"))
12395 .with_avg_px(dec!(100.00));
12396
12397 if report_kind == "order" {
12398 ExecutionReport::Order(Box::new(report))
12399 } else {
12400 ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
12401 }
12402 };
12403 node.process_exec_event(ExecutionEvent::Report(event));
12404 assert_eq!(
12405 node.kernel
12406 .cache
12407 .borrow()
12408 .order(&client_order_id)
12409 .unwrap()
12410 .status(),
12411 OrderStatus::Filled
12412 );
12413
12414 node.kernel
12416 .cache
12417 .borrow_mut()
12418 .add_order(order.clone(), None, Some(client_id), true)
12419 .unwrap();
12420 node.kernel
12421 .cache
12422 .borrow_mut()
12423 .update_order(&TestOrderEventStubs::submitted(&order, account_id))
12424 .unwrap();
12425 node.kernel
12426 .cache
12427 .borrow_mut()
12428 .update_order(&TestOrderEventStubs::accepted(
12429 &order,
12430 account_id,
12431 venue_order_id,
12432 ))
12433 .unwrap();
12434 let should_clear = terminal_report && explicit_client_id;
12435 assert_eq!(
12436 node.exec_manager.check_open_order_queries().len(),
12437 usize::from(should_clear)
12438 );
12439
12440 if !should_clear {
12441 advance_clock(Duration::from_secs(5)).await;
12442 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12443 }
12444 }
12445
12446 #[rstest]
12447 #[case(stub_order_event, 1)]
12448 #[case(stub_submitted_batch_event, 2)]
12449 #[case(stub_accepted_batch_event, 2)]
12450 #[case(stub_canceled_batch_event, 2)]
12451 fn test_pending_events_preserve_roots_when_splitting_batches(
12452 #[values(false, true)] rooted: bool,
12453 #[case] make_event: fn() -> ExecutionEvent,
12454 #[case] orders: usize,
12455 ) {
12456 let runner = AsyncRunner::new();
12457 runner.bind_senders();
12458 let mut channels = runner.take_channels();
12459
12460 let send = move || {
12461 get_data_event_sender().send(stub_data_event()).unwrap();
12462 get_exec_event_sender().send(stub_exec_event()).unwrap();
12463 get_exec_event_sender().send(make_event()).unwrap();
12464 };
12465
12466 if rooted {
12467 msgbus::register_trading_command_endpoint(
12468 MessagingSwitchboard::risk_engine_execute(),
12469 TypedIntoHandler::from(move |_| send()),
12470 );
12471 TradingCommandSender::execute(
12472 &SyncTradingCommandSender,
12473 TradingCommandMessage::new(
12474 MessagingSwitchboard::risk_engine_execute(),
12475 TradingCommand::QueryAccount(QueryAccount::new(
12476 "TRADER-001".into(),
12477 None,
12478 "SIM-001".into(),
12479 UUID4::new(),
12480 37.into(),
12481 None,
12482 None,
12483 )),
12484 ),
12485 );
12486 nautilus_common::runner::drain_trading_cmd_queue();
12487 } else {
12488 send();
12489 }
12490
12491 let mut pending = PendingEvents::default();
12492 pending
12493 .data_evts
12494 .push(channels.data_evt_rx.try_recv().unwrap());
12495 while let Ok(event) = channels.exec_evt_rx.try_recv() {
12496 pending.push_exec_event(event);
12497 }
12498
12499 assert_eq!(pending.data_evts.len(), 1);
12500 assert_eq!(pending.exec_reports.len(), 1);
12501 assert_eq!(pending.order_evts.len(), orders);
12502 assert_eq!(pending.data_evts[0].is_rooted(), rooted);
12503 assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
12504 assert!(
12505 pending
12506 .order_evts
12507 .iter()
12508 .all(|event| event.is_rooted() == rooted)
12509 );
12510
12511 let results = std::thread::spawn(move || {
12512 let mut results = Vec::new();
12513
12514 for event in pending.data_evts {
12515 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12516 || event.dispatch(drop),
12517 )));
12518 }
12519
12520 for event in pending.exec_reports {
12521 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12522 || event.dispatch(drop),
12523 )));
12524 }
12525
12526 for event in pending.order_evts {
12527 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12528 || event.dispatch(drop),
12529 )));
12530 }
12531
12532 results
12533 })
12534 .join()
12535 .unwrap();
12536
12537 assert_eq!(results.len(), orders + 2);
12538
12539 for result in results {
12540 if rooted {
12541 let error = result.unwrap_err();
12542 assert!(
12543 error
12544 .downcast_ref::<String>()
12545 .unwrap()
12546 .contains("command context dispatched outside its owner thread")
12547 );
12548 } else {
12549 result.unwrap();
12550 }
12551 }
12552 }
12553
12554 fn stub_accepted_batch_event() -> ExecutionEvent {
12555 ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
12556 OrderAcceptedSpec::builder()
12557 .client_order_id(ClientOrderId::from("O-017"))
12558 .build(),
12559 OrderAcceptedSpec::builder()
12560 .client_order_id(ClientOrderId::from("O-023"))
12561 .build(),
12562 ]))
12563 }
12564
12565 #[derive(Debug)]
12566 struct CapturedEgressMessage {
12567 topic: String,
12568 payload: Bytes,
12569 }
12570
12571 type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
12572 type SharedClosed = Rc<Cell<bool>>;
12573
12574 #[derive(Debug)]
12575 struct CapturingExternalIngress {
12576 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12577 closed: SharedClosed,
12578 }
12579
12580 impl CapturingExternalIngress {
12581 fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
12582 Self {
12583 rx: Some(rx),
12584 closed,
12585 }
12586 }
12587 }
12588
12589 impl MessageBusExternalIngress for CapturingExternalIngress {
12590 fn is_closed(&self) -> bool {
12591 self.closed.get()
12592 }
12593
12594 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12595 self.rx
12596 .take()
12597 .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
12598 }
12599
12600 fn close(&mut self) {
12601 self.closed.set(true);
12602 }
12603 }
12604
12605 #[derive(Debug)]
12606 struct FailingExternalIngress {
12607 closed: SharedClosed,
12608 }
12609
12610 impl FailingExternalIngress {
12611 fn new(closed: SharedClosed) -> Self {
12612 Self { closed }
12613 }
12614 }
12615
12616 impl MessageBusExternalIngress for FailingExternalIngress {
12617 fn is_closed(&self) -> bool {
12618 self.closed.get()
12619 }
12620
12621 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12622 anyhow::bail!("external ingress receiver unavailable")
12623 }
12624
12625 fn close(&mut self) {
12626 self.closed.set(true);
12627 }
12628 }
12629
12630 struct CapturingExternalEgress {
12631 publications: CapturedEgressMessages,
12632 closed: SharedClosed,
12633 }
12634
12635 impl CapturingExternalEgress {
12636 fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
12637 let publications = Rc::new(RefCell::new(Vec::new()));
12638 let closed = Rc::new(Cell::new(false));
12639 (
12640 Self {
12641 publications: publications.clone(),
12642 closed: closed.clone(),
12643 },
12644 publications,
12645 closed,
12646 )
12647 }
12648 }
12649
12650 impl MessageBusExternalEgress for CapturingExternalEgress {
12651 fn is_closed(&self) -> bool {
12652 self.closed.get()
12653 }
12654
12655 fn publish(&self, message: BusMessage) {
12656 self.publications.borrow_mut().push(CapturedEgressMessage {
12657 topic: message.topic.to_string(),
12658 payload: message.payload,
12659 });
12660 }
12661
12662 fn close(&mut self) {
12663 self.closed.set(true);
12664 }
12665 }
12666
12667 struct CapturingBackingFactory {
12668 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12669 closed: Arc<AtomicBool>,
12670 rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
12671 }
12672
12673 impl CapturingBackingFactory {
12674 fn new(
12675 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12676 closed: Arc<AtomicBool>,
12677 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12678 ) -> Self {
12679 Self {
12680 publications,
12681 closed,
12682 rx: Mutex::new(rx),
12683 }
12684 }
12685 }
12686
12687 impl Debug for CapturingBackingFactory {
12688 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
12689 f.debug_struct(stringify!(CapturingBackingFactory))
12690 .finish_non_exhaustive()
12691 }
12692 }
12693
12694 impl MessageBusBackingFactory for CapturingBackingFactory {
12695 fn create(
12696 &self,
12697 _trader_id: TraderId,
12698 _instance_id: UUID4,
12699 _config: MessageBusConfig,
12700 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
12701 let rx = self.rx.lock().take();
12702 Ok(Box::new(CapturingBacking {
12703 publications: self.publications.clone(),
12704 closed: self.closed.clone(),
12705 rx,
12706 }))
12707 }
12708 }
12709
12710 struct CapturingBacking {
12711 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12712 closed: Arc<AtomicBool>,
12713 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12714 }
12715
12716 impl MessageBusBacking for CapturingBacking {
12717 fn is_closed(&self) -> bool {
12718 self.closed.load(Ordering::Relaxed)
12719 }
12720
12721 fn publish(&self, message: BusMessage) {
12722 self.publications.lock().push(CapturedEgressMessage {
12723 topic: message.topic.to_string(),
12724 payload: message.payload,
12725 });
12726 }
12727
12728 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12729 self.rx
12730 .take()
12731 .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
12732 }
12733
12734 fn close(&mut self) {
12735 self.closed.store(true, Ordering::Relaxed);
12736 }
12737 }
12738}