1use std::{
29 cell::{Ref, RefCell},
30 fmt::Debug,
31 rc::Rc,
32 str::FromStr,
33 sync::LazyLock,
34 time::Duration,
35};
36
37use indexmap::{IndexMap, IndexSet};
38use nautilus_common::{
39 cache::Cache,
40 clients::{DEFAULT_POSITION_RECONCILIATION_TOLERANCE, ExecutionClient},
41 clock::Clock,
42 config::ConfigResult,
43 enums::{LogColor, LogLevel},
44 live::dst,
45 log_info,
46 messages::{
47 ExecutionReport,
48 execution::{
49 QueryOrder, TradingCommand,
50 report::{
51 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
52 GeneratePositionStatusReports,
53 },
54 },
55 },
56 msgbus::{self, MessagingSwitchboard, switchboard},
57};
58use nautilus_core::{DurationNanos, UUID4, UnixNanos, datetime::mins_to_secs};
59use nautilus_execution::{
60 engine::ExecutionEngine,
61 reconciliation::{
62 calculate_reconciliation_price, create_inferred_fill_for_qty,
63 create_position_reconciliation_venue_order_id, create_reconciliation_rejected,
64 create_reconciliation_triggered, generate_external_order_status_events_with_commission,
65 generate_reconciliation_order_pre_fill_events,
66 generate_reconciliation_order_snapshot_events_with_commission,
67 incremental_inferred_fill_price_and_liquidity, inferred_fill_price_and_liquidity,
68 process_mass_status_for_reconciliation,
69 process_mass_status_for_reconciliation_without_synthetic_reports,
70 reconcile_order_report_with_commission,
71 },
72};
73use nautilus_model::{
74 enums::{OmsType, OrderSide, OrderStatus, OrderType, TimeInForce},
75 events::{OrderCanceled, OrderEventAny, OrderFilled, OrderInitialized},
76 identifiers::{
77 AccountId, ClientId, ClientOrderId, InstrumentId, PositionId, StrategyId, TradeId,
78 VenueOrderId,
79 },
80 instruments::{Instrument, InstrumentAny},
81 orders::{Order, OrderAny, TRIGGERABLE_ORDER_TYPES},
82 position::{Position, PositionReplayEvent},
83 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
84 types::{Money, Price, Quantity},
85};
86use rust_decimal::Decimal;
87use ustr::Ustr;
88
89pub(crate) use super::reconciliation::{
90 OpenOrderReconciliationResult, OpenOrderReportCheck, SourcedOrderStatusReport,
91 TargetedOrderQuery, TargetedOrderReportResult, request_targeted_order_reports,
92 resolve_position_report_client_coverage,
93};
94pub use super::{
95 config::ExecutionManagerConfig,
96 reconciliation::{
97 ExternalOrderMetadata, InflightCheckResult, InstrumentAccountKey, PositionFillReportPlan,
98 PositionFillReportPreparation, PositionFillReportQuery, PositionReportCheck,
99 ReconciliationResult, ReportClientCoverage,
100 },
101};
102use super::{
103 recency::RecencyMap,
104 reconciliation::{
105 AccountInstrumentKey, AccountInstrumentStrategyKey, FillKey, HistoricalFillGroup,
106 InflightCheck, PositionQuantityComparison, PositionReconciliationState,
107 PositionReportShape, ReconciliationFillQueue, RetainedFillState,
108 create_cross_zero_leg_report, create_orphan_fill_order_report, has_active_inferred_fill,
109 is_exact_order_match, position_avg_px, position_qty_aggregates,
110 resolve_inferred_fill_commission, should_project_fill, terminal_report_has_missing_fills,
111 },
112};
113
114static TAG_VENUE: LazyLock<Ustr> = LazyLock::new(|| Ustr::from("VENUE"));
116
117static TAG_RECONCILIATION: LazyLock<Ustr> = LazyLock::new(|| Ustr::from("RECONCILIATION"));
119
120#[derive(Clone)]
134pub struct ExecutionManager {
135 clock: Rc<RefCell<dyn Clock>>,
136 cache: Rc<RefCell<Cache>>,
137 config: ExecutionManagerConfig,
138
139 order_activity: RecencyMap<ClientOrderId>,
140 order_inflight_checks: IndexMap<ClientOrderId, InflightCheck>,
141 order_query_recency: RecencyMap<ClientOrderId>,
142 order_query_pending: IndexSet<ClientOrderId>,
143 order_recon_retries: IndexMap<ClientOrderId, u32>,
144 order_coverage_unresolved: IndexSet<ClientOrderId>,
145 order_coverage_warnings: IndexSet<ClientOrderId>,
146 order_lookback_warnings: IndexSet<ClientOrderId>,
147
148 fills_processed: RecencyMap<FillKey>,
149 fills_recent: RecencyMap<FillKey>,
150
151 position_activity: RecencyMap<InstrumentAccountKey>,
152 position_activity_revisions: IndexMap<InstrumentAccountKey, u64>,
153 position_recon: IndexMap<InstrumentAccountKey, PositionReconciliationState>,
154 position_recon_tolerances: IndexMap<AccountId, Decimal>,
155}
156
157impl Debug for ExecutionManager {
158 #[rustfmt::skip]
159 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
160 f.debug_struct(stringify!(ExecutionManager))
161 .field("clock", &self.clock)
162 .field("cache", &self.cache)
163 .field("config", &self.config)
164 .field("order_activity", &self.order_activity)
165 .field("order_inflight_checks", &self.order_inflight_checks)
166 .field("order_query_recency", &self.order_query_recency)
167 .field("order_query_pending", &self.order_query_pending)
168 .field("order_recon_retries", &self.order_recon_retries)
169 .field("order_coverage_unresolved", &self.order_coverage_unresolved)
170 .field("order_coverage_warnings", &self.order_coverage_warnings)
171 .field("order_lookback_warnings", &self.order_lookback_warnings)
172 .field("fills_processed", &self.fills_processed)
173 .field("fills_recent", &self.fills_recent)
174 .field("position_activity", &self.position_activity)
175 .field("position_activity_revisions", &self.position_activity_revisions)
176 .field("position_recon", &self.position_recon)
177 .field("position_recon_tolerances", &self.position_recon_tolerances)
178 .finish()
179 }
180}
181
182impl ExecutionManager {
183 pub fn new(
189 clock: Rc<RefCell<dyn Clock>>,
190 cache: Rc<RefCell<Cache>>,
191 config: ExecutionManagerConfig,
192 ) -> ConfigResult<Self> {
193 config.validate()?;
194
195 Ok(Self {
196 clock,
197 cache,
198 config,
199 order_activity: RecencyMap::default(),
200 order_inflight_checks: IndexMap::new(),
201 order_query_recency: RecencyMap::default(),
202 order_query_pending: IndexSet::new(),
203 order_recon_retries: IndexMap::new(),
204 order_coverage_unresolved: IndexSet::new(),
205 order_coverage_warnings: IndexSet::new(),
206 order_lookback_warnings: IndexSet::new(),
207 fills_processed: RecencyMap::default(),
208 fills_recent: RecencyMap::default(),
209 position_activity: RecencyMap::default(),
210 position_activity_revisions: IndexMap::new(),
211 position_recon: IndexMap::new(),
212 position_recon_tolerances: IndexMap::new(),
213 })
214 }
215
216 pub(crate) const fn config(&self) -> &ExecutionManagerConfig {
218 &self.config
219 }
220
221 pub(crate) fn timestamp_ns(&self) -> UnixNanos {
223 self.clock.borrow().timestamp_ns()
224 }
225
226 pub(crate) fn cache(&self) -> Ref<'_, Cache> {
228 self.cache.borrow()
229 }
230
231 pub fn register_inflight(&mut self, client_order_id: ClientOrderId) {
233 if self
234 .config
235 .filtered_client_order_ids
236 .contains(&client_order_id)
237 {
238 return;
239 }
240
241 self.order_inflight_checks.insert(
242 client_order_id,
243 InflightCheck {
244 submitted_at: dst::time::Instant::now(),
245 retry_count: 0,
246 last_query_at: None,
247 },
248 );
249
250 self.order_recon_retries.insert(client_order_id, 0);
251 self.order_query_recency.remove(&client_order_id);
252 self.order_activity.remove(&client_order_id);
253 }
254
255 pub fn record_local_activity(&mut self, client_order_id: ClientOrderId) {
262 self.order_activity.mark(client_order_id);
263 }
264
265 #[must_use]
268 pub fn recon_check_retry_count(&self, client_order_id: &ClientOrderId) -> u32 {
269 self.order_recon_retries
270 .get(client_order_id)
271 .copied()
272 .unwrap_or(0)
273 }
274
275 pub(crate) fn remove_targeted_order_queries(&mut self, client_order_ids: &[ClientOrderId]) {
277 for client_order_id in client_order_ids {
278 self.order_query_pending.shift_remove(client_order_id);
279 }
280 }
281
282 pub fn clear_recon_tracking(&mut self, client_order_id: &ClientOrderId, drop_last_query: bool) {
284 self.order_inflight_checks.shift_remove(client_order_id);
285 self.order_recon_retries.shift_remove(client_order_id);
286 self.order_coverage_warnings.shift_remove(client_order_id);
287 self.order_lookback_warnings.shift_remove(client_order_id);
288 self.order_coverage_unresolved.shift_remove(client_order_id);
289 self.remove_targeted_order_queries(&[*client_order_id]);
290
291 if drop_last_query {
292 self.order_query_recency.remove(client_order_id);
293 }
294
295 self.order_activity.remove(client_order_id);
296 }
297
298 pub fn prune_order_local_activity(&mut self) {
300 self.order_activity
301 .prune_older_than(Duration::from(self.config.open_check_threshold_ns));
302 }
303
304 #[must_use]
306 pub fn is_fill_recently_processed(
307 &self,
308 account_id: AccountId,
309 instrument_id: InstrumentId,
310 trade_id: TradeId,
311 ) -> bool {
312 self.fills_recent
313 .contains_key(&(account_id, instrument_id, trade_id))
314 }
315
316 pub fn commit_recent_fill_if_applied(&mut self, fill: &OrderFilled) {
318 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
319 if self.is_fill_applied(fill, fill_key) {
320 self.mark_fill_processed(fill_key.0, fill_key.1, fill_key.2);
321 }
322 }
323
324 pub fn mark_fill_processed(
326 &mut self,
327 account_id: AccountId,
328 instrument_id: InstrumentId,
329 trade_id: TradeId,
330 ) {
331 self.fills_recent
332 .mark((account_id, instrument_id, trade_id));
333 }
334
335 pub fn prune_recent_fills_cache(&mut self, ttl_secs: f64) {
339 let ttl = match Duration::try_from_secs_f64(ttl_secs) {
347 Ok(ttl) => ttl,
348 Err(_) if ttl_secs > 0.0 => Duration::MAX,
349 Err(_) => Duration::ZERO,
350 };
351
352 self.fills_recent.prune_older_than(ttl);
353 }
354
355 pub fn prune_processed_fills(&mut self) {
360 let Some(lookback_mins) = self.config.lookback_mins else {
361 return;
362 };
363
364 let ttl = Duration::from_mins(lookback_mins).max(Duration::from_mins(1));
365 self.fills_processed.prune_older_than(ttl);
366 }
367
368 pub(crate) fn set_position_reconciliation_tolerance(
370 &mut self,
371 account_id: AccountId,
372 tolerance: Decimal,
373 ) {
374 let tolerance = if tolerance < Decimal::ZERO {
375 log::error!(
376 "Invalid negative position reconciliation tolerance {tolerance} for \
377 {account_id}; using the default"
378 );
379 DEFAULT_POSITION_RECONCILIATION_TOLERANCE
380 } else {
381 tolerance
382 };
383
384 self.position_recon_tolerances.insert(account_id, tolerance);
385 }
386
387 pub(crate) fn position_reconciliation_tolerance(&self, account_id: AccountId) -> Decimal {
389 self.position_recon_tolerances
390 .get(&account_id)
391 .copied()
392 .unwrap_or(DEFAULT_POSITION_RECONCILIATION_TOLERANCE)
393 }
394
395 pub fn record_position_activity(&mut self, instrument_id: InstrumentId, account_id: AccountId) {
398 let key = (instrument_id, account_id);
399 self.position_activity.mark(key);
400 let revision = self.position_activity_revisions.entry(key).or_default();
401 *revision = revision.saturating_add(1);
402 }
403
404 pub(crate) fn position_activity_is_recent(&self, key: &InstrumentAccountKey) -> bool {
406 self.position_activity
407 .within(key, Duration::from(self.config.position_check_threshold_ns))
408 }
409
410 pub(crate) fn position_activity_revision(&self, key: &InstrumentAccountKey) -> u64 {
412 self.position_activity_revisions
413 .get(key)
414 .copied()
415 .unwrap_or_default()
416 }
417
418 fn set_position_reconciliation_retries(
419 &mut self,
420 key: InstrumentAccountKey,
421 report_shape: PositionReportShape,
422 retries: u32,
423 ) {
424 self.position_recon.insert(
425 key,
426 PositionReconciliationState {
427 report_shape,
428 retries,
429 },
430 );
431 }
432
433 pub(crate) fn position_reconciliation_retries(
435 &self,
436 key: &InstrumentAccountKey,
437 report_shape: PositionReportShape,
438 ) -> u32 {
439 self.position_recon
440 .get(key)
441 .filter(|state| state.report_shape == report_shape)
442 .map_or(0, |state| state.retries)
443 }
444
445 #[must_use]
448 pub fn position_recon_retry_count(&self, key: &InstrumentAccountKey) -> u32 {
449 self.position_recon
450 .get(key)
451 .map_or(0, |state| state.retries)
452 }
453
454 pub(crate) fn clear_position_reconciliation(&mut self, key: &InstrumentAccountKey) {
456 self.position_recon.shift_remove(key);
457 }
458
459 pub(crate) fn retain_position_reconciliation(
461 &mut self,
462 active_keys: &IndexSet<InstrumentAccountKey>,
463 ) {
464 self.position_recon
465 .retain(|key, _| active_keys.contains(key));
466 }
467
468 pub fn reconcile_execution_mass_status(
474 &mut self,
475 mass_status: &ExecutionMassStatus,
476 exec_engine: &RefCell<ExecutionEngine>,
477 ) -> ReconciliationResult {
478 if exec_engine
479 .borrow()
480 .get_client(&mass_status.client_id)
481 .is_none()
482 {
483 log::error!(
484 "Cannot reconcile ExecutionMassStatus from unknown client {}",
485 mass_status.client_id
486 );
487 return ReconciliationResult::default();
488 }
489
490 self.validate_mass_status_order_sources(mass_status);
491
492 let raw_order_status_topic =
497 MessagingSwitchboard::reconciliation_raw_order_status_report_topic();
498
499 for report in mass_status.order_reports().values() {
500 msgbus::publish_any(raw_order_status_topic, report);
501 }
502
503 let raw_fill_topic = MessagingSwitchboard::reconciliation_raw_fill_report_topic();
504
505 for fills in mass_status.fill_reports().values() {
506 for fill in fills {
507 msgbus::publish_any(raw_fill_topic, fill);
508 }
509 }
510
511 let raw_position_topic =
512 MessagingSwitchboard::reconciliation_raw_position_status_report_topic();
513
514 for reports in mass_status.position_reports().values() {
515 for report in reports {
516 msgbus::publish_any(raw_position_topic, report);
517 }
518 }
519
520 if exec_engine
521 .borrow()
522 .get_client(&mass_status.client_id)
523 .is_none()
524 {
525 log::error!(
526 "Execution client {} disappeared while publishing raw mass status reports",
527 mass_status.client_id
528 );
529 return ReconciliationResult::default();
530 }
531
532 let venue = mass_status.venue;
533 let order_count = mass_status.order_reports().len();
534 let fill_count: usize = mass_status.fill_reports().values().map(Vec::len).sum();
535 let position_count: usize = mass_status.position_reports().values().map(Vec::len).sum();
536
537 log_info!(
538 "Reconciling ExecutionMassStatus for {venue}",
539 color = LogColor::Blue
540 );
541 log_info!(
542 "Received {order_count} order(s), {fill_count} fill(s), {position_count} position(s)",
543 color = LogColor::Blue
544 );
545
546 let retained_fill_state = self.retained_fill_state();
547 let reported_fill_keys: IndexSet<(AccountId, InstrumentId, TradeId)> = mass_status
548 .fill_reports()
549 .values()
550 .flatten()
551 .filter(|fill| !fill.last_qty.is_zero())
552 .map(|fill| (fill.account_id, fill.instrument_id, fill.trade_id))
553 .collect();
554 let (adjusted_order_reports, adjusted_fill_reports) =
555 self.adjust_mass_status_fills(mass_status);
556 let order_only_venue_order_ids = self.order_only_venue_order_ids(
557 mass_status,
558 &adjusted_order_reports,
559 &adjusted_fill_reports,
560 &retained_fill_state,
561 );
562
563 let mut events = Vec::new();
564 let mut external_orders = Vec::new();
565 let mut orders_reconciled = 0usize;
566 let mut external_orders_created = 0usize;
567 let mut open_orders_initialized = 0usize;
568 let mut orders_skipped_no_instrument = 0usize;
569 let mut orders_skipped_duplicate = 0usize;
570 let mut fills_applied = 0usize;
571 let mut fill_queue = ReconciliationFillQueue::default();
572
573 let fill_reports = &adjusted_fill_reports;
574 let mut seen_fill_keys: IndexSet<FillKey> = IndexSet::new();
575
576 for fills in fill_reports.values() {
577 for fill in fills {
578 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
579 if !seen_fill_keys.insert(fill_key) {
580 log::warn!(
581 "Duplicate trade_id {} for {} in mass status",
582 fill.trade_id,
583 fill.instrument_id
584 );
585 }
586 }
587 }
588
589 let order_reports = &adjusted_order_reports;
590 let mut orders_skipped_filtered = 0usize;
591
592 for report in order_reports.values() {
593 if self.should_skip_order_report(report) {
594 orders_skipped_filtered += 1;
595 continue;
596 }
597
598 if let Some(client_order_id) = &report.client_order_id {
599 if let Some(cached_order) = self.get_order(*client_order_id)
600 && is_exact_order_match(&cached_order, report)
601 {
602 log::debug!("Skipping order {client_order_id}: already in sync with venue");
603 orders_skipped_duplicate += 1;
604
605 if let Err(e) = self
607 .cache
608 .borrow_mut()
609 .index_venue_order_id(client_order_id, &report.venue_order_id)
610 {
611 log::warn!("Failed to index venue order ID: {e}");
612 }
613
614 continue;
615 }
616
617 if let Some(cached_order) = self.get_order(*client_order_id)
619 && cached_order.is_closed()
620 && cached_order
621 .tags()
622 .is_some_and(|tags| tags.contains(&*TAG_RECONCILIATION))
623 {
624 log::debug!(
625 "Skipping closed reconciliation order {client_order_id}: \
626 synthetic position adjustment from previous session",
627 );
628 orders_skipped_duplicate += 1;
629 continue;
630 }
631
632 if let Some(order) = self.get_order(*client_order_id) {
633 let instrument = self.get_instrument(&report.instrument_id);
634 log::info!(
635 color = LogColor::Blue as u8;
636 "Reconciling {} {} {} [{}] -> [{}]",
637 client_order_id,
638 report.venue_order_id,
639 report.instrument_id,
640 order.status(),
641 report.order_status,
642 );
643
644 let order_fills: Vec<&FillReport> = fill_reports
645 .get(&report.venue_order_id)
646 .map(|f| f.iter().collect())
647 .unwrap_or_default();
648 let engine_ref = exec_engine.borrow();
649 let commission_client = engine_ref.get_client(&mass_status.client_id);
650
651 let order_events = self.reconcile_order_with_fills(
652 true,
653 &order,
654 report,
655 &order_fills,
656 instrument.as_ref(),
657 &mut fill_queue,
658 commission_client,
659 );
660
661 drop(engine_ref);
662
663 if !order_events.is_empty() {
664 orders_reconciled += 1;
665 fills_applied += order_events
666 .iter()
667 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
668 .count();
669 events.extend(order_events);
670 }
671
672 if let Err(e) = self
674 .cache
675 .borrow_mut()
676 .index_venue_order_id(client_order_id, &report.venue_order_id)
677 {
678 log::warn!("Failed to index venue order ID: {e}");
679 }
680 } else if let Some(order) = self.get_order_by_venue_order_id(report.venue_order_id)
681 {
682 let instrument = self.get_instrument(&report.instrument_id);
684
685 log::info!(
686 color = LogColor::Blue as u8;
687 "Reconciling {} (matched by venue_order_id {}) {} [{}] -> [{}]",
688 order.client_order_id(),
689 report.venue_order_id,
690 report.instrument_id,
691 order.status(),
692 report.order_status,
693 );
694
695 let order_fills: Vec<&FillReport> = fill_reports
696 .get(&report.venue_order_id)
697 .map(|f| f.iter().collect())
698 .unwrap_or_default();
699 let engine_ref = exec_engine.borrow();
700 let commission_client = engine_ref.get_client(&mass_status.client_id);
701
702 let order_events = self.reconcile_order_with_fills(
703 true,
704 &order,
705 report,
706 &order_fills,
707 instrument.as_ref(),
708 &mut fill_queue,
709 commission_client,
710 );
711
712 drop(engine_ref);
713
714 if !order_events.is_empty() {
715 orders_reconciled += 1;
716 fills_applied += order_events
717 .iter()
718 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
719 .count();
720 events.extend(order_events);
721 }
722
723 if let Err(e) = self
724 .cache
725 .borrow_mut()
726 .index_venue_order_id(&order.client_order_id(), &report.venue_order_id)
727 {
728 log::warn!("Failed to index venue order ID: {e}");
729 }
730 } else if let Some(instrument) = self.get_instrument(&report.instrument_id) {
731 let order_fills: Vec<&FillReport> = fill_reports
732 .get(&report.venue_order_id)
733 .map(|f| f.iter().collect())
734 .unwrap_or_default();
735 let engine_ref = exec_engine.borrow();
736 let commission_client = engine_ref.get_client(&mass_status.client_id);
737
738 let (external_events, metadata) = self.handle_external_order(
739 report,
740 mass_status.account_id,
741 &instrument,
742 &order_fills,
743 false, Some(&mut fill_queue),
745 commission_client,
746 );
747
748 drop(engine_ref);
749
750 if !external_events.is_empty() {
751 external_orders_created += 1;
752 fills_applied += external_events
753 .iter()
754 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
755 .count();
756
757 if report.order_status.is_open() {
758 open_orders_initialized += 1;
759 }
760
761 events.extend(external_events);
762
763 if let Some(m) = metadata {
764 external_orders.push(m);
765 }
766 }
767 } else {
768 orders_skipped_no_instrument += 1;
769 }
770 } else if let Some(order) = self.get_order_by_venue_order_id(report.venue_order_id) {
771 let instrument = self.get_instrument(&report.instrument_id);
773 log::info!(
774 color = LogColor::Blue as u8;
775 "Reconciling {} (matched by venue_order_id {}) {} [{}] -> [{}]",
776 order.client_order_id(),
777 report.venue_order_id,
778 report.instrument_id,
779 order.status(),
780 report.order_status,
781 );
782
783 let order_fills: Vec<&FillReport> = fill_reports
784 .get(&report.venue_order_id)
785 .map(|f| f.iter().collect())
786 .unwrap_or_default();
787 let engine_ref = exec_engine.borrow();
788 let commission_client = engine_ref.get_client(&mass_status.client_id);
789
790 let order_events = self.reconcile_order_with_fills(
791 true,
792 &order,
793 report,
794 &order_fills,
795 instrument.as_ref(),
796 &mut fill_queue,
797 commission_client,
798 );
799
800 drop(engine_ref);
801
802 if !order_events.is_empty() {
803 orders_reconciled += 1;
804 fills_applied += order_events
805 .iter()
806 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
807 .count();
808 events.extend(order_events);
809 }
810
811 if let Err(e) = self
812 .cache
813 .borrow_mut()
814 .index_venue_order_id(&order.client_order_id(), &report.venue_order_id)
815 {
816 log::warn!("Failed to index venue order ID: {e}");
817 }
818 } else if let Some(instrument) = self.get_instrument(&report.instrument_id) {
819 let is_synthetic = report.venue_order_id.as_str().starts_with("S-");
821
822 let order_fills: Vec<&FillReport> = fill_reports
823 .get(&report.venue_order_id)
824 .map(|f| f.iter().collect())
825 .unwrap_or_default();
826 let engine_ref = exec_engine.borrow();
827 let commission_client = engine_ref.get_client(&mass_status.client_id);
828
829 let (external_events, metadata) = self.handle_external_order(
830 report,
831 mass_status.account_id,
832 &instrument,
833 &order_fills,
834 is_synthetic,
835 Some(&mut fill_queue),
836 commission_client,
837 );
838
839 drop(engine_ref);
840
841 if !external_events.is_empty() {
842 external_orders_created += 1;
843 fills_applied += external_events
844 .iter()
845 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
846 .count();
847
848 if report.order_status.is_open() {
849 open_orders_initialized += 1;
850 }
851
852 events.extend(external_events);
853
854 if let Some(m) = metadata {
855 external_orders.push(m);
856 }
857 }
858 } else {
859 orders_skipped_no_instrument += 1;
860 }
861 }
862
863 let processed_venue_order_ids: IndexSet<VenueOrderId> =
865 order_reports.keys().copied().collect();
866
867 for (venue_order_id, fills) in fill_reports {
868 if processed_venue_order_ids.contains(venue_order_id) {
869 continue;
870 }
871
872 let Some(first_fill) = fills.first() else {
873 continue;
874 };
875
876 if !self.should_reconcile_instrument(&first_fill.instrument_id) {
877 log::debug!(
878 "Skipping orphan fills for {}: not in reconciliation_instrument_ids",
879 first_fill.instrument_id
880 );
881 continue;
882 }
883
884 if let Some(client_order_id) = &first_fill.client_order_id
886 && self
887 .config
888 .filtered_client_order_ids
889 .contains(client_order_id)
890 {
891 log::debug!(
892 "Skipping orphan fills for {client_order_id}: in filtered_client_order_ids"
893 );
894 continue;
895 }
896
897 let order = first_fill
898 .client_order_id
899 .as_ref()
900 .and_then(|id| self.get_order(*id))
901 .or_else(|| self.get_order_by_venue_order_id(*venue_order_id));
902
903 if let Some(ref order) = order
905 && self
906 .config
907 .filtered_client_order_ids
908 .contains(&order.client_order_id())
909 {
910 log::debug!(
911 "Skipping orphan fills for {}: in filtered_client_order_ids",
912 order.client_order_id()
913 );
914 continue;
915 }
916
917 if let Some(order) = order {
918 let instrument_id = order.instrument_id();
919 if let Some(instrument) = self.get_instrument(&instrument_id) {
920 let mut sorted_fills: Vec<&FillReport> = fills.iter().collect();
921 sorted_fills.sort_by_key(|f| f.ts_event);
922
923 for fill in sorted_fills {
924 if let Some((event, fill_key)) = self.create_order_fill(
925 &order,
926 fill,
927 &instrument,
928 &fill_queue.pending_fill_keys,
929 ) {
930 fills_applied += 1;
931 fill_queue.push(&mut events, event, fill_key);
932 }
933 }
934 } else {
935 orders_skipped_no_instrument += 1;
936 }
937 } else if fills.iter().any(FillReport::has_venue_position_id) {
938 if !self.config.generate_missing_orders {
939 log::debug!(
940 "Skipping orphan fills for venue order {venue_order_id}: \
941 `generate_missing_orders` is disabled"
942 );
943 orders_skipped_filtered += 1;
944 continue;
945 }
946
947 let Some(instrument) = self.get_instrument(&first_fill.instrument_id) else {
948 orders_skipped_no_instrument += 1;
949 continue;
950 };
951
952 let mut sorted_fills: Vec<&FillReport> = fills.iter().collect();
953 sorted_fills.sort_by_key(|fill| fill.ts_event);
954
955 let report = match create_orphan_fill_order_report(&sorted_fills, &instrument) {
956 Ok(report) => report,
957 Err(e) => {
958 log::error!(
959 "Cannot materialize orphan fills for venue order {venue_order_id}: {e}"
960 );
961
962 continue;
963 }
964 };
965
966 let engine_ref = exec_engine.borrow();
967 let commission_client = engine_ref.get_client(&mass_status.client_id);
968
969 let (external_events, metadata) = self.handle_external_order(
970 &report,
971 mass_status.account_id,
972 &instrument,
973 &sorted_fills,
974 false,
975 Some(&mut fill_queue),
976 commission_client,
977 );
978
979 drop(engine_ref);
980
981 if !external_events.is_empty() {
982 external_orders_created += 1;
983 fills_applied += external_events
984 .iter()
985 .filter(|event| matches!(event, OrderEventAny::Filled(_)))
986 .count();
987
988 events.extend(external_events);
989
990 if let Some(metadata) = metadata {
991 external_orders.push(metadata);
992 }
993 }
994 }
995 }
996
997 events.sort_by_key(OrderEventAny::ts_event);
998
999 let mut unapplied_fill_position_ids = IndexSet::new();
1000
1001 for event in &events {
1002 if let OrderEventAny::Filled(fill) = event
1003 && should_project_fill(
1004 fill,
1005 &retained_fill_state,
1006 &reported_fill_keys,
1007 &order_only_venue_order_ids,
1008 )
1009 {
1010 exec_engine.borrow_mut().project_reconciliation_fill(fill);
1011 } else {
1012 exec_engine.borrow_mut().process(event);
1013 }
1014
1015 if let OrderEventAny::Filled(fill) = event
1016 && let Some(fill_key) = fill_queue.event_fill_keys.get(&fill.event_id).copied()
1017 {
1018 if self.is_fill_applied(fill, fill_key) {
1019 self.fills_processed.mark(fill_key);
1020 } else if let Some(venue_position_id) = fill.position_id {
1021 log::error!(
1022 "Skipping reconciliation for venue position {venue_position_id}: historical fill {} was not applied",
1023 fill.trade_id,
1024 );
1025
1026 unapplied_fill_position_ids.insert(venue_position_id);
1027 }
1028 }
1029 }
1030
1031 let mut positions_created = 0usize;
1032
1033 if !self.config.filter_position_reports {
1034 let instruments_with_unattributed_fills: IndexSet<InstrumentId> = mass_status
1037 .fill_reports()
1038 .values()
1039 .flatten()
1040 .filter(|f| !f.last_qty.is_zero() && f.venue_position_id.is_none())
1041 .map(|f| f.instrument_id)
1042 .chain(
1043 mass_status
1044 .order_reports()
1045 .values()
1046 .filter(|r| !r.filled_qty.is_zero() && r.venue_position_id.is_none())
1047 .map(|r| r.instrument_id),
1048 )
1049 .collect();
1050
1051 for (instrument_id, reports) in mass_status.position_reports() {
1052 if !self.should_reconcile_instrument(&instrument_id) {
1053 log::debug!(
1054 "Skipping position reports for {instrument_id}: not in reconciliation_instrument_ids"
1055 );
1056 continue;
1057 }
1058
1059 for report in reports {
1060 if report.venue_position_id.is_some_and(|venue_position_id| {
1061 unapplied_fill_position_ids.contains(&venue_position_id)
1062 }) {
1063 continue;
1064 }
1065
1066 if let Some(position_events) = self.reconcile_position_report(
1067 &report,
1068 mass_status.account_id,
1069 &instruments_with_unattributed_fills,
1070 ) {
1071 for event in position_events {
1072 exec_engine.borrow_mut().process(&event);
1073 events.push(event);
1074 }
1075
1076 positions_created += 1;
1077 }
1078 }
1079 }
1080 }
1081
1082 if orders_skipped_no_instrument > 0 {
1083 log::warn!("{orders_skipped_no_instrument} orders skipped (instrument not in cache)");
1084 }
1085
1086 if orders_skipped_duplicate > 0 {
1087 log::debug!("{orders_skipped_duplicate} orders skipped (already in sync)");
1088 }
1089
1090 if orders_skipped_filtered > 0 {
1091 log::debug!("{orders_skipped_filtered} orders skipped (filtered by config)");
1092 }
1093
1094 log::info!(
1095 color = LogColor::Blue as u8;
1096 "Reconciliation complete for {venue}: reconciled={orders_reconciled}, external={external_orders_created}, open={open_orders_initialized}, fills={fills_applied}, positions={positions_created}, skipped={orders_skipped_duplicate}, filtered={orders_skipped_filtered}",
1097 );
1098
1099 ReconciliationResult {
1100 events,
1101 external_orders,
1102 unresolved_positions: self.unresolved_mass_status_positions(mass_status),
1103 }
1104 }
1105
1106 fn unresolved_mass_status_positions(&self, mass_status: &ExecutionMassStatus) -> Vec<String> {
1107 if self.config.filter_position_reports {
1108 return Vec::new();
1109 }
1110
1111 let mut unresolved = Vec::new();
1112
1113 for (instrument_id, reports) in mass_status.position_reports() {
1114 if !self.should_reconcile_instrument(&instrument_id) {
1115 continue;
1116 }
1117
1118 if self
1119 .netting_position_reports_match((instrument_id, mass_status.account_id), &reports)
1120 {
1121 continue;
1122 }
1123
1124 for report in reports {
1125 if let Some(reason) =
1126 self.unresolved_position_report(&report, mass_status.account_id)
1127 {
1128 unresolved.push(reason);
1129 }
1130 }
1131 }
1132
1133 unresolved
1134 }
1135
1136 fn netting_position_reports_match(
1137 &self,
1138 key: InstrumentAccountKey,
1139 reports: &[PositionStatusReport],
1140 ) -> bool {
1141 let comparison = self.position_quantity_comparison(key, reports);
1142 let cache = self.cache.borrow();
1143
1144 !comparison.cached_positions.is_empty()
1145 && comparison
1146 .cached_positions
1147 .iter()
1148 .all(|position| cache.oms_type(&position.id) == Some(OmsType::Netting))
1149 && comparison.quantities_match(self.position_reconciliation_tolerance(key.1))
1150 }
1151
1152 fn unresolved_position_report(
1153 &self,
1154 report: &PositionStatusReport,
1155 account_id: AccountId,
1156 ) -> Option<String> {
1157 let venue_qty = report.signed_decimal_qty;
1158
1159 if venue_qty == Decimal::ZERO {
1160 return None;
1161 }
1162
1163 let cache = self.cache.borrow();
1164 let instrument_id = report.instrument_id;
1165
1166 let cached_qty = if let Some(position_id) = report.venue_position_id {
1167 cache
1168 .position(&position_id)
1169 .filter(|p| p.account_id == account_id && p.instrument_id == instrument_id)
1170 .map_or(Decimal::ZERO, |p| p.signed_decimal_qty())
1171 } else {
1172 cache
1173 .positions_open(None, Some(&instrument_id), None, Some(&account_id), None)
1174 .iter()
1175 .map(|p| p.signed_decimal_qty())
1176 .sum()
1177 };
1178
1179 let matches = if report.venue_position_id.is_some() {
1180 cached_qty == venue_qty
1181 } else {
1182 (cached_qty - venue_qty).abs() <= self.position_reconciliation_tolerance(account_id)
1183 };
1184
1185 if matches {
1186 return None;
1187 }
1188
1189 let reason = if cache.instrument(&instrument_id).is_none() {
1190 "instrument missing from cache"
1191 } else if cache.account(&account_id).is_none() {
1192 "account missing from cache"
1193 } else if !self.config.generate_missing_orders {
1194 "generate_missing_orders is disabled"
1195 } else if report.avg_px_open.is_none() && cached_qty == Decimal::ZERO {
1196 "missing avg_px_open for position recovery"
1197 } else {
1198 "position recovery did not restore the reported quantity"
1199 };
1200
1201 Some(format!(
1202 "account={account_id}, instrument={instrument_id}, venue_position_id={:?}, venue_quantity={venue_qty}: {reason}",
1203 report.venue_position_id,
1204 ))
1205 }
1206
1207 fn retained_fill_state(&self) -> RetainedFillState {
1208 let cache = self.cache.borrow();
1209 let positions = cache.positions(None, None, None, None, None);
1210 let mut fill_keys = IndexSet::new();
1211 let mut missing_order_ids = IndexSet::new();
1212 let mut missing_venue_order_ids = IndexSet::new();
1213 let mut netting_lifecycle_starts = IndexMap::new();
1214
1215 for position in positions {
1216 for fill in &position.events {
1217 fill_keys.insert((position.account_id, position.instrument_id, fill.trade_id));
1218 if cache.order(&fill.client_order_id).is_none() {
1219 missing_order_ids.insert((
1220 position.account_id,
1221 position.instrument_id,
1222 fill.client_order_id,
1223 ));
1224 missing_venue_order_ids.insert((
1225 position.account_id,
1226 position.instrument_id,
1227 fill.venue_order_id,
1228 ));
1229 }
1230 }
1231
1232 if cache.oms_type(&position.id) == Some(OmsType::Netting) {
1233 netting_lifecycle_starts.insert(
1234 (
1235 position.account_id,
1236 position.instrument_id,
1237 position.strategy_id,
1238 ),
1239 position.ts_opened,
1240 );
1241 }
1242 }
1243
1244 RetainedFillState {
1245 fill_keys,
1246 missing_order_ids,
1247 missing_venue_order_ids,
1248 netting_lifecycle_starts,
1249 }
1250 }
1251
1252 fn order_only_venue_order_ids(
1253 &self,
1254 mass_status: &ExecutionMassStatus,
1255 order_reports: &IndexMap<VenueOrderId, OrderStatusReport>,
1256 fill_reports: &IndexMap<VenueOrderId, Vec<FillReport>>,
1257 retained_fill_state: &RetainedFillState,
1258 ) -> IndexSet<VenueOrderId> {
1259 if mass_status.lookback_start().is_none() {
1260 return IndexSet::new();
1261 }
1262
1263 let expected_quantities: IndexMap<AccountInstrumentKey, Decimal> =
1264 if mass_status.reports_complete() {
1265 mass_status
1266 .position_reports()
1267 .into_iter()
1268 .filter_map(|(instrument_id, reports)| {
1269 let [report] = reports.as_slice() else {
1270 return None;
1271 };
1272
1273 report.venue_position_id.is_none().then_some((
1274 (report.account_id, instrument_id),
1275 report.signed_decimal_qty,
1276 ))
1277 })
1278 .collect()
1279 } else {
1280 IndexMap::new()
1281 };
1282
1283 let candidate_instruments: IndexSet<InstrumentId> = order_reports
1284 .values()
1285 .filter(|report| !report.filled_qty.is_zero())
1286 .map(|report| report.instrument_id)
1287 .chain(
1288 fill_reports
1289 .values()
1290 .flatten()
1291 .map(|fill| fill.instrument_id),
1292 )
1293 .collect();
1294
1295 if candidate_instruments.is_empty() {
1296 return IndexSet::new();
1297 }
1298
1299 let mut venue_order_ids: IndexSet<VenueOrderId> = order_reports
1300 .iter()
1301 .filter(|(_, report)| {
1302 candidate_instruments.contains(&report.instrument_id)
1303 && !report.filled_qty.is_zero()
1304 })
1305 .map(|(venue_order_id, _)| *venue_order_id)
1306 .collect();
1307
1308 venue_order_ids.extend(fill_reports.iter().filter_map(|(venue_order_id, fills)| {
1309 fills
1310 .first()
1311 .is_some_and(|fill| candidate_instruments.contains(&fill.instrument_id))
1312 .then_some(*venue_order_id)
1313 }));
1314
1315 if !mass_status.reports_complete() {
1316 log::error!(
1317 "Bounded reconciliation report set is incomplete; projecting {} historical order(s) without position or portfolio effects",
1318 venue_order_ids.len(),
1319 );
1320
1321 return venue_order_ids;
1322 }
1323
1324 let mut order_only = IndexSet::new();
1325 let mut groups = Vec::new();
1326
1327 for venue_order_id in venue_order_ids {
1328 let report = order_reports.get(&venue_order_id);
1329 let fills = fill_reports.get(&venue_order_id);
1330
1331 if report.and_then(|report| report.venue_position_id).is_some()
1332 || fills.is_some_and(|fills| fills.iter().any(FillReport::has_venue_position_id))
1333 {
1334 continue;
1335 }
1336
1337 let cached_order = report
1338 .and_then(|report| report.client_order_id)
1339 .and_then(|client_order_id| self.get_order(client_order_id))
1340 .or_else(|| self.get_order_by_venue_order_id(venue_order_id));
1341 let account_id = report
1342 .map(|report| report.account_id)
1343 .or_else(|| fills.and_then(|fills| fills.first().map(|fill| fill.account_id)));
1344 let instrument_id = report
1345 .map(|report| report.instrument_id)
1346 .or_else(|| fills.and_then(|fills| fills.first().map(|fill| fill.instrument_id)));
1347 let order_side = report
1348 .and_then(|report| report.order_side)
1349 .or_else(|| fills.and_then(|fills| fills.first().map(|fill| fill.order_side)));
1350
1351 let (Some(account_id), Some(instrument_id), Some(order_side)) =
1352 (account_id, instrument_id, order_side)
1353 else {
1354 order_only.insert(venue_order_id);
1355 continue;
1356 };
1357
1358 let coherent_fills = fills.is_none_or(|fills| {
1359 fills.iter().all(|fill| {
1360 fill.account_id == account_id
1361 && fill.instrument_id == instrument_id
1362 && fill.order_side == order_side
1363 })
1364 });
1365
1366 let coherent_cached_order = cached_order.as_ref().is_none_or(|order| {
1367 order.instrument_id() == instrument_id
1368 && order.order_side() == order_side
1369 && order.account_id().is_none_or(|id| id == account_id)
1370 });
1371
1372 if !coherent_fills
1373 || !coherent_cached_order
1374 || (report.is_none() && cached_order.is_none())
1375 {
1376 order_only.insert(venue_order_id);
1377 continue;
1378 }
1379
1380 let strategy_id = cached_order.as_ref().map_or_else(
1381 || {
1382 self.cache
1383 .borrow()
1384 .external_order_claim(&instrument_id)
1385 .unwrap_or_else(|| StrategyId::from("EXTERNAL"))
1386 },
1387 Order::strategy_id,
1388 );
1389
1390 let reduce_only = report.is_some_and(|report| report.reduce_only)
1391 || cached_order.as_ref().is_some_and(Order::is_reduce_only);
1392
1393 let cached_filled_qty = cached_order
1394 .as_ref()
1395 .map_or(Decimal::ZERO, |order| order.filled_qty().as_decimal());
1396
1397 let reported_fill_qty = fills.map_or(Decimal::ZERO, |fills| {
1398 fills.iter().map(|fill| fill.last_qty.as_decimal()).sum()
1399 });
1400
1401 let unretained_fills: Vec<&FillReport> = fills
1402 .into_iter()
1403 .flatten()
1404 .filter(|fill| {
1405 !retained_fill_state.fill_keys.contains(&(
1406 fill.account_id,
1407 fill.instrument_id,
1408 fill.trade_id,
1409 ))
1410 })
1411 .collect();
1412
1413 let unretained_fill_qty: Decimal = unretained_fills
1414 .iter()
1415 .map(|fill| fill.last_qty.as_decimal())
1416 .sum();
1417
1418 let inferred_qty = report.map_or(Decimal::ZERO, |report| {
1419 (report.filled_qty.as_decimal() - cached_filled_qty - reported_fill_qty)
1420 .max(Decimal::ZERO)
1421 });
1422
1423 let quantity = unretained_fill_qty + inferred_qty;
1424
1425 if quantity.is_zero() {
1426 continue;
1427 }
1428
1429 let inferred_ts = (!inferred_qty.is_zero())
1430 .then(|| report.map(|report| report.ts_last))
1431 .flatten();
1432 let ts_event = unretained_fills
1433 .iter()
1434 .map(|fill| fill.ts_event)
1435 .chain(inferred_ts)
1436 .min()
1437 .unwrap_or(mass_status.ts_init);
1438 let ts_last = unretained_fills
1439 .iter()
1440 .map(|fill| fill.ts_event)
1441 .chain(inferred_ts)
1442 .max()
1443 .unwrap_or(mass_status.ts_init);
1444
1445 groups.push(HistoricalFillGroup {
1446 venue_order_id,
1447 account_id,
1448 instrument_id,
1449 strategy_id,
1450 order_side,
1451 quantity,
1452 reduce_only,
1453 ts_event,
1454 ts_last,
1455 });
1456 }
1457
1458 groups.sort_by_key(|group| group.ts_event);
1459
1460 let mut quantities: IndexMap<AccountInstrumentStrategyKey, Option<Decimal>> =
1461 IndexMap::new();
1462 let mut group_ids: IndexMap<AccountInstrumentStrategyKey, Vec<VenueOrderId>> =
1463 IndexMap::new();
1464 let mut interval_ends: IndexMap<AccountInstrumentStrategyKey, UnixNanos> = IndexMap::new();
1465 let mut ambiguous_keys = IndexSet::new();
1466
1467 for group in &groups {
1468 let key = (group.account_id, group.instrument_id, group.strategy_id);
1469
1470 if interval_ends
1471 .get(&key)
1472 .is_some_and(|end| group.ts_event <= *end)
1473 {
1474 ambiguous_keys.insert(key);
1475 }
1476
1477 interval_ends
1478 .entry(key)
1479 .and_modify(|end| *end = (*end).max(group.ts_last))
1480 .or_insert(group.ts_last);
1481 }
1482
1483 if !ambiguous_keys.is_empty() {
1484 log::error!(
1485 "Bounded reconciliation contains interleaved order fills for {} position key(s); projecting their historical order state only",
1486 ambiguous_keys.len(),
1487 );
1488 }
1489
1490 for group in groups {
1491 let key = (group.account_id, group.instrument_id, group.strategy_id);
1492 group_ids.entry(key).or_default().push(group.venue_order_id);
1493 if ambiguous_keys.contains(&key) {
1494 order_only.insert(group.venue_order_id);
1495 continue;
1496 }
1497
1498 let current_qty = quantities.entry(key).or_insert_with(|| {
1499 let cache = self.cache.borrow();
1500 let positions = cache.positions_open(
1501 None,
1502 Some(&group.instrument_id),
1503 Some(&group.strategy_id),
1504 Some(&group.account_id),
1505 None,
1506 );
1507
1508 if positions.len() > 1
1509 || positions.first().is_some_and(|position| {
1510 cache.oms_type(&position.id) != Some(OmsType::Netting)
1511 })
1512 {
1513 None
1514 } else {
1515 Some(
1516 positions
1517 .first()
1518 .map_or(Decimal::ZERO, |position| position.signed_decimal_qty()),
1519 )
1520 }
1521 });
1522
1523 let Some(current_qty) = current_qty else {
1524 order_only.insert(group.venue_order_id);
1525 continue;
1526 };
1527
1528 let signed_fill_qty = match group.order_side {
1529 OrderSide::Buy => group.quantity,
1530 OrderSide::Sell => -group.quantity,
1531 };
1532
1533 let reduces = !current_qty.is_zero()
1534 && current_qty.is_sign_negative() != signed_fill_qty.is_sign_negative()
1535 && group.quantity <= current_qty.abs();
1536 if group.reduce_only && !reduces {
1537 log::warn!(
1538 "Cannot apply bounded reduce-only order {} for {} without a coherent predecessor; projecting order state only",
1539 group.venue_order_id,
1540 group.instrument_id,
1541 );
1542 order_only.insert(group.venue_order_id);
1543 continue;
1544 }
1545
1546 *current_qty += signed_fill_qty;
1547 }
1548
1549 let mut keys_by_position: IndexMap<
1550 AccountInstrumentKey,
1551 Vec<AccountInstrumentStrategyKey>,
1552 > = IndexMap::new();
1553
1554 for key in quantities.keys() {
1555 keys_by_position
1556 .entry((key.0, key.1))
1557 .or_default()
1558 .push(*key);
1559 }
1560
1561 for (position_key, keys) in keys_by_position {
1562 let expected_qty = expected_quantities.get(&position_key).copied();
1563
1564 let matches_report = if expected_qty.is_some_and(|quantity| quantity.is_zero()) {
1565 keys.iter().all(|key| {
1566 quantities
1567 .get(key)
1568 .copied()
1569 .flatten()
1570 .is_some_and(|quantity| quantity.is_zero())
1571 })
1572 } else if let (Some(expected_qty), [key]) = (expected_qty, keys.as_slice()) {
1573 let cache = self.cache.borrow();
1574 let positions = cache.positions_open(
1575 None,
1576 Some(&position_key.1),
1577 None,
1578 Some(&position_key.0),
1579 None,
1580 );
1581
1582 let cache_is_unambiguous = positions.len() <= 1
1583 && positions.first().is_none_or(|position| {
1584 position.strategy_id == key.2
1585 && cache.oms_type(&position.id) == Some(OmsType::Netting)
1586 });
1587
1588 cache_is_unambiguous
1589 && quantities
1590 .get(key)
1591 .copied()
1592 .flatten()
1593 .is_some_and(|quantity| quantity == expected_qty)
1594 } else {
1595 false
1596 };
1597
1598 if matches_report {
1599 continue;
1600 }
1601
1602 let venue_order_ids: Vec<VenueOrderId> = keys
1603 .iter()
1604 .filter_map(|key| group_ids.get(key))
1605 .flatten()
1606 .copied()
1607 .collect();
1608 log::error!(
1609 "Bounded reconciliation does not explain the reported position for {}; projecting {} historical order(s) without position or portfolio effects",
1610 position_key.1,
1611 venue_order_ids.len(),
1612 );
1613 order_only.extend(venue_order_ids);
1614 }
1615
1616 order_only
1617 }
1618
1619 fn validate_mass_status_order_sources(&self, mass_status: &ExecutionMassStatus) {
1623 let cache = self.cache.borrow();
1624 let mut checked_client_order_ids = IndexSet::new();
1625 let mut missing_origins: Vec<ClientOrderId> = Vec::new();
1626 let mut mismatched_origins: Vec<(ClientOrderId, ClientId)> = Vec::new();
1627
1628 let mut validate_report_source =
1629 |direct_client_order_id: Option<ClientOrderId>, venue_order_id: VenueOrderId| {
1630 let direct_client_order_id = direct_client_order_id
1631 .filter(|client_order_id| cache.order_exists(client_order_id));
1632 let indexed_client_order_id = cache
1633 .client_order_id(&venue_order_id)
1634 .copied()
1635 .filter(|client_order_id| cache.order_exists(client_order_id));
1636
1637 for client_order_id in [direct_client_order_id, indexed_client_order_id]
1638 .into_iter()
1639 .flatten()
1640 .filter(|client_order_id| checked_client_order_ids.insert(*client_order_id))
1641 {
1642 match cache.client_id(&client_order_id) {
1643 Some(cached_client_id) if *cached_client_id == mass_status.client_id => {}
1644 Some(cached_client_id) => {
1645 mismatched_origins.push((client_order_id, *cached_client_id));
1646 }
1647 None => missing_origins.push(client_order_id),
1648 }
1649 }
1650 };
1651
1652 for report in mass_status.order_reports().values() {
1653 validate_report_source(report.client_order_id, report.venue_order_id);
1654 }
1655
1656 for fills in mass_status.fill_reports().values() {
1657 for fill in fills {
1658 validate_report_source(fill.client_order_id, fill.venue_order_id);
1659 }
1660 }
1661
1662 if !missing_origins.is_empty() {
1663 let samples = missing_origins
1664 .iter()
1665 .take(5)
1666 .map(ToString::to_string)
1667 .collect::<Vec<_>>()
1668 .join(", ");
1669
1670 log::warn!(
1671 "Found {} cached order(s) without an execution client origin ({}): \
1672 continuing reconciliation against mass status client {} for compatibility \
1673 with existing cache data",
1674 missing_origins.len(),
1675 samples,
1676 mass_status.client_id,
1677 );
1678 }
1679
1680 if !mismatched_origins.is_empty() {
1681 let samples = mismatched_origins
1682 .iter()
1683 .take(5)
1684 .map(|(client_order_id, cached)| format!("{client_order_id} -> {cached}"))
1685 .collect::<Vec<_>>()
1686 .join(", ");
1687
1688 log::warn!(
1689 "Found {} cached order(s) with an execution client origin conflicting with \
1690 mass status client {} ({}): continuing reconciliation for compatibility; \
1691 this conflict will become a startup error in a future release, verify cached \
1692 order ownership and execution client configuration",
1693 mismatched_origins.len(),
1694 mass_status.client_id,
1695 samples,
1696 );
1697 }
1698 }
1699
1700 pub fn check_inflight_orders(&mut self) -> InflightCheckResult {
1706 let mut result = InflightCheckResult::default();
1707 let now = dst::time::Instant::now();
1708 let threshold = Duration::from_millis(self.config().inflight_threshold_ms);
1709
1710 let mut to_check = Vec::new();
1711
1712 for (client_order_id, check) in &self.order_inflight_checks {
1713 if now
1714 .checked_duration_since(check.submitted_at)
1715 .is_some_and(|elapsed| elapsed > threshold)
1716 {
1717 to_check.push(*client_order_id);
1718 }
1719 }
1720
1721 for client_order_id in to_check {
1722 if self
1723 .config
1724 .filtered_client_order_ids
1725 .contains(&client_order_id)
1726 {
1727 self.clear_recon_tracking(&client_order_id, true);
1728 continue;
1729 }
1730
1731 if self.order_query_pending.contains(&client_order_id) {
1732 continue;
1733 }
1734
1735 if let Some(check) = self.order_inflight_checks.get_mut(&client_order_id) {
1736 if let Some(last_query_at) = check.last_query_at
1737 && now
1738 .checked_duration_since(last_query_at)
1739 .is_none_or(|elapsed| elapsed < threshold)
1740 {
1741 continue;
1742 }
1743
1744 check.retry_count += 1;
1745 check.last_query_at = Some(now);
1746 self.order_query_recency.mark(client_order_id);
1747 self.order_recon_retries
1748 .insert(client_order_id, check.retry_count);
1749
1750 if check.retry_count >= self.config.inflight_max_retries {
1751 let ts_now = self.clock.borrow().timestamp_ns();
1752
1753 if let Some(order) = self.get_order(client_order_id) {
1754 match order.status() {
1755 OrderStatus::Submitted => {
1756 if let Some(event) = create_reconciliation_rejected(
1758 &order,
1759 Some("INFLIGHT_TIMEOUT"),
1760 ts_now,
1761 ) {
1762 result.events.push(event);
1763 }
1764 }
1765 OrderStatus::PendingUpdate | OrderStatus::PendingCancel => {
1766 let event = OrderEventAny::Canceled(OrderCanceled::new(
1768 order.trader_id(),
1769 order.strategy_id(),
1770 order.instrument_id(),
1771 order.client_order_id(),
1772 UUID4::new(),
1773 ts_now,
1774 ts_now,
1775 true, order.venue_order_id(),
1777 order.account_id(),
1778 None,
1779 ));
1780 result.events.push(event);
1781 }
1782 _ => {
1783 }
1785 }
1786 }
1787
1788 self.clear_recon_tracking(&client_order_id, true);
1790 } else if let Some(order) = self.get_order(client_order_id) {
1791 let ts_now = self.clock.borrow().timestamp_ns();
1793 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
1794 let query = TradingCommand::QueryOrder(QueryOrder::new(
1795 order.trader_id(),
1796 client_id,
1797 order.strategy_id(),
1798 order.instrument_id(),
1799 order.client_order_id(),
1800 order.venue_order_id(),
1801 UUID4::new(),
1802 ts_now,
1803 None,
1804 None, ));
1806 result.queries.push(query);
1807 }
1808 }
1809 }
1810
1811 result
1812 }
1813
1814 fn filtered_open_orders_for_reconciliation(&self) -> Vec<OrderAny> {
1815 let cache = self.cache.borrow();
1816 let mut orders = cache.orders_open(None, None, None, None, None);
1817 orders.extend(cache.orders_inflight(None, None, None, None, None));
1818 let mut seen_client_order_ids = IndexSet::new();
1819
1820 orders
1821 .into_iter()
1822 .filter(|order| {
1823 seen_client_order_ids.insert(order.client_order_id())
1824 && !self
1825 .config
1826 .filtered_client_order_ids
1827 .contains(&order.client_order_id())
1828 && self.should_reconcile_instrument(&order.instrument_id())
1829 })
1830 .map(|order| order.clone())
1831 .collect()
1832 }
1833
1834 fn open_position_keys_for_reconciliation(&self) -> IndexSet<InstrumentAccountKey> {
1835 let cache = self.cache.borrow();
1836 let positions = cache.positions_open(None, None, None, None, None);
1837 let mut position_keys = IndexSet::new();
1838
1839 for position in positions {
1840 if !self.should_reconcile_instrument(&position.instrument_id) {
1841 continue;
1842 }
1843
1844 position_keys.insert((position.instrument_id, position.account_id));
1845 }
1846
1847 position_keys
1848 }
1849
1850 pub async fn check_open_orders(
1854 &mut self,
1855 clients: &[&dyn ExecutionClient],
1856 ) -> Vec<OrderEventAny> {
1857 log::debug!("Checking order consistency between cached-state and venues");
1858
1859 let check = self.prepare_open_order_report_check(UUID4::new(), clients);
1860 let mut all_reports = Vec::new();
1861 let mut queried_clients = IndexSet::new();
1862 let mut failed_clients = IndexSet::new();
1863
1864 for client in clients {
1865 let client_id = client.client_id();
1866 queried_clients.insert(client_id);
1867
1868 match client.generate_order_status_reports(&check.command).await {
1869 Ok(reports) => {
1870 all_reports.extend(
1871 reports
1872 .into_iter()
1873 .map(|report| SourcedOrderStatusReport { client_id, report }),
1874 );
1875 }
1876 Err(e) => {
1877 failed_clients.insert(client_id);
1878 log::warn!(
1879 "Failed to query order reports from {}: {e}",
1880 client.client_id()
1881 );
1882 }
1883 }
1884 }
1885
1886 let result = self.reconcile_open_order_reports(
1887 &check,
1888 all_reports,
1889 &queried_clients,
1890 &failed_clients,
1891 clients,
1892 );
1893 let mut events = result.events;
1894
1895 if !result.targeted_queries.is_empty() {
1896 let query_delay =
1897 Duration::from_millis(u64::from(self.config.single_order_query_delay_ms));
1898 let query_results =
1899 request_targeted_order_reports(result.targeted_queries, clients, query_delay).await;
1900 events.extend(self.reconcile_targeted_order_reports(query_results, clients));
1901 }
1902
1903 events
1904 }
1905
1906 pub(crate) fn prepare_open_order_report_check(
1908 &mut self,
1909 command_id: UUID4,
1910 clients: &[&dyn ExecutionClient],
1911 ) -> OpenOrderReportCheck {
1912 let filtered_orders = self.filtered_open_orders_for_reconciliation();
1913 let active_order_ids: IndexSet<ClientOrderId> =
1914 filtered_orders.iter().map(Order::client_order_id).collect();
1915 self.order_coverage_warnings
1916 .retain(|client_order_id| active_order_ids.contains(client_order_id));
1917 self.order_lookback_warnings
1918 .retain(|client_order_id| active_order_ids.contains(client_order_id));
1919 self.order_coverage_unresolved
1920 .retain(|client_order_id| active_order_ids.contains(client_order_id));
1921
1922 let mut client_coverage = IndexMap::new();
1923
1924 for order in &filtered_orders {
1925 let client_order_id = order.client_order_id();
1926 let coverage = self.resolve_order_report_client_coverage(order, clients);
1927
1928 match &coverage {
1929 ReportClientCoverage::Resolved(_) => {
1930 if self
1931 .order_coverage_unresolved
1932 .shift_remove(&client_order_id)
1933 {
1934 self.order_coverage_warnings.shift_remove(&client_order_id);
1935 }
1936 }
1937 ReportClientCoverage::Unavailable(_) | ReportClientCoverage::Unresolved => {
1938 self.order_coverage_unresolved.insert(client_order_id);
1939 }
1940 }
1941
1942 client_coverage.insert(client_order_id, coverage);
1943 }
1944
1945 log::debug!(
1946 "Found {} order{} open in cache",
1947 filtered_orders.len(),
1948 if filtered_orders.len() == 1 { "" } else { "s" }
1949 );
1950
1951 let ts_now = self.clock.borrow().timestamp_ns();
1952 let start = self
1953 .config
1954 .open_check_lookback_mins
1955 .map(DurationNanos::from_mins)
1956 .map(|lookback| ts_now.saturating_sub(lookback));
1957
1958 let mut command = GenerateOrderStatusReports::new(
1959 command_id,
1960 ts_now,
1961 self.config.open_check_open_only,
1962 None,
1963 start,
1964 None,
1965 None,
1966 None,
1967 );
1968 command.log_receipt_level = LogLevel::Debug;
1969
1970 OpenOrderReportCheck {
1971 command,
1972 filtered_orders,
1973 client_coverage,
1974 }
1975 }
1976
1977 fn resolve_order_report_client_coverage(
1978 &self,
1979 order: &OrderAny,
1980 clients: &[&dyn ExecutionClient],
1981 ) -> ReportClientCoverage {
1982 if let Some(client_id) = self.cache.borrow().client_id(&order.client_order_id()) {
1983 return ReportClientCoverage::Resolved(IndexSet::from([*client_id]));
1984 }
1985
1986 if let Some(account_id) = order.account_id() {
1987 let account_clients = clients
1988 .iter()
1989 .filter(|client| client.account_id() == account_id)
1990 .map(|client| client.client_id())
1991 .collect::<IndexSet<_>>();
1992
1993 if !account_clients.is_empty() {
1994 return ReportClientCoverage::Resolved(account_clients);
1995 }
1996 }
1997
1998 let venue_clients = clients
1999 .iter()
2000 .filter(|client| client.handles_order_venue(order.instrument_id().venue))
2001 .map(|client| client.client_id())
2002 .collect::<IndexSet<_>>();
2003
2004 if venue_clients.is_empty() {
2005 ReportClientCoverage::Unresolved
2006 } else {
2007 ReportClientCoverage::Resolved(venue_clients)
2008 }
2009 }
2010
2011 pub fn check_open_order_queries(&mut self) -> Vec<TradingCommand> {
2013 self.check_open_order_queries_for_clients(None)
2014 }
2015
2016 pub(crate) fn check_open_order_queries_for_clients(
2018 &mut self,
2019 client_ids: Option<&IndexSet<ClientId>>,
2020 ) -> Vec<TradingCommand> {
2021 let now = dst::time::Instant::now();
2022 let query_delay = Duration::from_millis(u64::from(self.config.single_order_query_delay_ms));
2023 let query_limit = self.config.max_single_order_queries_per_cycle as usize;
2024
2025 if query_limit == 0 {
2026 return Vec::new();
2027 }
2028
2029 let mut filtered_orders = self.filtered_open_orders_for_reconciliation();
2030 filtered_orders.sort_by_key(|order| {
2031 let client_order_id = order.client_order_id();
2032 (
2033 self.order_query_recency.last_marked(&client_order_id),
2034 client_order_id,
2035 )
2036 });
2037
2038 let mut queries = Vec::new();
2039
2040 for order in filtered_orders {
2041 if queries.len() >= query_limit {
2042 break;
2043 }
2044
2045 let client_order_id = order.client_order_id();
2046 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
2047
2048 if let Some(client_ids) = client_ids
2049 && !client_id.is_some_and(|client_id| client_ids.contains(&client_id))
2050 {
2051 continue;
2052 }
2053
2054 let threshold = Duration::from(self.config.open_check_threshold_ns);
2055 if let Some(elapsed) = self.order_activity.elapsed_at(&client_order_id, now)
2056 && elapsed < threshold
2057 {
2058 let elapsed_ms = elapsed.as_millis();
2059 let threshold_ms = threshold.as_millis();
2060 log::debug!(
2061 "Deferring open order query for {client_order_id}: recent local activity \
2062 ({elapsed_ms}ms < threshold={threshold_ms}ms)",
2063 );
2064 continue;
2065 }
2066
2067 if self
2068 .order_query_recency
2069 .within_at(&client_order_id, now, query_delay)
2070 {
2071 continue;
2072 }
2073
2074 self.order_query_recency.mark(client_order_id);
2075 let ts_now = self.clock.borrow().timestamp_ns();
2076
2077 let cmd = TradingCommand::QueryOrder(QueryOrder::new(
2078 order.trader_id(),
2079 client_id,
2080 order.strategy_id(),
2081 order.instrument_id(),
2082 client_order_id,
2083 order.venue_order_id(),
2084 UUID4::new(),
2085 ts_now,
2086 None,
2087 None,
2088 ));
2089 queries.push(cmd);
2090 }
2091
2092 queries
2093 }
2094
2095 pub(crate) fn reconcile_open_order_reports(
2097 &mut self,
2098 check: &OpenOrderReportCheck,
2099 mut all_reports: Vec<SourcedOrderStatusReport>,
2100 queried_clients: &IndexSet<ClientId>,
2101 failed_clients: &IndexSet<ClientId>,
2102 clients: &[&dyn ExecutionClient],
2103 ) -> OpenOrderReconciliationResult {
2104 all_reports.retain(|sourced| !self.should_skip_order_report(&sourced.report));
2105 let mut venue_reported_ids = IndexSet::new();
2106
2107 for sourced in &all_reports {
2108 let report = &sourced.report;
2109 if let Some(client_order_id) = &report.client_order_id {
2110 venue_reported_ids.insert(*client_order_id);
2111 self.order_coverage_warnings.shift_remove(client_order_id);
2112 self.order_lookback_warnings.shift_remove(client_order_id);
2113 self.order_recon_retries.shift_remove(client_order_id);
2117 } else {
2118 let mapped_client_order_id = self
2119 .cache
2120 .borrow()
2121 .client_order_id(&report.venue_order_id)
2122 .copied();
2123
2124 if let Some(client_order_id) = mapped_client_order_id {
2128 venue_reported_ids.insert(client_order_id);
2129 self.order_coverage_warnings.shift_remove(&client_order_id);
2130 self.order_lookback_warnings.shift_remove(&client_order_id);
2131 self.order_recon_retries.shift_remove(&client_order_id);
2132 }
2133 }
2134 }
2135
2136 let mut events = Vec::new();
2137 let mut targeted_candidates = Vec::new();
2138
2139 for sourced in all_reports {
2140 let report = sourced.report;
2141
2142 let order = match report.client_order_id {
2143 Some(client_order_id) => self.get_order(client_order_id),
2144 None => self.get_order_by_venue_order_id(report.venue_order_id),
2145 };
2146
2147 let Some(order) = order else {
2148 continue;
2149 };
2150
2151 let client_order_id = order.client_order_id();
2152
2153 let threshold = Duration::from(self.config.open_check_threshold_ns);
2155 if let Some(elapsed) = self.order_activity.elapsed(&client_order_id)
2156 && elapsed < threshold
2157 {
2158 let elapsed_ms = elapsed.as_millis();
2159 let threshold_ms = threshold.as_millis();
2160 log::debug!(
2161 "Deferring reconciliation for {client_order_id}: recent local activity ({elapsed_ms}ms < threshold={threshold_ms}ms)",
2162 );
2163 continue;
2164 }
2165
2166 let instrument = self.get_instrument(&report.instrument_id);
2167
2168 if terminal_report_has_missing_fills(&report, order.filled_qty()) {
2169 targeted_candidates.push((
2170 order,
2171 IndexSet::from([sourced.client_id]),
2172 Some(report),
2173 ));
2174 continue;
2175 }
2176
2177 let commission_client = clients
2178 .iter()
2179 .find(|client| client.client_id() == sourced.client_id)
2180 .copied();
2181
2182 match self.reconcile_order_report(
2183 &order,
2184 &report,
2185 instrument.as_ref(),
2186 commission_client,
2187 ) {
2188 Ok(order_events) => events.extend(order_events),
2189 Err(e) => log::error!(
2190 "Deferring reconciliation for {client_order_id}: venue commission calculation failed: {e}"
2191 ),
2192 }
2193 }
2194
2195 if self.config.open_check_open_only {
2200 let cached_ids: IndexSet<ClientOrderId> = check
2201 .filtered_orders
2202 .iter()
2203 .map(Order::client_order_id)
2204 .collect();
2205 let missing_at_venue: IndexSet<ClientOrderId> = cached_ids
2206 .difference(&venue_reported_ids)
2207 .copied()
2208 .collect();
2209
2210 if !missing_at_venue.is_empty() {
2211 log::debug!(
2212 "{} cached open order{} not present in venue current response",
2213 missing_at_venue.len(),
2214 if missing_at_venue.len() == 1 {
2215 " is"
2216 } else {
2217 "s are"
2218 },
2219 );
2220
2221 for client_order_id in missing_at_venue {
2222 log::debug!("Cached open order missing from venue response: {client_order_id}");
2223 }
2224 }
2225 } else {
2226 let candidates: Vec<&OrderAny> = if let Some(cutoff) = check.command.start {
2227 let mut candidates = Vec::new();
2228
2229 for order in &check.filtered_orders {
2230 let client_order_id = order.client_order_id();
2231 if order.ts_last() >= cutoff {
2232 self.order_lookback_warnings.shift_remove(&client_order_id);
2233 candidates.push(order);
2234 } else if !venue_reported_ids.contains(&client_order_id)
2235 && self.order_lookback_warnings.insert(client_order_id)
2236 {
2237 log::warn!(
2238 "Skipping missing-order reconciliation for {client_order_id}: its last update predates the configured open-check lookback window; absence from the bulk response cannot be treated as evidence and no targeted query will be issued from it"
2239 );
2240 }
2241 }
2242
2243 candidates
2244 } else {
2245 check.filtered_orders.iter().collect()
2246 };
2247
2248 for order in candidates {
2249 let client_order_id = order.client_order_id();
2250 if venue_reported_ids.contains(&client_order_id) {
2251 continue;
2252 }
2253
2254 let coverage = check
2255 .client_coverage
2256 .get(&client_order_id)
2257 .unwrap_or(&ReportClientCoverage::Unresolved);
2258
2259 let ReportClientCoverage::Resolved(responsible_clients) = coverage else {
2260 if self.order_coverage_warnings.insert(client_order_id) {
2261 log::warn!(
2262 "Skipping order reconciliation for {client_order_id}: responsible execution client coverage is unresolved"
2263 );
2264 }
2265
2266 continue;
2267 };
2268
2269 if responsible_clients.is_empty() {
2270 if self.order_coverage_warnings.insert(client_order_id) {
2271 log::warn!(
2272 "Skipping order reconciliation for {client_order_id}: responsible execution client coverage is unresolved"
2273 );
2274 }
2275
2276 continue;
2277 }
2278
2279 let missing_clients = responsible_clients
2280 .difference(queried_clients)
2281 .copied()
2282 .collect::<IndexSet<_>>();
2283
2284 if !missing_clients.is_empty() {
2285 if self.order_coverage_warnings.insert(client_order_id) {
2286 log::warn!(
2287 "Skipping order reconciliation for {client_order_id}: responsible execution clients were not queried: {missing_clients:?}"
2288 );
2289 }
2290
2291 continue;
2292 }
2293
2294 let failed_responsible_clients = responsible_clients
2295 .intersection(failed_clients)
2296 .copied()
2297 .collect::<IndexSet<_>>();
2298
2299 if !failed_responsible_clients.is_empty() {
2300 log::warn!(
2301 "Skipping order reconciliation for {client_order_id}: failed to query responsible execution clients: {failed_responsible_clients:?}"
2302 );
2303 continue;
2304 }
2305
2306 self.order_coverage_warnings.shift_remove(&client_order_id);
2307 if let Some(order) = self.prepare_missing_order_query(client_order_id) {
2308 targeted_candidates.push((order, responsible_clients.clone(), None));
2309 }
2310 }
2311 }
2312
2313 targeted_candidates.sort_by_key(|(order, _, _)| {
2314 let client_order_id = order.client_order_id();
2315 (
2316 self.order_query_recency.last_marked(&client_order_id),
2317 client_order_id,
2318 )
2319 });
2320
2321 let query_limit = self.config.max_single_order_queries_per_cycle as usize;
2322 let mut planned_queries = 0usize;
2323 let mut cap_deferred_orders = 0usize;
2324 let mut targeted_queries = Vec::new();
2325
2326 for (order, responsible_clients, report) in targeted_candidates {
2327 let client_order_id = order.client_order_id();
2328
2329 let required_queries = responsible_clients.len();
2330 let exceeds_query_limit = planned_queries + required_queries > query_limit;
2331 let can_run_oversized_group = planned_queries == 0 && query_limit > 0;
2332 if required_queries == 0 || (exceeds_query_limit && !can_run_oversized_group) {
2333 cap_deferred_orders += 1;
2334 continue;
2335 }
2336
2337 if required_queries > query_limit {
2338 log::warn!(
2339 "Targeted order query for {client_order_id} requires {required_queries} responsible clients, exceeding the per-cycle limit {query_limit} to avoid indefinite deferral"
2340 );
2341 }
2342
2343 planned_queries += required_queries;
2344 self.order_query_recency.mark(client_order_id);
2345 self.order_query_pending.insert(client_order_id);
2346 let command_id = UUID4::new();
2347 let ts_now = self.clock.borrow().timestamp_ns();
2348
2349 let command = GenerateOrderStatusReport::new(
2350 command_id,
2351 ts_now,
2352 Some(order.instrument_id()),
2353 Some(client_order_id),
2354 order.venue_order_id(),
2355 None,
2356 None,
2357 );
2358 targeted_queries.push(TargetedOrderQuery {
2359 client_order_id,
2360 responsible_clients,
2361 report,
2362 filled_qty: order.filled_qty(),
2363 command,
2364 });
2365 }
2366
2367 if cap_deferred_orders > 0 {
2368 log::warn!(
2369 "Reached max single-order queries ({query_limit}) this cycle, deferring {cap_deferred_orders} order(s)"
2370 );
2371 }
2372
2373 OpenOrderReconciliationResult {
2374 events,
2375 targeted_queries,
2376 }
2377 }
2378
2379 pub(crate) fn reconcile_targeted_order_reports(
2381 &mut self,
2382 results: Vec<TargetedOrderReportResult>,
2383 clients: &[&dyn ExecutionClient],
2384 ) -> Vec<OrderEventAny> {
2385 let mut events = Vec::new();
2386 let mut fill_queue = ReconciliationFillQueue::default();
2387
2388 for result in results {
2389 let client_order_id = result.client_order_id;
2390 self.remove_targeted_order_queries(&[client_order_id]);
2391
2392 if let Some(report) = result.report {
2393 self.order_recon_retries.shift_remove(&client_order_id);
2394 self.order_coverage_warnings.shift_remove(&client_order_id);
2395
2396 let Some(order) = self.get_order(client_order_id) else {
2397 continue;
2398 };
2399
2400 let instrument = self.get_instrument(&report.instrument_id);
2401
2402 let commission_client = result.client_id.and_then(|client_id| {
2403 clients
2404 .iter()
2405 .find(|client| client.client_id() == client_id)
2406 .copied()
2407 });
2408
2409 log::info!(
2410 color = LogColor::Blue as u8;
2411 "Found {client_order_id} via targeted order status query: {}",
2412 report.order_status,
2413 );
2414
2415 let fills = result.fills.iter().collect::<Vec<_>>();
2416 events.extend(self.reconcile_order_with_fills(
2417 false,
2418 &order,
2419 &report,
2420 &fills,
2421 instrument.as_ref(),
2422 &mut fill_queue,
2423 commission_client,
2424 ));
2425 continue;
2426 }
2427
2428 if result.coverage_complete {
2429 events.extend(self.resolve_missing_order(client_order_id));
2430 } else {
2431 log::warn!(
2432 "Deferring missing-order resolution for {client_order_id}: targeted order status coverage was incomplete"
2433 );
2434 }
2435 }
2436
2437 events
2438 }
2439
2440 pub async fn check_positions_consistency(
2445 &mut self,
2446 clients: &[&dyn ExecutionClient],
2447 ) -> Vec<OrderEventAny> {
2448 let check = self.prepare_position_report_check(UUID4::new(), clients);
2449 let mut reports = Vec::new();
2450 let mut queried_clients = IndexSet::new();
2451 let mut failed_clients = IndexSet::new();
2452
2453 for client in clients {
2454 let client_id = client.client_id();
2455 queried_clients.insert(client_id);
2456 self.set_position_reconciliation_tolerance(
2457 client.account_id(),
2458 client.position_reconciliation_tolerance(),
2459 );
2460
2461 match client
2462 .generate_position_status_reports(&check.command)
2463 .await
2464 {
2465 Ok(client_reports) => {
2466 reports.extend(client_reports);
2467 }
2468 Err(e) => {
2469 failed_clients.insert(client_id);
2470 log::warn!(
2471 "Failed to query position reports from {}: {e}",
2472 client.client_id()
2473 );
2474 }
2475 }
2476 }
2477
2478 let active_keys = self
2479 .open_position_keys_for_reconciliation()
2480 .into_iter()
2481 .chain(reports.iter().filter_map(|report| {
2482 (self.should_reconcile_instrument(&report.instrument_id)
2483 && report.signed_decimal_qty != Decimal::ZERO)
2484 .then_some((report.instrument_id, report.account_id))
2485 }))
2486 .collect();
2487
2488 let events =
2489 self.reconcile_position_reports(&check, reports, &queried_clients, &failed_clients);
2490
2491 self.retain_position_reconciliation(&active_keys);
2493
2494 events
2495 }
2496
2497 #[must_use]
2502 pub fn prepare_position_report_check(
2503 &self,
2504 command_id: UUID4,
2505 clients: &[&dyn ExecutionClient],
2506 ) -> PositionReportCheck {
2507 let position_keys = self.open_position_keys_for_reconciliation();
2508
2509 let client_coverage = position_keys
2510 .iter()
2511 .map(|key| (*key, resolve_position_report_client_coverage(*key, clients)))
2512 .collect();
2513
2514 let mut activity_revisions = self.position_activity_revisions.clone();
2515 for key in &position_keys {
2516 activity_revisions
2517 .entry(*key)
2518 .or_insert_with(|| self.position_activity_revision(key));
2519 }
2520
2521 log::debug!(
2522 "Found {} unique instrument/account combination{} with open positions",
2523 position_keys.len(),
2524 if position_keys.len() == 1 { "" } else { "s" }
2525 );
2526
2527 let ts_now = self.clock.borrow().timestamp_ns();
2528
2529 let mut command = GeneratePositionStatusReports::new(
2530 command_id, ts_now, None, None, None, None, None, );
2536 command.log_receipt_level = LogLevel::Debug;
2537
2538 PositionReportCheck {
2539 command,
2540 client_coverage,
2541 activity_revisions,
2542 }
2543 }
2544
2545 pub fn plan_position_fill_reports(
2550 &mut self,
2551 check: &mut PositionReportCheck,
2552 reports: &[PositionStatusReport],
2553 queried_clients: &IndexSet<ClientId>,
2554 failed_clients: &IndexSet<ClientId>,
2555 clients: &[&dyn ExecutionClient],
2556 ) -> PositionFillReportPlan {
2557 let mut venue_positions: IndexMap<InstrumentAccountKey, Vec<PositionStatusReport>> =
2558 IndexMap::new();
2559
2560 for report in reports {
2561 if self.should_reconcile_instrument(&report.instrument_id) {
2562 venue_positions
2563 .entry((report.instrument_id, report.account_id))
2564 .or_default()
2565 .push(report.clone());
2566 }
2567 }
2568
2569 let keys = check
2570 .client_coverage
2571 .keys()
2572 .copied()
2573 .chain(venue_positions.iter().filter_map(|(key, reports)| {
2574 reports
2575 .iter()
2576 .any(|report| report.signed_decimal_qty != Decimal::ZERO)
2577 .then_some(*key)
2578 }))
2579 .collect::<IndexSet<_>>();
2580
2581 let active_keys = keys.clone();
2582 let query_end = self.timestamp_ns();
2583 let lookback = DurationNanos::from_mins(self.config.position_check_lookback_mins);
2584 let query_start = query_end.saturating_sub(lookback);
2585 let mut discrepancy_keys = IndexSet::new();
2586 let mut queries = Vec::new();
2587
2588 for key in keys {
2589 let coverage = check
2590 .client_coverage
2591 .entry(key)
2592 .or_insert_with(|| resolve_position_report_client_coverage(key, clients));
2593 let prepared_revision = *check.activity_revisions.entry(key).or_default();
2594 let venue_reports = venue_positions
2595 .get(&key)
2596 .map(Vec::as_slice)
2597 .unwrap_or_default();
2598 let comparison = self.position_quantity_comparison(key, venue_reports);
2599 let tolerance = self.position_reconciliation_tolerance(key.1);
2600
2601 if comparison.quantities_match(tolerance) {
2602 self.clear_position_reconciliation(&key);
2603 continue;
2604 }
2605
2606 discrepancy_keys.insert(key);
2607
2608 if self.position_activity_revision(&key) > prepared_revision
2609 || self.position_activity_is_recent(&key)
2610 {
2611 continue;
2612 }
2613
2614 let report_shape = comparison.report_shape();
2615 let retries = self.position_reconciliation_retries(&key, report_shape);
2616 if retries >= self.config().position_check_retries {
2617 continue;
2618 }
2619
2620 let ReportClientCoverage::Resolved(responsible_clients) = coverage else {
2621 log::warn!(
2622 "Skipping fill report query for {}/{}: responsible execution client coverage is unavailable",
2623 key.0,
2624 key.1,
2625 );
2626 continue;
2627 };
2628
2629 if responsible_clients.is_empty()
2630 || !responsible_clients.is_subset(queried_clients)
2631 || !responsible_clients.is_disjoint(failed_clients)
2632 {
2633 log::warn!(
2634 "Skipping fill report query for {}/{}: responsible position report coverage is incomplete",
2635 key.0,
2636 key.1,
2637 );
2638 continue;
2639 }
2640
2641 for client_id in responsible_clients.iter() {
2642 let mut command = GenerateFillReports::new(
2643 UUID4::new(),
2644 query_end,
2645 Some(key.0),
2646 None,
2647 Some(query_start),
2648 Some(query_end),
2649 None,
2650 Some(check.command.command_id),
2651 );
2652 command.log_receipt_level = LogLevel::Debug;
2653 queries.push(PositionFillReportQuery {
2654 key,
2655 client_id: *client_id,
2656 command,
2657 });
2658 }
2659 }
2660
2661 self.retain_position_reconciliation(&active_keys);
2662
2663 PositionFillReportPlan {
2664 queries,
2665 discrepancy_keys,
2666 }
2667 }
2668
2669 #[must_use]
2671 pub fn position_report_check_is_current(
2672 &self,
2673 check: &PositionReportCheck,
2674 key: &InstrumentAccountKey,
2675 ) -> bool {
2676 check
2677 .activity_revisions
2678 .get(key)
2679 .is_some_and(|revision| self.position_activity_revision(key) == *revision)
2680 }
2681
2682 pub fn prepare_position_fill_report(
2689 &self,
2690 report: &mut FillReport,
2691 venue_reports: &[PositionStatusReport],
2692 ) -> anyhow::Result<PositionFillReportPreparation> {
2693 let cache = self.cache();
2694 let venue_client_order_id = cache.client_order_id(&report.venue_order_id).copied();
2695 if let (Some(report_client_order_id), Some(venue_client_order_id)) =
2696 (report.client_order_id, venue_client_order_id)
2697 {
2698 anyhow::ensure!(
2699 report_client_order_id == venue_client_order_id,
2700 "fill {} client order ID {report_client_order_id} conflicts with venue order mapping {venue_client_order_id}",
2701 report.trade_id,
2702 );
2703 }
2704
2705 let client_order_id = report.client_order_id.or(venue_client_order_id);
2706 let order = client_order_id.and_then(|id| cache.order(&id));
2707 if let Some(order) = &order {
2708 anyhow::ensure!(
2709 order.instrument_id() == report.instrument_id
2710 && order.order_side() == report.order_side
2711 && order
2712 .account_id()
2713 .is_none_or(|account_id| account_id == report.account_id)
2714 && order
2715 .venue_order_id()
2716 .is_none_or(|venue_order_id| venue_order_id == report.venue_order_id),
2717 "fill {} conflicts with cached order {}",
2718 report.trade_id,
2719 order.client_order_id(),
2720 );
2721 }
2722
2723 let hedge_context = report.venue_position_id.is_some()
2724 || venue_reports
2725 .iter()
2726 .any(|venue_report| venue_report.venue_position_id.is_some());
2727 let mapped_position_id = client_order_id
2728 .and_then(|client_order_id| cache.position_id(&client_order_id))
2729 .copied();
2730
2731 if hedge_context
2732 && let (Some(venue_position_id), Some(mapped_position_id)) =
2733 (report.venue_position_id, mapped_position_id)
2734 {
2735 anyhow::ensure!(
2736 venue_position_id == mapped_position_id,
2737 "fill {} position ID {venue_position_id} conflicts with cached order position {mapped_position_id}",
2738 report.trade_id,
2739 );
2740 }
2741
2742 if let Some(order) = order
2743 && has_active_inferred_fill(&order)?
2744 {
2745 return Ok(PositionFillReportPreparation::InferredOverlap);
2746 }
2747
2748 if !hedge_context {
2749 return Ok(PositionFillReportPreparation::Ready);
2750 }
2751
2752 if report.venue_position_id.is_some() {
2753 return Ok(PositionFillReportPreparation::Ready);
2754 }
2755
2756 let Some(position_id) = mapped_position_id else {
2757 return Ok(PositionFillReportPreparation::Unattributed);
2758 };
2759
2760 let position = cache.position(&position_id).ok_or_else(|| {
2761 anyhow::anyhow!(
2762 "fill {} maps to position {position_id}, which is not cached",
2763 report.trade_id,
2764 )
2765 })?;
2766
2767 anyhow::ensure!(
2768 position.account_id == report.account_id
2769 && position.instrument_id == report.instrument_id,
2770 "fill {} maps to position {position_id} with a different account or instrument",
2771 report.trade_id,
2772 );
2773 anyhow::ensure!(
2774 position.is_open(),
2775 "fill {} maps to non-open position {position_id}",
2776 report.trade_id,
2777 );
2778 anyhow::ensure!(
2779 !position.is_opposite_side(report.order_side) || report.last_qty <= position.quantity,
2780 "fill {} without a venue position ID would cross position {position_id}",
2781 report.trade_id,
2782 );
2783
2784 report.venue_position_id = Some(position_id);
2785 Ok(PositionFillReportPreparation::Ready)
2786 }
2787
2788 #[must_use]
2790 pub fn position_contains_fill_report(&self, report: &FillReport) -> bool {
2791 let cache = self.cache();
2792 let client_order_id = report
2793 .client_order_id
2794 .or_else(|| cache.client_order_id(&report.venue_order_id).copied());
2795 let positions = cache.positions(
2796 None,
2797 Some(&report.instrument_id),
2798 None,
2799 Some(&report.account_id),
2800 None,
2801 );
2802 let mut matched = false;
2803 let mut quantity = Quantity::zero(report.last_qty.precision);
2804 let mut commission = Money::zero(report.commission.currency);
2805
2806 for position in positions {
2807 if report
2808 .venue_position_id
2809 .is_some_and(|position_id| position.id != position_id)
2810 {
2811 continue;
2812 }
2813
2814 for replay_event in &position.replay_events {
2815 let PositionReplayEvent::Filled(fill) = replay_event else {
2816 continue;
2817 };
2818
2819 if fill.account_id != report.account_id
2820 || fill.instrument_id != report.instrument_id
2821 || fill.venue_order_id != report.venue_order_id
2822 || fill.trade_id != report.trade_id
2823 || fill.order_side != report.order_side
2824 || fill.last_px != report.last_px
2825 || fill.liquidity_side != report.liquidity_side
2826 || client_order_id.is_some_and(|id| fill.client_order_id != id)
2827 || report
2828 .venue_position_id
2829 .is_some_and(|id| fill.position_id != Some(id))
2830 {
2831 continue;
2832 }
2833
2834 let Some(fill_commission) = fill.commission else {
2835 return false;
2836 };
2837
2838 if fill_commission.currency != report.commission.currency {
2839 return false;
2840 }
2841
2842 let Some(next_quantity) = quantity.checked_add(fill.last_qty) else {
2843 return false;
2844 };
2845
2846 let Some(next_commission) = commission.checked_add(fill_commission) else {
2847 return false;
2848 };
2849
2850 matched = true;
2851 quantity = next_quantity;
2852 commission = next_commission;
2853 }
2854 }
2855
2856 matched && quantity == report.last_qty && commission == report.commission
2857 }
2858
2859 #[must_use]
2865 pub fn reconcile_position_reports(
2866 &mut self,
2867 check: &PositionReportCheck,
2868 reports: Vec<PositionStatusReport>,
2869 queried_clients: &IndexSet<ClientId>,
2870 failed_clients: &IndexSet<ClientId>,
2871 ) -> Vec<OrderEventAny> {
2872 log::debug!("Checking position consistency between cached-state and venues");
2873
2874 let mut venue_positions: IndexMap<InstrumentAccountKey, Vec<PositionStatusReport>> =
2875 IndexMap::new();
2876
2877 for report in reports {
2878 if !self.should_reconcile_instrument(&report.instrument_id) {
2879 continue;
2880 }
2881
2882 venue_positions
2883 .entry((report.instrument_id, report.account_id))
2884 .or_default()
2885 .push(report);
2886 }
2887
2888 let mut events = Vec::new();
2889
2890 for key in check.client_coverage.keys() {
2891 let prepared_revision = check
2892 .activity_revisions
2893 .get(key)
2894 .copied()
2895 .unwrap_or_default();
2896
2897 if self.position_activity_revision(key) > prepared_revision {
2898 log::debug!(
2899 "Deferring position reconciliation for {}/{}: local activity recorded during report request",
2900 key.0,
2901 key.1,
2902 );
2903 continue;
2904 }
2905
2906 let venue_reports = venue_positions
2907 .get(key)
2908 .map(Vec::as_slice)
2909 .unwrap_or_default();
2910
2911 if venue_reports.is_empty() {
2912 match check.client_coverage.get(key) {
2913 Some(ReportClientCoverage::Resolved(responsible_clients))
2914 if !responsible_clients.is_empty()
2915 && responsible_clients.is_subset(queried_clients)
2916 && responsible_clients.is_disjoint(failed_clients) => {}
2917 Some(ReportClientCoverage::Resolved(responsible_clients))
2918 if responsible_clients.is_empty() =>
2919 {
2920 log::warn!(
2921 "Skipping position reconciliation for {}/{}: responsible execution client coverage is unresolved",
2922 key.0,
2923 key.1,
2924 );
2925 continue;
2926 }
2927 Some(ReportClientCoverage::Resolved(responsible_clients))
2928 if !responsible_clients.is_subset(queried_clients) =>
2929 {
2930 log::warn!(
2931 "Skipping position reconciliation for {}/{}: responsible execution clients were not all queried",
2932 key.0,
2933 key.1,
2934 );
2935 continue;
2936 }
2937 Some(ReportClientCoverage::Resolved(responsible_clients)) => {
2938 let failed_responsible_clients = responsible_clients
2939 .intersection(failed_clients)
2940 .copied()
2941 .collect::<IndexSet<_>>();
2942 log::warn!(
2943 "Skipping position reconciliation for {}/{}: failed to query responsible execution clients: {failed_responsible_clients:?}",
2944 key.0,
2945 key.1,
2946 );
2947 continue;
2948 }
2949 Some(ReportClientCoverage::Unavailable(responsible_clients)) => {
2950 log::debug!(
2951 "Skipping position reconciliation for {}/{}: complete bulk position coverage is unavailable from responsible execution clients: {responsible_clients:?}",
2952 key.0,
2953 key.1,
2954 );
2955 continue;
2956 }
2957 Some(ReportClientCoverage::Unresolved) | None => {
2958 log::warn!(
2959 "Skipping position reconciliation for {}/{}: responsible execution client coverage is unresolved",
2960 key.0,
2961 key.1,
2962 );
2963 continue;
2964 }
2965 }
2966 }
2967
2968 if let Some(discrepancy_events) = self.check_position_discrepancy(*key, venue_reports) {
2969 events.extend(discrepancy_events);
2970 }
2971 }
2972
2973 let current_position_keys = self.open_position_keys_for_reconciliation();
2974
2975 for (key, venue_reports) in &venue_positions {
2976 if check.client_coverage.contains_key(key)
2977 || venue_reports
2978 .iter()
2979 .all(|report| report.signed_decimal_qty == Decimal::ZERO)
2980 {
2981 continue;
2982 }
2983
2984 if current_position_keys.contains(key) {
2985 log::debug!(
2986 "Deferring position reconciliation for {}/{}: position opened after client coverage was recorded",
2987 key.0,
2988 key.1,
2989 );
2990 continue;
2991 }
2992
2993 if let Some(discrepancy_events) = self.check_position_discrepancy(*key, venue_reports) {
2994 events.extend(discrepancy_events);
2995 }
2996 }
2997
2998 events
2999 }
3000
3001 #[must_use]
3003 pub fn get_external_order_claim(&self, instrument_id: &InstrumentId) -> Option<StrategyId> {
3004 self.cache.borrow().external_order_claim(instrument_id)
3005 }
3006
3007 pub fn claim_external_orders(
3013 &mut self,
3014 instrument_id: InstrumentId,
3015 strategy_id: StrategyId,
3016 ) -> anyhow::Result<()> {
3017 self.cache
3018 .borrow_mut()
3019 .register_external_order_claims(strategy_id, &[instrument_id])
3020 }
3021
3022 pub fn observe_order_event(&mut self, event: &OrderEventAny) {
3032 match event {
3033 OrderEventAny::Filled(fill) => {
3034 self.record_position_activity(fill.instrument_id, fill.account_id);
3035 }
3036 OrderEventAny::Accepted(_)
3037 | OrderEventAny::Rejected(_)
3038 | OrderEventAny::Canceled(_)
3039 | OrderEventAny::Expired(_)
3040 | OrderEventAny::Denied(_)
3041 | OrderEventAny::Updated(_)
3042 | OrderEventAny::ModifyRejected(_)
3043 | OrderEventAny::CancelRejected(_) => {
3044 self.clear_recon_tracking(&event.client_order_id(), true);
3045 }
3046 _ => {}
3047 }
3048
3049 self.record_local_activity(event.client_order_id());
3050 }
3051
3052 pub fn observe_execution_report(&mut self, report: &ExecutionReport) {
3065 match report {
3066 ExecutionReport::Order(order_report) => {
3067 self.observe_order_status_report(order_report);
3068 }
3069 ExecutionReport::Fill(fill_report) => {
3070 let client_order_id = fill_report.client_order_id.or_else(|| {
3071 self.cache
3072 .borrow()
3073 .client_order_id(&fill_report.venue_order_id)
3074 .copied()
3075 });
3076
3077 if let Some(coid) = client_order_id {
3078 self.record_local_activity(coid);
3079 }
3080
3081 self.record_position_activity(fill_report.instrument_id, fill_report.account_id);
3082 }
3083 ExecutionReport::OrderWithFills(order_report, fills) => {
3084 self.observe_order_status_report(order_report);
3085
3086 for fill_report in fills {
3087 self.record_position_activity(
3088 fill_report.instrument_id,
3089 fill_report.account_id,
3090 );
3091 }
3092 }
3093 ExecutionReport::Position(position_report) => {
3094 self.record_position_activity(
3095 position_report.instrument_id,
3096 position_report.account_id,
3097 );
3098 }
3099 ExecutionReport::MassStatus(_) => {
3100 }
3102 }
3103 }
3104
3105 fn observe_order_status_report(&mut self, report: &OrderStatusReport) {
3106 let Some(client_order_id) = report.client_order_id else {
3107 return;
3108 };
3109
3110 let accepted_during_pending_command = report.order_status == OrderStatus::Accepted
3111 && self.get_order(client_order_id).is_some_and(|order| {
3112 matches!(
3113 order.status(),
3114 OrderStatus::PendingUpdate | OrderStatus::PendingCancel
3115 )
3116 });
3117
3118 if !matches!(
3119 report.order_status,
3120 OrderStatus::PendingUpdate | OrderStatus::PendingCancel
3121 ) && !accepted_during_pending_command
3122 {
3123 self.clear_recon_tracking(&client_order_id, report.order_status.is_closed());
3124 }
3125
3126 self.record_local_activity(client_order_id);
3130 }
3131
3132 pub fn purge_closed_orders(&mut self) {
3134 let Some(buffer_mins) = self.config.purge_closed_orders_buffer_mins else {
3135 return;
3136 };
3137
3138 let ts_now = self.timestamp_ns();
3139 let buffer_secs = mins_to_secs(u64::from(buffer_mins));
3140
3141 self.cache
3142 .borrow_mut()
3143 .purge_closed_orders(ts_now, buffer_secs);
3144 }
3145
3146 pub fn purge_closed_positions(&mut self) {
3148 let Some(buffer_mins) = self.config.purge_closed_positions_buffer_mins else {
3149 return;
3150 };
3151
3152 let ts_now = self.clock.borrow().timestamp_ns();
3153 let buffer_secs = mins_to_secs(u64::from(buffer_mins));
3154
3155 self.cache
3156 .borrow_mut()
3157 .purge_closed_positions(ts_now, buffer_secs);
3158 }
3159
3160 pub fn purge_account_events(&mut self) {
3162 let Some(lookback_mins) = self.config.purge_account_events_lookback_mins else {
3163 return;
3164 };
3165
3166 let ts_now = self.clock.borrow().timestamp_ns();
3167 let lookback_secs = mins_to_secs(u64::from(lookback_mins));
3168
3169 self.cache
3170 .borrow_mut()
3171 .purge_account_events(ts_now, lookback_secs);
3172 }
3173
3174 fn get_order(&self, client_order_id: ClientOrderId) -> Option<OrderAny> {
3175 self.cache().order(&client_order_id).map(|o| o.clone())
3176 }
3177
3178 fn get_order_by_venue_order_id(&self, venue_order_id: VenueOrderId) -> Option<OrderAny> {
3179 let cache = self.cache();
3180 cache
3181 .client_order_id(&venue_order_id)
3182 .and_then(|client_order_id| cache.order(client_order_id).map(|o| o.clone()))
3183 }
3184
3185 fn get_instrument(&self, instrument_id: &InstrumentId) -> Option<InstrumentAny> {
3186 self.cache().instrument(instrument_id).cloned()
3187 }
3188
3189 fn should_skip_order_report(&self, report: &OrderStatusReport) -> bool {
3190 let client_order_id = report.client_order_id.or_else(|| {
3191 self.cache
3192 .borrow()
3193 .client_order_id(&report.venue_order_id)
3194 .copied()
3195 });
3196
3197 if let Some(client_order_id) = client_order_id
3198 && self
3199 .config
3200 .filtered_client_order_ids
3201 .contains(&client_order_id)
3202 {
3203 log::debug!(
3204 "Skipping order report {client_order_id}: in filtered_client_order_ids list"
3205 );
3206 return true;
3207 }
3208
3209 if !self.should_reconcile_instrument(&report.instrument_id) {
3210 log::debug!(
3211 "Skipping order report for {}: not in reconciliation_instrument_ids",
3212 report.instrument_id
3213 );
3214 return true;
3215 }
3216
3217 false
3218 }
3219
3220 pub(crate) fn should_reconcile_instrument(&self, instrument_id: &InstrumentId) -> bool {
3222 self.config.reconciliation_instrument_ids.is_empty()
3223 || self
3224 .config
3225 .reconciliation_instrument_ids
3226 .contains(instrument_id)
3227 }
3228
3229 fn prepare_missing_order_query(&mut self, client_order_id: ClientOrderId) -> Option<OrderAny> {
3230 let order = self.get_order(client_order_id)?;
3231
3232 if order.status().is_closed() {
3236 log::debug!(
3237 "Skipping missing-order resolution for {client_order_id}: already {}",
3238 order.status()
3239 );
3240 self.clear_recon_tracking(&client_order_id, true);
3241 return None;
3242 }
3243
3244 if self.order_activity.within(
3248 &client_order_id,
3249 Duration::from(self.config.open_check_threshold_ns),
3250 ) {
3251 return None;
3252 }
3253
3254 let retries = self.order_recon_retries.entry(client_order_id).or_insert(0);
3255 *retries = retries.saturating_add(1);
3256
3257 if *retries < self.config.open_check_missing_retries {
3258 log::debug!(
3259 "Order {} not found at venue, retry {}/{}",
3260 client_order_id,
3261 retries,
3262 self.config.open_check_missing_retries
3263 );
3264 return None;
3265 }
3266
3267 Some(order)
3268 }
3269
3270 fn resolve_missing_order(&mut self, client_order_id: ClientOrderId) -> Vec<OrderEventAny> {
3271 let mut events = Vec::new();
3272
3273 let Some(order) = self.get_order(client_order_id) else {
3274 return events;
3275 };
3276
3277 if order.status().is_closed() {
3278 log::debug!(
3279 "Skipping missing-order resolution for {client_order_id}: already {}",
3280 order.status()
3281 );
3282 self.clear_recon_tracking(&client_order_id, true);
3283 return events;
3284 }
3285
3286 if self.order_activity.within(
3287 &client_order_id,
3288 Duration::from(self.config.open_check_threshold_ns),
3289 ) {
3290 log::debug!(
3291 "Deferring missing-order resolution for {client_order_id}: recent local activity"
3292 );
3293 return events;
3294 }
3295
3296 let retries = self
3297 .order_recon_retries
3298 .get(&client_order_id)
3299 .copied()
3300 .unwrap_or_default();
3301 let ts_now = self.clock.borrow().timestamp_ns();
3302
3303 match order.status() {
3304 OrderStatus::Accepted | OrderStatus::Submitted => {
3305 log::warn!(
3306 "Order {client_order_id} not found at venue after {retries} retries and a targeted query, marking as REJECTED"
3307 );
3308
3309 if let Some(rejected) =
3310 create_reconciliation_rejected(&order, Some("NOT_FOUND_AT_VENUE"), ts_now)
3311 {
3312 events.push(rejected);
3313 }
3314 }
3315 OrderStatus::PartiallyFilled => {
3316 log::warn!(
3317 "Order {client_order_id} not found at venue after {retries} retries and a targeted query, marking as CANCELED"
3318 );
3319 events.push(OrderEventAny::Canceled(OrderCanceled::new(
3320 order.trader_id(),
3321 order.strategy_id(),
3322 order.instrument_id(),
3323 client_order_id,
3324 UUID4::new(),
3325 ts_now,
3326 ts_now,
3327 true,
3328 order.venue_order_id(),
3329 order.account_id(),
3330 None,
3331 )));
3332 }
3333 OrderStatus::PendingUpdate | OrderStatus::PendingCancel => {
3334 log::debug!(
3335 "Deferring resolution for {client_order_id}: still inflight as {}",
3336 order.status()
3337 );
3338 self.order_recon_retries.shift_remove(&client_order_id);
3347 if let Some(check) = self.order_inflight_checks.get_mut(&client_order_id) {
3348 check.retry_count = 0;
3349 check.last_query_at = Some(dst::time::Instant::now());
3350 }
3351
3352 self.order_query_recency.mark(client_order_id);
3353 return events;
3354 }
3355 status => {
3356 log::warn!(
3357 "Skipping missing-order resolution for {client_order_id}: unexpected status {status}"
3358 );
3359 }
3360 }
3361
3362 self.clear_recon_tracking(&client_order_id, true);
3363 events
3364 }
3365
3366 pub(crate) fn position_quantity_comparison(
3368 &self,
3369 key: InstrumentAccountKey,
3370 venue_reports: &[PositionStatusReport],
3371 ) -> PositionQuantityComparison {
3372 let (instrument_id, account_id) = key;
3373
3374 let cached_positions = {
3375 let cache = self.cache.borrow();
3376 cache
3377 .positions_open(None, Some(&instrument_id), None, Some(&account_id), None)
3378 .into_iter()
3379 .map(|position| (*position).clone())
3380 .collect::<Vec<_>>()
3381 };
3382
3383 let (cached_signed_qty, cached_long_qty, cached_short_qty) =
3384 position_qty_aggregates(cached_positions.iter().map(Position::signed_decimal_qty));
3385 let (venue_signed_qty, venue_long_qty, venue_short_qty) =
3386 position_qty_aggregates(venue_reports.iter().map(|report| report.signed_decimal_qty));
3387 let nonflat_count = venue_reports
3388 .iter()
3389 .filter(|report| report.signed_decimal_qty != Decimal::ZERO)
3390 .count();
3391 let venue_report = venue_reports
3392 .iter()
3393 .find(|report| report.signed_decimal_qty != Decimal::ZERO)
3394 .or_else(|| venue_reports.last())
3395 .cloned();
3396 let venue_has_side_reports = venue_reports.iter().any(PositionStatusReport::is_long)
3397 && venue_reports.iter().any(PositionStatusReport::is_short);
3398
3399 PositionQuantityComparison {
3400 cached_positions,
3401 cached_signed_qty,
3402 cached_long_qty,
3403 cached_short_qty,
3404 venue_signed_qty,
3405 venue_long_qty,
3406 venue_short_qty,
3407 nonflat_count,
3408 venue_report,
3409 venue_has_side_reports,
3410 }
3411 }
3412
3413 fn check_position_discrepancy(
3414 &mut self,
3415 key: InstrumentAccountKey,
3416 venue_reports: &[PositionStatusReport],
3417 ) -> Option<Vec<OrderEventAny>> {
3418 let (instrument_id, account_id) = key;
3419 let comparison = self.position_quantity_comparison(key, venue_reports);
3420 let tolerance = self.position_reconciliation_tolerance(account_id);
3421 let quantities_match = comparison.quantities_match(tolerance);
3422 let report_shape = comparison.report_shape();
3423 let PositionQuantityComparison {
3424 cached_positions,
3425 cached_signed_qty,
3426 cached_long_qty,
3427 cached_short_qty,
3428 venue_signed_qty,
3429 venue_long_qty,
3430 venue_short_qty,
3431 venue_report,
3432 ..
3433 } = comparison;
3434
3435 if quantities_match {
3436 self.clear_position_reconciliation(&key);
3437 return None;
3438 }
3439
3440 if !self.config.generate_missing_orders {
3441 log::debug!(
3442 "Discrepancy for {instrument_id} position when `generate_missing_orders` disabled, skipping"
3443 );
3444 return None;
3445 }
3446
3447 let ts_now = self.clock.borrow().timestamp_ns();
3448
3449 if self.position_activity_is_recent(&key) {
3451 log::debug!(
3452 "Skipping position reconciliation for {instrument_id}: recent activity within threshold"
3453 );
3454 return None;
3455 }
3456
3457 let retries = self.position_reconciliation_retries(&key, report_shape);
3458
3459 if retries >= self.config.position_check_retries {
3460 return None;
3461 }
3462
3463 if report_shape == PositionReportShape::MultiLeg {
3464 let new_retries = retries + 1;
3465 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3466 log::warn!(
3467 "Deferring position reconciliation for {instrument_id}/{account_id}: venue reports have ambiguous side aggregates (cached net={cached_signed_qty}, long={cached_long_qty}, short={cached_short_qty}; venue net={venue_signed_qty}, long={venue_long_qty}, short={venue_short_qty})"
3468 );
3469
3470 if new_retries >= self.config.position_check_retries {
3471 log::error!(
3472 "Position discrepancy for {instrument_id}/{account_id} unresolved after {} attempts; no further reconciliation attempts will be made for the current report shape",
3473 self.config.position_check_retries,
3474 );
3475 }
3476
3477 return None;
3478 }
3479
3480 log::warn!(
3481 "Position discrepancy detected for {instrument_id}: cached_signed_qty={cached_signed_qty}, venue_signed_qty={venue_signed_qty}"
3482 );
3483
3484 let Some(instrument) = self.cache.borrow().instrument(&instrument_id).cloned() else {
3485 log::debug!("Cannot reconcile position for {instrument_id}: instrument not in cache");
3486 let new_retries = retries + 1;
3487 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3488 if new_retries >= self.config.position_check_retries {
3489 log::error!(
3490 "Position discrepancy for {instrument_id} unresolved after {} attempts \
3491 (cached_qty={cached_signed_qty}, venue_qty={venue_signed_qty}); \
3492 no further reconciliation attempts will be made for the current report shape",
3493 self.config.position_check_retries,
3494 );
3495 }
3496
3497 return None;
3498 };
3499
3500 let cached_avg_px = position_avg_px(&cached_positions);
3501 let venue_avg_px = venue_report.as_ref().and_then(|r| r.avg_px_open);
3502
3503 let crosses_zero = (cached_signed_qty > Decimal::ZERO && venue_signed_qty < Decimal::ZERO)
3504 || (cached_signed_qty < Decimal::ZERO && venue_signed_qty > Decimal::ZERO);
3505
3506 let result = if crosses_zero {
3507 let venue_ts_last = venue_report.as_ref().map_or(ts_now, |r| r.ts_last);
3508 let venue_position_id = venue_report
3509 .as_ref()
3510 .and_then(|report| report.venue_position_id);
3511
3512 let position_ids = match venue_position_id {
3513 Some(open_position_id) => match cached_positions.as_slice() {
3514 [position] => Some((Some(position.id), Some(open_position_id))),
3515 _ => {
3516 log::warn!(
3517 "Deferring hedge cross-zero reconciliation for {instrument_id}/{account_id}: cached and venue position identities are ambiguous"
3518 );
3519 None
3520 }
3521 },
3522 None => Some((None, None)),
3523 };
3524
3525 position_ids.and_then(|(close_position_id, open_position_id)| {
3526 self.reconcile_cross_zero_position(
3527 &instrument,
3528 account_id,
3529 instrument_id,
3530 cached_signed_qty,
3531 cached_avg_px,
3532 venue_signed_qty,
3533 venue_avg_px,
3534 close_position_id,
3535 open_position_id,
3536 ts_now,
3537 venue_ts_last,
3538 )
3539 })
3540 } else {
3541 let qty_diff = venue_signed_qty - cached_signed_qty;
3542
3543 let order_side = if qty_diff > Decimal::ZERO {
3544 OrderSide::Buy
3545 } else {
3546 OrderSide::Sell
3547 };
3548
3549 let reconciliation_px = calculate_reconciliation_price(
3550 cached_signed_qty,
3551 cached_avg_px,
3552 venue_signed_qty,
3553 venue_avg_px,
3554 );
3555
3556 match reconciliation_px.or(venue_avg_px).or(cached_avg_px) {
3557 Some(fill_px) => {
3558 let fill_qty = qty_diff.abs();
3559 let venue_position_id = venue_report
3560 .as_ref()
3561 .and_then(|report| report.venue_position_id);
3562 let venue_ts_last = venue_report.as_ref().map_or(ts_now, |r| r.ts_last);
3563 Quantity::from_decimal_dp(fill_qty, instrument.size_precision())
3564 .ok()
3565 .map(|order_qty| {
3566 let fill_price =
3567 Price::from_decimal_dp(fill_px, instrument.price_precision()).ok();
3568 let venue_order_id = create_position_reconciliation_venue_order_id(
3569 account_id,
3570 instrument_id,
3571 order_side,
3572 OrderType::Market,
3573 order_qty,
3574 fill_price,
3575 venue_position_id,
3576 None,
3577 venue_ts_last,
3578 );
3579
3580 let mut order_report = OrderStatusReport::new(
3581 account_id,
3582 instrument_id,
3583 None,
3584 venue_order_id,
3585 order_side.into(),
3586 OrderType::Market,
3587 TimeInForce::Gtc,
3588 OrderStatus::Filled,
3589 order_qty,
3590 order_qty,
3591 ts_now,
3592 ts_now,
3593 ts_now,
3594 None,
3595 )
3596 .with_avg_px(fill_px);
3597
3598 if let Some(venue_position_id) = venue_position_id {
3599 order_report =
3600 order_report.with_venue_position_id(venue_position_id);
3601 }
3602
3603 order_report
3604 })
3605 .map(|order_report| {
3606 log::info!(
3607 color = LogColor::Blue as u8;
3608 "Generating synthetic fill for position reconciliation {instrument_id}: side={order_side:?}, qty={}, px={fill_px}", qty_diff.abs(),
3609 );
3610
3611 let (events, _) = self.handle_external_order(
3612 &order_report,
3613 account_id,
3614 &instrument,
3615 &[],
3616 true,
3617 None,
3618 None,
3619 );
3620 events
3621 })
3622 }
3623 None => None,
3624 }
3625 };
3626
3627 if result.is_none() || result.as_ref().is_some_and(Vec::is_empty) {
3629 let new_retries = retries + 1;
3630 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3631 if new_retries >= self.config.position_check_retries {
3632 log::error!(
3633 "Position discrepancy for {} unresolved after {} attempts \
3634 (cached_qty={}, venue_qty={}); \
3635 no further reconciliation attempts will be made for the current report shape",
3636 instrument_id,
3637 self.config.position_check_retries,
3638 cached_signed_qty,
3639 venue_signed_qty,
3640 );
3641 }
3642 } else {
3643 self.clear_position_reconciliation(&key);
3644 }
3645
3646 result
3647 }
3648
3649 #[expect(clippy::too_many_arguments)]
3652 fn reconcile_cross_zero_position(
3653 &self,
3654 instrument: &InstrumentAny,
3655 account_id: AccountId,
3656 instrument_id: InstrumentId,
3657 cached_signed_qty: Decimal,
3658 cached_avg_px: Option<Decimal>,
3659 venue_signed_qty: Decimal,
3660 venue_avg_px: Option<Decimal>,
3661 close_position_id: Option<PositionId>,
3662 open_position_id: Option<PositionId>,
3663 ts_now: UnixNanos,
3664 venue_ts_last: UnixNanos,
3665 ) -> Option<Vec<OrderEventAny>> {
3666 log::info!(
3667 color = LogColor::Blue as u8;
3668 "Position crosses zero for {instrument_id}: cached={cached_signed_qty}, venue={venue_signed_qty}. Splitting into two fills",
3669 );
3670
3671 let close_qty = cached_signed_qty.abs();
3672
3673 let close_side = if cached_signed_qty < Decimal::ZERO {
3674 OrderSide::Buy } else {
3676 OrderSide::Sell };
3678
3679 let open_qty = venue_signed_qty.abs();
3680
3681 let open_side = if venue_signed_qty > Decimal::ZERO {
3682 OrderSide::Buy } else {
3684 OrderSide::Sell };
3686
3687 let Some(close_px) = cached_avg_px else {
3688 log::warn!("Cannot close position for {instrument_id}: no cached average price");
3689 return None;
3690 };
3691
3692 let open_report = match venue_avg_px {
3693 Some(open_px) => Some((
3694 create_cross_zero_leg_report(
3695 instrument,
3696 account_id,
3697 instrument_id,
3698 open_side,
3699 open_qty,
3700 open_px,
3701 open_position_id,
3702 "OPEN",
3703 ts_now,
3704 venue_ts_last,
3705 )?,
3706 open_px,
3707 )),
3708 None => None,
3709 };
3710
3711 let close_report = create_cross_zero_leg_report(
3712 instrument,
3713 account_id,
3714 instrument_id,
3715 close_side,
3716 close_qty,
3717 close_px,
3718 close_position_id,
3719 "CLOSE",
3720 ts_now,
3721 venue_ts_last,
3722 )?;
3723
3724 log::info!(
3725 color = LogColor::Blue as u8;
3726 "Generating close fill for cross-zero {instrument_id}: side={close_side:?}, qty={close_qty}, px={close_px}",
3727 );
3728
3729 let (close_events, _) = self.handle_external_order(
3730 &close_report,
3731 account_id,
3732 instrument,
3733 &[],
3734 true,
3735 None,
3736 None,
3737 );
3738 let mut all_events = close_events;
3739
3740 if let Some((open_report, open_px)) = open_report {
3741 log::info!(
3742 color = LogColor::Blue as u8;
3743 "Generating open fill for cross-zero {instrument_id}: side={open_side:?}, qty={open_qty}, px={open_px}",
3744 );
3745
3746 let (open_events, _) = self.handle_external_order(
3747 &open_report,
3748 account_id,
3749 instrument,
3750 &[],
3751 true,
3752 None,
3753 None,
3754 );
3755 all_events.extend(open_events);
3756 } else {
3757 log::warn!("Cannot open new position for {instrument_id}: no venue average price");
3758 }
3759
3760 Some(all_events)
3761 }
3762
3763 fn create_position_from_report(
3768 &self,
3769 report: &PositionStatusReport,
3770 account_id: AccountId,
3771 instrument: &InstrumentAny,
3772 ) -> Option<Vec<OrderEventAny>> {
3773 let instrument_id = report.instrument_id;
3774 let venue_signed_qty = report.signed_decimal_qty;
3775
3776 if venue_signed_qty == Decimal::ZERO {
3777 return None;
3778 }
3779
3780 let order_side = if venue_signed_qty > Decimal::ZERO {
3781 OrderSide::Buy
3782 } else {
3783 OrderSide::Sell
3784 };
3785
3786 let qty_abs = venue_signed_qty.abs();
3787 let venue_avg_px = report.avg_px_open?;
3788
3789 let ts_now = self.clock.borrow().timestamp_ns();
3790 let order_qty = Quantity::from_decimal_dp(qty_abs, instrument.size_precision()).ok()?;
3791 let fill_price = Price::from_decimal_dp(venue_avg_px, instrument.price_precision()).ok();
3792 let venue_order_id = create_position_reconciliation_venue_order_id(
3793 account_id,
3794 instrument_id,
3795 order_side,
3796 OrderType::Market,
3797 order_qty,
3798 fill_price,
3799 report.venue_position_id,
3800 None,
3801 report.ts_last,
3802 );
3803
3804 let mut order_report = OrderStatusReport::new(
3805 account_id,
3806 instrument_id,
3807 None,
3808 venue_order_id,
3809 order_side.into(),
3810 OrderType::Market,
3811 TimeInForce::Gtc,
3812 OrderStatus::Filled,
3813 order_qty,
3814 order_qty,
3815 ts_now,
3816 ts_now,
3817 ts_now,
3818 None,
3819 )
3820 .with_avg_px(venue_avg_px);
3821
3822 if let Some(venue_position_id) = report.venue_position_id {
3824 order_report = order_report.with_venue_position_id(venue_position_id);
3825 }
3826
3827 log::info!(
3828 color = LogColor::Blue as u8;
3829 "Creating position from venue report for {instrument_id}: side={order_side:?}, qty={qty_abs}, avg_px={venue_avg_px}",
3830 );
3831
3832 let (events, _) = self.handle_external_order(
3833 &order_report,
3834 account_id,
3835 instrument,
3836 &[],
3837 true,
3838 None,
3839 None,
3840 );
3841 Some(events)
3842 }
3843
3844 fn reconcile_position_report(
3845 &self,
3846 report: &PositionStatusReport,
3847 account_id: AccountId,
3848 instruments_with_unattributed_fills: &IndexSet<InstrumentId>,
3849 ) -> Option<Vec<OrderEventAny>> {
3850 if report.venue_position_id.is_some() {
3851 self.reconcile_position_report_hedging(
3852 report,
3853 account_id,
3854 instruments_with_unattributed_fills,
3855 )
3856 } else {
3857 self.reconcile_position_report_netting(report, account_id)
3858 }
3859 }
3860
3861 fn reconcile_position_report_hedging(
3862 &self,
3863 report: &PositionStatusReport,
3864 account_id: AccountId,
3865 instruments_with_unattributed_fills: &IndexSet<InstrumentId>,
3866 ) -> Option<Vec<OrderEventAny>> {
3867 let venue_position_id = report.venue_position_id?;
3868
3869 if instruments_with_unattributed_fills.contains(&report.instrument_id) {
3872 log::debug!(
3873 "Skipping hedge position {venue_position_id} reconciliation: unattributed fills in batch"
3874 );
3875 return None;
3876 }
3877
3878 log::debug!(
3879 "Reconciling HEDGE position for {}, venue_position_id={}",
3880 report.instrument_id,
3881 venue_position_id
3882 );
3883
3884 let position = {
3885 let cache = self.cache.borrow();
3886 cache.position_owned(&venue_position_id)
3887 };
3888
3889 match position {
3890 Some(position) => {
3891 let cached_signed_qty = position.signed_decimal_qty();
3892 let venue_signed_qty = report.signed_decimal_qty;
3893
3894 if cached_signed_qty == venue_signed_qty {
3895 log::debug!(
3896 "Hedge position {venue_position_id} matches venue: qty={cached_signed_qty}"
3897 );
3898 return None;
3899 }
3900
3901 if venue_signed_qty == Decimal::ZERO && cached_signed_qty == Decimal::ZERO {
3902 return None;
3903 }
3904
3905 if !self.config.generate_missing_orders {
3906 log::error!(
3907 "Cannot reconcile {} {}: position net qty {} != reported net qty {} \
3908 and `generate_missing_orders` is disabled",
3909 report.instrument_id,
3910 venue_position_id,
3911 cached_signed_qty,
3912 venue_signed_qty
3913 );
3914 return None;
3915 }
3916
3917 self.reconcile_hedge_position_discrepancy(
3918 report,
3919 account_id,
3920 &position,
3921 cached_signed_qty,
3922 )
3923 }
3924 None => {
3925 if report.signed_decimal_qty == Decimal::ZERO {
3926 return None;
3927 }
3928
3929 if !self.config.generate_missing_orders {
3930 log::error!(
3931 "Cannot reconcile position: {venue_position_id} not found and `generate_missing_orders` is disabled"
3932 );
3933 return None;
3934 }
3935
3936 self.reconcile_missing_hedge_position(report, account_id)
3937 }
3938 }
3939 }
3940
3941 fn reconcile_hedge_position_discrepancy(
3942 &self,
3943 report: &PositionStatusReport,
3944 account_id: AccountId,
3945 position: &Position,
3946 cached_signed_qty: Decimal,
3947 ) -> Option<Vec<OrderEventAny>> {
3948 let instrument = self.get_instrument(&report.instrument_id)?;
3949 let venue_signed_qty = report.signed_decimal_qty;
3950
3951 let diff = (cached_signed_qty - venue_signed_qty).abs();
3952 let diff_qty = Quantity::from_decimal_dp(diff, instrument.size_precision()).ok()?;
3953
3954 if diff_qty.is_zero() {
3955 log::debug!(
3956 "Difference quantity rounds to zero for {}, skipping",
3957 instrument.id()
3958 );
3959 return None;
3960 }
3961
3962 let venue_position_id = report.venue_position_id?;
3963 log::warn!(
3964 "Hedge position discrepancy for {} {}: cached={}, venue={}, generating reconciliation order",
3965 report.instrument_id,
3966 venue_position_id,
3967 cached_signed_qty,
3968 venue_signed_qty
3969 );
3970
3971 let current_avg_px = if position.avg_px_open > 0.0 {
3972 Decimal::from_str(&position.avg_px_open.to_string()).ok()
3973 } else {
3974 None
3975 };
3976
3977 self.create_position_reconciliation_order(
3978 report,
3979 account_id,
3980 &instrument,
3981 cached_signed_qty,
3982 diff_qty,
3983 current_avg_px,
3984 )
3985 }
3986
3987 fn reconcile_missing_hedge_position(
3988 &self,
3989 report: &PositionStatusReport,
3990 account_id: AccountId,
3991 ) -> Option<Vec<OrderEventAny>> {
3992 let instrument = self.get_instrument(&report.instrument_id)?;
3993 let venue_signed_qty = report.signed_decimal_qty;
3994
3995 let qty = venue_signed_qty.abs();
3996 let diff_qty = Quantity::from_decimal_dp(qty, instrument.size_precision()).ok()?;
3997
3998 if diff_qty.is_zero() {
3999 return None;
4000 }
4001
4002 let venue_position_id = report.venue_position_id?;
4003 log::warn!(
4004 "Missing hedge position for {} {}: venue reports {}, generating reconciliation order",
4005 report.instrument_id,
4006 venue_position_id,
4007 venue_signed_qty
4008 );
4009
4010 self.create_position_reconciliation_order(
4011 report,
4012 account_id,
4013 &instrument,
4014 Decimal::ZERO,
4015 diff_qty,
4016 None,
4017 )
4018 }
4019
4020 fn reconcile_position_report_netting(
4021 &self,
4022 report: &PositionStatusReport,
4023 account_id: AccountId,
4024 ) -> Option<Vec<OrderEventAny>> {
4025 let instrument_id = report.instrument_id;
4026
4027 log::debug!("Reconciling NET position for {instrument_id}");
4028
4029 let instrument = self.get_instrument(&instrument_id)?;
4030
4031 let (cached_signed_qty, cached_avg_px) = {
4032 let cache = self.cache.borrow();
4033 let positions =
4034 cache.positions_open(None, Some(&instrument_id), None, Some(&account_id), None);
4035
4036 if positions.is_empty() {
4037 (Decimal::ZERO, None)
4038 } else {
4039 let mut total_signed_qty = Decimal::ZERO;
4040 let mut total_value = Decimal::ZERO;
4041 let mut total_qty = Decimal::ZERO;
4042
4043 for pos in positions {
4044 total_signed_qty += pos.signed_decimal_qty();
4045 let qty = pos.signed_decimal_qty().abs();
4046 if pos.avg_px_open > 0.0
4047 && qty > Decimal::ZERO
4048 && let Ok(avg_px) = Decimal::from_str(&pos.avg_px_open.to_string())
4049 {
4050 total_value += avg_px * qty;
4051 total_qty += qty;
4052 }
4053 }
4054
4055 let avg_px = if total_qty > Decimal::ZERO {
4056 Some(total_value / total_qty)
4057 } else {
4058 None
4059 };
4060
4061 (total_signed_qty, avg_px)
4062 }
4063 };
4064
4065 let venue_signed_qty = report.signed_decimal_qty;
4066
4067 log::debug!("venue_signed_qty={venue_signed_qty}, cached_signed_qty={cached_signed_qty}");
4068
4069 let tolerance = self.position_reconciliation_tolerance(account_id);
4070 if (cached_signed_qty - venue_signed_qty).abs() <= tolerance {
4071 log::debug!("Position quantities match for {instrument_id}, no reconciliation needed");
4072 return None;
4073 }
4074
4075 if !self.config.generate_missing_orders {
4076 log::debug!(
4077 "Discrepancy for {instrument_id} position when `generate_missing_orders` disabled, skipping"
4078 );
4079 return None;
4080 }
4081
4082 let diff = (cached_signed_qty - venue_signed_qty).abs();
4083 let diff_qty = Quantity::from_decimal_dp(diff, instrument.size_precision()).ok()?;
4084
4085 if diff_qty.is_zero() {
4086 log::debug!(
4087 "Difference quantity rounds to zero for {instrument_id}, skipping order generation"
4088 );
4089 return None;
4090 }
4091
4092 let crosses_zero = cached_signed_qty != Decimal::ZERO
4093 && venue_signed_qty != Decimal::ZERO
4094 && ((cached_signed_qty > Decimal::ZERO && venue_signed_qty < Decimal::ZERO)
4095 || (cached_signed_qty < Decimal::ZERO && venue_signed_qty > Decimal::ZERO));
4096
4097 if crosses_zero {
4098 let ts_now = self.clock.borrow().timestamp_ns();
4099 return self.reconcile_cross_zero_position(
4100 &instrument,
4101 account_id,
4102 instrument_id,
4103 cached_signed_qty,
4104 cached_avg_px,
4105 venue_signed_qty,
4106 report.avg_px_open,
4107 None,
4108 None,
4109 ts_now,
4110 report.ts_last,
4111 );
4112 }
4113
4114 if cached_signed_qty == Decimal::ZERO {
4115 return self.create_position_from_report(report, account_id, &instrument);
4116 }
4117
4118 self.create_position_reconciliation_order(
4119 report,
4120 account_id,
4121 &instrument,
4122 cached_signed_qty,
4123 diff_qty,
4124 cached_avg_px,
4125 )
4126 }
4127
4128 fn create_position_reconciliation_order(
4129 &self,
4130 report: &PositionStatusReport,
4131 account_id: AccountId,
4132 instrument: &InstrumentAny,
4133 cached_signed_qty: Decimal,
4134 diff_qty: Quantity,
4135 current_avg_px: Option<Decimal>,
4136 ) -> Option<Vec<OrderEventAny>> {
4137 let venue_signed_qty = report.signed_decimal_qty;
4138 let instrument_id = report.instrument_id;
4139
4140 let order_side = if venue_signed_qty > cached_signed_qty {
4141 OrderSide::Buy
4142 } else {
4143 OrderSide::Sell
4144 };
4145
4146 let reconciliation_px = calculate_reconciliation_price(
4147 cached_signed_qty,
4148 current_avg_px,
4149 venue_signed_qty,
4150 report.avg_px_open,
4151 );
4152
4153 let fill_px = reconciliation_px
4154 .or(report.avg_px_open)
4155 .or(current_avg_px)?;
4156
4157 let ts_now = self.clock.borrow().timestamp_ns();
4158 let fill_price = Price::from_decimal_dp(fill_px, instrument.price_precision()).ok();
4159 let venue_order_id = create_position_reconciliation_venue_order_id(
4160 account_id,
4161 instrument_id,
4162 order_side,
4163 OrderType::Market,
4164 diff_qty,
4165 fill_price,
4166 report.venue_position_id,
4167 None,
4168 report.ts_last,
4169 );
4170
4171 let mut order_report = OrderStatusReport::new(
4172 account_id,
4173 instrument_id,
4174 None,
4175 venue_order_id,
4176 order_side.into(),
4177 OrderType::Market,
4178 TimeInForce::Gtc,
4179 OrderStatus::Filled,
4180 diff_qty,
4181 diff_qty,
4182 ts_now,
4183 ts_now,
4184 ts_now,
4185 None,
4186 )
4187 .with_avg_px(fill_px);
4188
4189 if let Some(venue_position_id) = report.venue_position_id {
4190 order_report = order_report.with_venue_position_id(venue_position_id);
4191 }
4192
4193 log::info!(
4194 color = LogColor::Blue as u8;
4195 "Generating reconciliation order for {instrument_id}: side={order_side:?}, qty={diff_qty}, px={fill_px}",
4196 );
4197
4198 let (events, _) = self.handle_external_order(
4199 &order_report,
4200 account_id,
4201 instrument,
4202 &[],
4203 true,
4204 None,
4205 None,
4206 );
4207 Some(events)
4208 }
4209
4210 fn reconcile_order_report(
4211 &self,
4212 order: &OrderAny,
4213 report: &OrderStatusReport,
4214 instrument: Option<&InstrumentAny>,
4215 commission_client: Option<&dyn ExecutionClient>,
4216 ) -> anyhow::Result<Vec<OrderEventAny>> {
4217 let has_missing_fills = terminal_report_has_missing_fills(report, order.filled_qty());
4218 anyhow::ensure!(
4219 !has_missing_fills,
4220 "terminal report for {} has unaccounted fills; waiting for fill reports",
4221 order.client_order_id(),
4222 );
4223 let ts_now = self.clock.borrow().timestamp_ns();
4224
4225 let commission = if matches!(
4226 report.order_status,
4227 OrderStatus::PartiallyFilled | OrderStatus::Filled
4228 ) && report.filled_qty > order.filled_qty()
4229 && let Some(instrument) = instrument
4230 {
4231 let fill_qty = report.filled_qty - order.filled_qty();
4232 let price_and_liquidity =
4233 incremental_inferred_fill_price_and_liquidity(order, report, instrument);
4234
4235 resolve_inferred_fill_commission(
4236 fill_qty,
4237 price_and_liquidity,
4238 instrument,
4239 commission_client,
4240 )?
4241 } else {
4242 None
4243 };
4244
4245 if matches!(
4246 report.order_status,
4247 OrderStatus::Canceled | OrderStatus::Expired
4248 ) && order.status() == OrderStatus::Filled
4249 && report.filled_qty == order.filled_qty()
4250 {
4251 return Ok(Vec::new());
4252 }
4253
4254 Ok(
4255 reconcile_order_report_with_commission(order, report, instrument, ts_now, commission)
4256 .into_iter()
4257 .collect(),
4258 )
4259 }
4260
4261 #[expect(
4263 clippy::too_many_arguments,
4264 reason = "Snapshot and continuous reports share fill projection"
4265 )]
4266 fn reconcile_order_with_fills(
4267 &mut self,
4268 is_snapshot: bool,
4269 order: &OrderAny,
4270 report: &OrderStatusReport,
4271 fills: &[&FillReport],
4272 instrument: Option<&InstrumentAny>,
4273 fill_queue: &mut ReconciliationFillQueue,
4274 commission_client: Option<&dyn ExecutionClient>,
4275 ) -> Vec<OrderEventAny> {
4276 let mut events = Vec::new();
4277 let mut working = order.clone();
4278 let mut sorted_fills: Vec<&FillReport> = fills.to_vec();
4279 sorted_fills.sort_by_key(|f| f.ts_event);
4280
4281 let ts_now = self.clock.borrow().timestamp_ns();
4282
4283 if matches!(
4284 report.order_status,
4285 OrderStatus::Canceled | OrderStatus::Expired
4286 ) && report.ts_triggered.is_some()
4287 && working.status() != OrderStatus::Triggered
4288 && TRIGGERABLE_ORDER_TYPES.contains(&working.order_type())
4289 {
4290 let triggered = create_reconciliation_triggered(&working, report, ts_now);
4291 if working.apply(triggered.clone()).is_ok() {
4292 events.push(triggered);
4293 }
4294 }
4295
4296 let requires_snapshot_projection = !sorted_fills.is_empty()
4297 || is_snapshot
4298 && (report.order_status == OrderStatus::Voided
4299 || report.filled_qty < working.filled_qty());
4300 if !requires_snapshot_projection {
4301 match self.reconcile_order_report(&working, report, instrument, commission_client) {
4302 Ok(order_events) => events.extend(order_events),
4303 Err(e) => log::error!(
4304 "Deferring order reconciliation for {}: {e}",
4305 order.client_order_id(),
4306 ),
4307 }
4308
4309 return events;
4310 }
4311
4312 for event in generate_reconciliation_order_pre_fill_events(&working, report, ts_now) {
4313 if let Err(e) = working.apply(event.clone()) {
4314 log::warn!(
4315 "Cannot project reconciliation event for {}: {e}",
4316 order.client_order_id()
4317 );
4318 return events;
4319 }
4320
4321 events.push(event);
4322 }
4323
4324 if let Some(inst) = instrument {
4325 for fill in sorted_fills {
4326 let Some((event, fill_key)) =
4327 self.create_order_fill(&working, fill, inst, &fill_queue.pending_fill_keys)
4328 else {
4329 continue;
4330 };
4331
4332 if let Err(e) = working.apply(OrderEventAny::Filled(event.clone())) {
4333 if self.is_fill_applied(&event, fill_key) {
4334 self.fills_processed.mark(fill_key);
4335 continue;
4336 }
4337
4338 log::warn!(
4339 "Cannot project reconciliation fill for {}: {e}",
4340 order.client_order_id()
4341 );
4342 return events;
4343 }
4344
4345 fill_queue.push(&mut events, event, fill_key);
4346 }
4347 }
4348
4349 if !is_snapshot {
4351 match self.reconcile_order_report(&working, report, instrument, commission_client) {
4352 Ok(order_events) => events.extend(order_events),
4353 Err(e) => log::warn!("Deferring order reconciliation: {e}"),
4354 }
4355
4356 return events;
4357 }
4358
4359 if terminal_report_has_missing_fills(report, working.filled_qty()) {
4360 log::warn!(
4361 "Deferring terminal reconciliation for {}: fill reports are incomplete",
4362 order.client_order_id(),
4363 );
4364 return events;
4365 }
4366
4367 let commission = if report.filled_qty > working.filled_qty()
4368 && let Some(instrument) = instrument
4369 {
4370 let fill_qty = report.filled_qty - working.filled_qty();
4371
4372 let price_and_liquidity =
4373 incremental_inferred_fill_price_and_liquidity(&working, report, instrument);
4374
4375 match resolve_inferred_fill_commission(
4376 fill_qty,
4377 price_and_liquidity,
4378 instrument,
4379 commission_client,
4380 ) {
4381 Ok(commission) => commission,
4382 Err(e) => {
4383 log::error!(
4384 "Deferring inferred fill for {}: venue commission calculation failed: {e}",
4385 order.client_order_id(),
4386 );
4387 return events;
4388 }
4389 }
4390 } else {
4391 None
4392 };
4393
4394 for event in generate_reconciliation_order_snapshot_events_with_commission(
4395 &working, report, instrument, ts_now, commission,
4396 ) {
4397 if let Err(e) = working.apply(event.clone()) {
4398 log::warn!(
4399 "Cannot project reconciliation snapshot event for {}: {e}",
4400 order.client_order_id()
4401 );
4402 break;
4403 }
4404
4405 events.push(event);
4406 }
4407
4408 events
4409 }
4410
4411 #[expect(clippy::too_many_arguments)]
4412 fn handle_external_order(
4413 &self,
4414 report: &OrderStatusReport,
4415 account_id: AccountId,
4416 instrument: &InstrumentAny,
4417 fills: &[&FillReport],
4418 is_synthetic: bool,
4419 fill_queue: Option<&mut ReconciliationFillQueue>,
4420 commission_client: Option<&dyn ExecutionClient>,
4421 ) -> (Vec<OrderEventAny>, Option<ExternalOrderMetadata>) {
4422 let claimed_strategy = self
4423 .cache
4424 .borrow()
4425 .external_order_claim(&report.instrument_id);
4426
4427 let (strategy_id, tags) = if let Some(claimed_strategy) = claimed_strategy {
4428 let order_id = report
4429 .client_order_id
4430 .map_or_else(|| report.venue_order_id.to_string(), |id| id.to_string());
4431 log::info!(
4432 color = LogColor::Blue as u8;
4433 "External order {} for {} claimed by strategy {}",
4434 order_id,
4435 report.instrument_id,
4436 claimed_strategy,
4437 );
4438 (claimed_strategy, None)
4439 } else {
4440 let tag = if is_synthetic {
4442 *TAG_RECONCILIATION
4443 } else {
4444 *TAG_VENUE
4445 };
4446
4447 (StrategyId::from("EXTERNAL"), Some(vec![tag]))
4448 };
4449
4450 if self.config.filter_unclaimed_external && claimed_strategy.is_none() && !is_synthetic {
4452 return (Vec::new(), None);
4453 }
4454
4455 let client_order_id = report
4456 .client_order_id
4457 .unwrap_or_else(|| ClientOrderId::from(report.venue_order_id.as_str()));
4458
4459 if !report.quantity.is_positive() {
4460 log::error!(
4461 "Skipping external order {} ({}) for {}: non-positive quantity in report {:?}",
4462 client_order_id,
4463 report.venue_order_id,
4464 report.instrument_id,
4465 report,
4466 );
4467 return (Vec::new(), None);
4468 }
4469
4470 let ts_now = self.clock.borrow().timestamp_ns();
4471
4472 let Some(order_side) = report.order_side else {
4473 log::error!(
4474 "Skipping external order {} ({}) for {}: order side is not specified",
4475 client_order_id,
4476 report.venue_order_id,
4477 report.instrument_id,
4478 );
4479 return (Vec::new(), None);
4480 };
4481
4482 let initialized = match OrderInitialized::new_checked(
4483 self.config.trader_id,
4484 strategy_id,
4485 report.instrument_id,
4486 client_order_id,
4487 order_side,
4488 report.order_type,
4489 report.quantity,
4490 report.time_in_force,
4491 report.post_only,
4492 report.reduce_only,
4493 false, true, UUID4::new(),
4496 ts_now,
4497 ts_now,
4498 report.price,
4499 report.activation_price,
4500 report.trigger_price,
4501 report.trigger_type,
4502 report.limit_offset,
4503 report.trailing_offset,
4504 report.trailing_offset_type,
4505 report.expire_time,
4506 report.display_qty,
4507 None, None, report.contingency_type,
4510 report.order_list_id,
4511 report.linked_order_ids.clone(),
4512 report.parent_order_id,
4513 None, None, None, tags,
4517 ) {
4518 Ok(initialized) => initialized,
4519 Err(e) => {
4520 log::error!("Failed to create order from report: {e}");
4521 return (Vec::new(), None);
4522 }
4523 };
4524
4525 let initialized = OrderEventAny::Initialized(initialized);
4526
4527 let order = match OrderAny::from_events(vec![initialized.clone()]) {
4528 Ok(order) => order,
4529 Err(e) => {
4530 log::error!("Failed to create order from report: {e}");
4531 return (Vec::new(), None);
4532 }
4533 };
4534
4535 let replace_inferred_fill = !fills.is_empty()
4536 && matches!(
4537 report.order_status,
4538 OrderStatus::Canceled
4539 | OrderStatus::Expired
4540 | OrderStatus::Filled
4541 | OrderStatus::PartiallyFilled
4542 );
4543 let mut prepared_fills = Vec::new();
4544 let mut prepared_fill_keys = fill_queue
4545 .as_deref()
4546 .map(|queue| queue.pending_fill_keys.clone())
4547 .unwrap_or_default();
4548 let mut real_fill_total = Decimal::ZERO;
4549
4550 if replace_inferred_fill {
4551 let mut sorted_fills: Vec<&FillReport> = fills.to_vec();
4552 sorted_fills.sort_by_key(|fill| fill.ts_event);
4553
4554 if fill_queue.is_none() {
4555 log::error!(
4556 "Cannot reconcile external order {client_order_id}: fill queue is unavailable"
4557 );
4558 return (Vec::new(), None);
4559 }
4560
4561 for fill in sorted_fills {
4562 if let Some((fill_event, fill_key)) =
4563 self.create_order_fill(&order, fill, instrument, &prepared_fill_keys)
4564 {
4565 real_fill_total += fill.last_qty.as_decimal();
4566 prepared_fill_keys.insert(fill_key);
4567 prepared_fills.push((fill_event, fill_key));
4568 }
4569 }
4570 }
4571
4572 let report_filled = report.filled_qty.as_decimal();
4573
4574 let inferred_qty = if report_filled.is_zero() {
4575 None
4576 } else if replace_inferred_fill {
4577 if real_fill_total < report_filled {
4578 match Quantity::from_decimal_dp(
4579 report_filled - real_fill_total,
4580 instrument.size_precision(),
4581 ) {
4582 Ok(quantity) => Some(quantity),
4583 Err(e) => {
4584 log::error!(
4585 "Cannot reconcile external order {client_order_id}: residual fill quantity is invalid: {e}"
4586 );
4587 return (Vec::new(), None);
4588 }
4589 }
4590 } else {
4591 None
4592 }
4593 } else if matches!(
4594 report.order_status,
4595 OrderStatus::PartiallyFilled
4596 | OrderStatus::Filled
4597 | OrderStatus::Canceled
4598 | OrderStatus::Expired
4599 | OrderStatus::Voided
4600 ) {
4601 Some(report.filled_qty)
4602 } else {
4603 None
4604 };
4605
4606 let defer_terminal = !is_synthetic
4607 && claimed_strategy.is_some()
4608 && matches!(
4609 report.order_status,
4610 OrderStatus::Canceled | OrderStatus::Expired
4611 )
4612 && inferred_qty.is_some();
4613
4614 if defer_terminal {
4615 log::warn!(
4616 "Deferring terminal reconciliation for claimed order {client_order_id}: fill reports are incomplete"
4617 );
4618
4619 if prepared_fills.is_empty() {
4620 return (Vec::new(), None);
4621 }
4622 }
4623
4624 let inferred_commission = if is_synthetic || defer_terminal {
4625 None
4626 } else if let Some(inferred_qty) = inferred_qty {
4627 let price_and_liquidity = inferred_fill_price_and_liquidity(&order, report, instrument);
4628
4629 match resolve_inferred_fill_commission(
4630 inferred_qty,
4631 price_and_liquidity,
4632 instrument,
4633 commission_client,
4634 ) {
4635 Ok(commission) => commission,
4636 Err(e) => {
4637 log::error!(
4638 "Deferring external order {client_order_id}: venue commission calculation failed: {e}"
4639 );
4640 return (Vec::new(), None);
4641 }
4642 }
4643 } else {
4644 None
4645 };
4646
4647 {
4648 let mut cache = self.cache.borrow_mut();
4649
4650 let source_client_id = if is_synthetic {
4651 None
4652 } else {
4653 commission_client.map(ExecutionClient::client_id)
4654 };
4655
4656 if let Err(e) = cache.add_order(order.clone(), None, source_client_id, false) {
4657 match cache.order(&client_order_id) {
4661 Some(existing) if is_synthetic && existing.is_closed() => {
4662 log::debug!(
4663 "Skipping synthetic reconciliation order {client_order_id} for {}: \
4664 replay deduped (cached status={:?})",
4665 report.instrument_id,
4666 existing.status(),
4667 );
4668 }
4669 Some(existing) if is_synthetic => {
4670 log::warn!(
4671 "Synthetic reconciliation order {client_order_id} for {} exists in \
4672 cache in non-terminal state {:?}; fill not regenerated",
4673 report.instrument_id,
4674 existing.status(),
4675 );
4676 }
4677 _ => {
4678 log::error!("Failed to add external order to cache: {e}");
4679 }
4680 }
4681
4682 return (Vec::new(), None);
4683 }
4684
4685 if let Err(e) = cache.index_venue_order_id(&client_order_id, &report.venue_order_id) {
4686 log::warn!("Failed to index venue order ID: {e}");
4687 }
4688 }
4689
4690 Self::publish_order_event(&initialized);
4691
4692 log::info!(
4693 color = LogColor::Blue as u8;
4694 "Created external order {} ({}) for {} [{}]",
4695 client_order_id,
4696 report.venue_order_id,
4697 report.instrument_id,
4698 report.order_status,
4699 );
4700
4701 let ts_now = self.clock.borrow().timestamp_ns();
4702 let mut order_events = generate_external_order_status_events_with_commission(
4703 &order,
4704 report,
4705 &account_id,
4706 instrument,
4707 ts_now,
4708 inferred_commission,
4709 );
4710
4711 if replace_inferred_fill {
4712 let terminal_event = if order_events.last().is_some_and(|event| {
4713 matches!(
4714 event,
4715 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_),
4716 )
4717 }) {
4718 order_events.pop()
4719 } else {
4720 None
4721 };
4722
4723 if order_events
4724 .last()
4725 .is_some_and(|event| matches!(event, OrderEventAny::Filled(_)))
4726 {
4727 order_events.pop();
4728 }
4729
4730 let fill_queue =
4731 fill_queue.expect("fill queue availability was checked before cache mutation");
4732 for (fill_event, fill_key) in prepared_fills {
4733 fill_queue.push(&mut order_events, fill_event, fill_key);
4734 }
4735
4736 if !defer_terminal
4737 && let Some(inferred_qty) = inferred_qty
4738 && let Some(inferred_fill) = create_inferred_fill_for_qty(
4739 &order,
4740 report,
4741 &account_id,
4742 instrument,
4743 inferred_qty,
4744 ts_now,
4745 inferred_commission,
4746 )
4747 {
4748 order_events.push(inferred_fill);
4749 }
4750
4751 if !defer_terminal && let Some(event) = terminal_event {
4752 order_events.push(event);
4753 }
4754 }
4755
4756 let metadata = ExternalOrderMetadata {
4757 client_order_id,
4758 venue_order_id: report.venue_order_id,
4759 instrument_id: report.instrument_id,
4760 strategy_id,
4761 ts_init: ts_now,
4762 };
4763
4764 (order_events, Some(metadata))
4765 }
4766
4767 fn publish_order_event(event: &OrderEventAny) {
4768 let topic = switchboard::get_event_order_topic(event.strategy_id());
4769 msgbus::publish_order_event(topic, event);
4770 }
4771
4772 fn adjust_mass_status_fills(
4777 &self,
4778 mass_status: &ExecutionMassStatus,
4779 ) -> (
4780 IndexMap<VenueOrderId, OrderStatusReport>,
4781 IndexMap<VenueOrderId, Vec<FillReport>>,
4782 ) {
4783 let mut final_orders: IndexMap<VenueOrderId, OrderStatusReport> =
4784 mass_status.order_reports();
4785 let mut final_fills: IndexMap<VenueOrderId, Vec<FillReport>> = mass_status.fill_reports();
4786
4787 final_fills.retain(|_, fills| {
4788 fills.retain(|fill| {
4789 if fill.last_qty.is_zero() {
4790 log::warn!("Skipping zero-quantity fill report: {fill}");
4791 return false;
4792 }
4793
4794 true
4795 });
4796
4797 !fills.is_empty()
4798 });
4799
4800 if mass_status.lookback_start().is_some() {
4801 return (final_orders, final_fills);
4802 }
4803
4804 let mut instruments_to_adjust = Vec::new();
4805
4806 for (instrument_id, position_reports) in mass_status.position_reports() {
4807 if !self.should_reconcile_instrument(&instrument_id) {
4808 log::debug!(
4809 "Skipping fill adjustment for {instrument_id}: not in reconciliation_instrument_ids"
4810 );
4811 continue;
4812 }
4813
4814 let is_hedge_mode = position_reports
4817 .iter()
4818 .any(|r| r.venue_position_id.is_some());
4819
4820 if is_hedge_mode {
4821 log::debug!(
4822 "Skipping fill adjustment for {instrument_id}: hedge mode (has venue_position_id)"
4823 );
4824 continue;
4825 }
4826
4827 let has_retained_position = {
4828 let cache = self.cache.borrow();
4829 !cache
4830 .positions_open(
4831 None,
4832 Some(&instrument_id),
4833 None,
4834 Some(&mass_status.account_id),
4835 None,
4836 )
4837 .is_empty()
4838 };
4839
4840 if has_retained_position {
4841 log::debug!(
4842 "Skipping fill adjustment for {instrument_id}: retained open position in cache"
4843 );
4844 continue;
4845 }
4846
4847 if let Some(instrument) = self.get_instrument(&instrument_id) {
4848 instruments_to_adjust.push(instrument);
4849 } else {
4850 log::debug!(
4851 "Skipping fill adjustment for {instrument_id}: instrument not found in cache"
4852 );
4853 }
4854 }
4855
4856 if instruments_to_adjust.is_empty() {
4857 return (final_orders, final_fills);
4858 }
4859
4860 log_info!(
4861 "Adjusting fills for {} instrument(s) with position reports",
4862 instruments_to_adjust.len(),
4863 color = LogColor::Blue
4864 );
4865
4866 for instrument in &instruments_to_adjust {
4867 let instrument_id = instrument.id();
4868
4869 let result = if self.config.generate_missing_orders {
4870 process_mass_status_for_reconciliation(mass_status, instrument, None)
4871 } else {
4872 process_mass_status_for_reconciliation_without_synthetic_reports(
4873 mass_status,
4874 instrument,
4875 None,
4876 )
4877 };
4878
4879 match result {
4880 Ok(result) => {
4881 final_orders.retain(|_, order| order.instrument_id != instrument_id);
4882 final_fills.retain(|_, fills| {
4883 fills
4884 .first()
4885 .is_none_or(|f| f.instrument_id != instrument_id)
4886 });
4887
4888 for (venue_order_id, order) in result.orders {
4889 final_orders.insert(venue_order_id, order);
4890 }
4891
4892 for (venue_order_id, fills) in result.fills {
4893 final_fills.insert(venue_order_id, fills);
4894 }
4895 }
4896 Err(e) => {
4897 log::warn!("Failed to adjust fills for {instrument_id}: {e}");
4898 }
4899 }
4900 }
4901
4902 log_info!(
4903 "After adjustment: {} order(s), {} fill group(s)",
4904 final_orders.len(),
4905 final_fills.len(),
4906 color = LogColor::Blue
4907 );
4908
4909 (final_orders, final_fills)
4910 }
4911
4912 fn is_fill_applied(&self, fill: &OrderFilled, fill_key: FillKey) -> bool {
4913 if fill.last_qty.is_zero() {
4914 return false;
4915 }
4916
4917 self.get_order(fill.client_order_id)
4918 .or_else(|| self.get_order_by_venue_order_id(fill.venue_order_id))
4919 .is_some_and(|order| {
4920 order.account_id() == Some(fill_key.0)
4921 && order.instrument_id() == fill_key.1
4922 && order.trade_ids().contains(&&fill_key.2)
4923 })
4924 }
4925
4926 fn create_order_fill(
4927 &self,
4928 order: &OrderAny,
4929 fill: &FillReport,
4930 instrument: &InstrumentAny,
4931 pending_fill_keys: &IndexSet<FillKey>,
4932 ) -> Option<(OrderFilled, FillKey)> {
4933 if fill.last_qty.is_zero() {
4934 log::warn!("Skipping zero-quantity fill report: {fill}");
4935 return None;
4936 }
4937
4938 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
4939 if self.fills_processed.contains_key(&fill_key) || pending_fill_keys.contains(&fill_key) {
4940 return None;
4941 }
4942
4943 let order_side = order.order_side();
4944 if fill.order_side != order_side {
4945 log::warn!(
4946 "Fill side mismatch for {}: cached={:?}, venue={:?}",
4947 order.client_order_id(),
4948 order_side,
4949 fill.order_side,
4950 );
4951 }
4952
4953 let ts_now = self.clock.borrow().timestamp_ns();
4954
4955 let event = OrderFilled::new(
4956 order.trader_id(),
4957 order.strategy_id(),
4958 order.instrument_id(),
4959 order.client_order_id(),
4960 fill.venue_order_id,
4961 fill.account_id,
4962 fill.trade_id,
4963 fill.order_side,
4964 order.order_type(),
4965 fill.last_qty,
4966 fill.last_px,
4967 instrument.quote_currency(),
4968 fill.liquidity_side,
4969 fill.report_id,
4970 fill.ts_event,
4971 ts_now,
4972 true, fill.venue_position_id,
4974 Some(fill.commission),
4975 None,
4976 );
4977
4978 Some((event, fill_key))
4979 }
4980}
4981
4982#[cfg(test)]
4983mod tests {
4984 use nautilus_common::{clock::VirtualClock, config::ConfigError};
4985 use nautilus_core::{DurationNanos, Params};
4986 use nautilus_execution::reconciliation::generate_reconciliation_order_events;
4987 use nautilus_model::{
4988 accounts::AccountAny,
4989 enums::{LiquiditySide, OmsType, PositionSide},
4990 events::order::spec::{OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec},
4991 identifiers::{Symbol, Venue},
4992 instruments::{
4993 CurrencyPair, Instrument,
4994 stubs::{crypto_perpetual_ethusdt, xbtusd_bitmex},
4995 },
4996 orders::{OrderTestBuilder, stubs::TestOrderEventStubs},
4997 types::{AccountBalance, Currency, MarginBalance, Money},
4998 };
4999 use rstest::rstest;
5000 use rust_decimal_macros::dec;
5001
5002 use super::{
5003 super::reconciliation::tests::{CommissionOutcome, CommissionStubClient},
5004 *,
5005 };
5006
5007 #[rstest]
5008 fn test_new_validates_open_check_lookback_mins_boundaries() {
5009 let create_manager = |mins| {
5010 ExecutionManager::new(
5011 Rc::new(RefCell::new(VirtualClock::new())),
5012 Rc::new(RefCell::new(Cache::default())),
5013 ExecutionManagerConfig {
5014 open_check_lookback_mins: Some(mins),
5015 ..Default::default()
5016 },
5017 )
5018 };
5019
5020 assert!(create_manager(307_445_734).is_ok());
5021 assert!(matches!(
5022 create_manager(307_445_735),
5023 Err(ConfigError::Range { field, .. })
5024 if field == "ExecutionManagerConfig.open_check_lookback_mins"
5025 ));
5026 }
5027
5028 #[rstest]
5029 fn test_new_validates_reconciliation_lookback_mins_boundaries() {
5030 let create_manager = |mins| {
5031 ExecutionManager::new(
5032 Rc::new(RefCell::new(VirtualClock::new())),
5033 Rc::new(RefCell::new(Cache::default())),
5034 ExecutionManagerConfig {
5035 lookback_mins: Some(mins),
5036 ..Default::default()
5037 },
5038 )
5039 };
5040
5041 assert!(create_manager(307_445_734_561_825_860).is_ok());
5042 assert!(matches!(
5043 create_manager(307_445_734_561_825_861),
5044 Err(ConfigError::Range { field, .. })
5045 if field == "ExecutionManagerConfig.lookback_mins"
5046 ));
5047 }
5048
5049 #[rstest]
5050 fn test_new_reports_every_invalid_lookback_field() {
5051 let error = ExecutionManager::new(
5052 Rc::new(RefCell::new(VirtualClock::new())),
5053 Rc::new(RefCell::new(Cache::default())),
5054 ExecutionManagerConfig {
5055 lookback_mins: Some(307_445_734_561_825_861),
5056 open_check_lookback_mins: Some(307_445_735),
5057 position_check_lookback_mins: 307_445_735,
5058 ..Default::default()
5059 },
5060 )
5061 .expect_err("all lookback fields are out of range");
5062
5063 let ConfigError::Multiple { errors } = error else {
5064 panic!("expected a `Multiple` error, was {error:?}");
5065 };
5066
5067 let fields = errors
5068 .iter()
5069 .map(|e| match e {
5070 ConfigError::Range { field, .. } => field.as_str(),
5071 other => panic!("expected a `Range` error, was {other:?}"),
5072 })
5073 .collect::<Vec<_>>();
5074
5075 assert_eq!(
5076 fields,
5077 [
5078 "ExecutionManagerConfig.lookback_mins",
5079 "ExecutionManagerConfig.open_check_lookback_mins",
5080 "ExecutionManagerConfig.position_check_lookback_mins",
5081 ]
5082 );
5083 }
5084
5085 struct PositionCoverageStubClient;
5086
5087 #[async_trait::async_trait(?Send)]
5088 impl ExecutionClient for PositionCoverageStubClient {
5089 fn is_connected(&self) -> bool {
5090 true
5091 }
5092
5093 fn client_id(&self) -> ClientId {
5094 ClientId::from("BYBIT")
5095 }
5096
5097 fn account_id(&self) -> AccountId {
5098 AccountId::from("TEST-001")
5099 }
5100
5101 fn venue(&self) -> Venue {
5102 Venue::from("BYBIT")
5103 }
5104
5105 fn oms_type(&self) -> OmsType {
5106 OmsType::Netting
5107 }
5108
5109 fn get_account(&self) -> Option<AccountAny> {
5110 None
5111 }
5112
5113 fn provides_bulk_position_coverage(&self, instrument_id: InstrumentId) -> bool {
5114 !instrument_id.symbol.as_str().ends_with("-SPOT")
5115 }
5116
5117 fn generate_account_state(
5118 &self,
5119 _balances: Vec<AccountBalance>,
5120 _margins: Vec<MarginBalance>,
5121 _reported: bool,
5122 _ts_event: UnixNanos,
5123 _info: Option<Params>,
5124 ) -> anyhow::Result<()> {
5125 Ok(())
5126 }
5127
5128 fn start(&mut self) -> anyhow::Result<()> {
5129 Ok(())
5130 }
5131
5132 fn stop(&mut self) -> anyhow::Result<()> {
5133 Ok(())
5134 }
5135 }
5136
5137 fn cached_commission_fixtures() -> (
5138 ExecutionManager,
5139 Rc<RefCell<Cache>>,
5140 OrderAny,
5141 OrderStatusReport,
5142 InstrumentAny,
5143 ) {
5144 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5145 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5146 let cache = Rc::new(RefCell::new(Cache::default()));
5147 cache
5148 .borrow_mut()
5149 .add_instrument(instrument.clone())
5150 .expect("instrument is cacheable");
5151 let client_order_id = ClientOrderId::from("O-COMMISSION-CACHED");
5152 let venue_order_id = VenueOrderId::from("V-COMMISSION-CACHED");
5153 insert_accepted_limit_order(
5154 &cache,
5155 client_order_id,
5156 venue_order_id,
5157 instrument.id(),
5158 ClientId::from("STUB"),
5159 );
5160 let order = cache
5161 .borrow()
5162 .order_owned(&client_order_id)
5163 .expect("accepted order is cached");
5164 let report = OrderStatusReport::new(
5165 AccountId::from("TEST-001"),
5166 instrument.id(),
5167 Some(client_order_id),
5168 venue_order_id,
5169 OrderSide::Buy.into(),
5170 OrderType::Limit,
5171 TimeInForce::Gtc,
5172 OrderStatus::Filled,
5173 Quantity::from("10.0"),
5174 Quantity::from("10.0"),
5175 UnixNanos::from(1),
5176 UnixNanos::from(1),
5177 UnixNanos::from(1),
5178 None,
5179 )
5180 .with_avg_px(dec!(100.0));
5181 let manager =
5182 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5183 .expect("valid config");
5184
5185 (manager, cache, order, report, instrument)
5186 }
5187
5188 fn external_report_with_partial_fill(
5189 instrument: &InstrumentAny,
5190 ) -> (OrderStatusReport, FillReport) {
5191 let account_id = AccountId::from("STUB-001");
5192 let venue_order_id = VenueOrderId::from("V-EXT-1");
5193 let report = OrderStatusReport::new(
5194 account_id,
5195 instrument.id(),
5196 None,
5197 venue_order_id,
5198 OrderSide::Buy.into(),
5199 OrderType::Limit,
5200 TimeInForce::Gtc,
5201 OrderStatus::Filled,
5202 Quantity::from("10.0"),
5203 Quantity::from("10.0"),
5204 UnixNanos::from(1),
5205 UnixNanos::from(1),
5206 UnixNanos::from(1),
5207 None,
5208 )
5209 .with_price(Price::from("100.00"))
5210 .with_avg_px(dec!(100.0));
5211
5212 let fill = FillReport::new(
5213 account_id,
5214 instrument.id(),
5215 venue_order_id,
5216 TradeId::from("T-EXT-1"),
5217 OrderSide::Buy,
5218 Quantity::from("4.0"),
5219 Price::from("100.00"),
5220 Money::new(0.1, Currency::USDT()),
5221 LiquiditySide::Taker,
5222 None,
5223 None,
5224 UnixNanos::from(1),
5225 UnixNanos::from(1),
5226 None,
5227 );
5228
5229 (report, fill)
5230 }
5231
5232 fn inferred_fills(events: &[OrderEventAny]) -> Vec<OrderFilled> {
5233 events
5234 .iter()
5235 .filter_map(|event| match event {
5236 OrderEventAny::Filled(filled) if filled.last_qty == Quantity::from("6.0") => {
5237 Some(filled.clone())
5238 }
5239 _ => None,
5240 })
5241 .collect()
5242 }
5243
5244 #[rstest]
5245 fn test_handle_external_order_applies_venue_commission_to_inferred_fill() {
5246 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5247 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5248 let cache = Rc::new(RefCell::new(Cache::default()));
5249 cache
5250 .borrow_mut()
5251 .add_instrument(instrument.clone())
5252 .expect("instrument is cacheable");
5253 let manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
5254 .expect("valid config");
5255 let (report, fill) = external_report_with_partial_fill(&instrument);
5256 let expected = Money::new(2.5, Currency::USDT());
5257 let client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5258 let mut fill_queue = ReconciliationFillQueue::default();
5259
5260 let (events, _) = manager.handle_external_order(
5261 &report,
5262 AccountId::from("STUB-001"),
5263 &instrument,
5264 &[&fill],
5265 false,
5266 Some(&mut fill_queue),
5267 Some(&client),
5268 );
5269
5270 let inferred = inferred_fills(&events);
5271 assert_eq!(inferred.len(), 1, "one inferred fill covers the 6.0 gap");
5272 assert_eq!(inferred[0].commission, Some(expected));
5273 }
5274
5275 #[rstest]
5276 fn test_handle_external_order_skips_inferred_fill_when_commission_fails() {
5277 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5278 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5279 let cache = Rc::new(RefCell::new(Cache::default()));
5280 cache
5281 .borrow_mut()
5282 .add_instrument(instrument.clone())
5283 .expect("instrument is cacheable");
5284 let manager =
5285 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5286 .expect("valid config");
5287 let (report, fill) = external_report_with_partial_fill(&instrument);
5288 let client = CommissionStubClient::new(CommissionOutcome::Failure);
5289 let mut fill_queue = ReconciliationFillQueue::default();
5290
5291 let (events, metadata) = manager.handle_external_order(
5292 &report,
5293 AccountId::from("STUB-001"),
5294 &instrument,
5295 &[&fill],
5296 false,
5297 Some(&mut fill_queue),
5298 Some(&client),
5299 );
5300
5301 assert!(events.is_empty());
5302 assert!(metadata.is_none());
5303 assert!(fill_queue.pending_fill_keys.is_empty());
5304 assert!(
5305 cache
5306 .borrow()
5307 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5308 .is_none(),
5309 "commission failure must precede external order cache mutation"
5310 );
5311
5312 let expected = Money::new(2.5, Currency::USDT());
5313 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5314 let (retry_events, retry_metadata) = manager.handle_external_order(
5315 &report,
5316 AccountId::from("STUB-001"),
5317 &instrument,
5318 &[&fill],
5319 false,
5320 Some(&mut fill_queue),
5321 Some(&retry_client),
5322 );
5323 let inferred = inferred_fills(&retry_events);
5324
5325 assert!(retry_metadata.is_some());
5326 assert_eq!(inferred.len(), 1);
5327 assert_eq!(inferred[0].commission, Some(expected));
5328 assert_eq!(fill_queue.pending_fill_keys.len(), 1);
5329 assert!(
5330 cache
5331 .borrow()
5332 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5333 .is_some()
5334 );
5335 }
5336
5337 #[rstest]
5338 fn test_handle_external_order_without_explicit_fills_resolves_commission_before_cache() {
5339 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5340 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5341 let cache = Rc::new(RefCell::new(Cache::default()));
5342 cache
5343 .borrow_mut()
5344 .add_instrument(instrument.clone())
5345 .expect("instrument is cacheable");
5346 let manager =
5347 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5348 .expect("valid config");
5349 let (report, _) = external_report_with_partial_fill(&instrument);
5350 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
5351
5352 let (failed_events, failed_metadata) = manager.handle_external_order(
5353 &report,
5354 AccountId::from("STUB-001"),
5355 &instrument,
5356 &[],
5357 false,
5358 None,
5359 Some(&failing_client),
5360 );
5361
5362 assert!(failed_events.is_empty());
5363 assert!(failed_metadata.is_none());
5364 assert!(
5365 cache
5366 .borrow()
5367 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5368 .is_none()
5369 );
5370
5371 let expected = Money::new(4.0, Currency::USDT());
5372 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5373 let (retry_events, retry_metadata) = manager.handle_external_order(
5374 &report,
5375 AccountId::from("STUB-001"),
5376 &instrument,
5377 &[],
5378 false,
5379 None,
5380 Some(&retry_client),
5381 );
5382
5383 let fills: Vec<_> = retry_events
5384 .iter()
5385 .filter_map(|event| match event {
5386 OrderEventAny::Filled(fill) => Some(fill),
5387 _ => None,
5388 })
5389 .collect();
5390
5391 assert!(retry_metadata.is_some());
5392 assert_eq!(fills.len(), 1);
5393 assert_eq!(fills[0].last_qty, Quantity::from("10.0"));
5394 assert_eq!(fills[0].commission, Some(expected));
5395 }
5396
5397 #[rstest]
5398 fn test_handle_external_order_with_no_override_emits_fill_without_commission() {
5399 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5400 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5401 let cache = Rc::new(RefCell::new(Cache::default()));
5402 cache
5403 .borrow_mut()
5404 .add_instrument(instrument.clone())
5405 .expect("instrument is cacheable");
5406 let manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
5407 .expect("valid config");
5408 let (report, fill) = external_report_with_partial_fill(&instrument);
5409 let client = CommissionStubClient::new(CommissionOutcome::NoOverride);
5410 let mut fill_queue = ReconciliationFillQueue::default();
5411
5412 let (events, _) = manager.handle_external_order(
5413 &report,
5414 AccountId::from("STUB-001"),
5415 &instrument,
5416 &[&fill],
5417 false,
5418 Some(&mut fill_queue),
5419 Some(&client),
5420 );
5421
5422 let inferred = inferred_fills(&events);
5423 assert_eq!(inferred.len(), 1);
5424 assert_eq!(inferred[0].commission, None);
5425 }
5426
5427 #[rstest]
5428 #[case::filled(OrderStatus::Filled, "10.0", "6.0", "33.33", 1)]
5429 #[case::canceled(OrderStatus::Canceled, "8.0", "4.0", "20.00", 2)]
5430 #[case::expired(OrderStatus::Expired, "8.0", "4.0", "20.00", 2)]
5431 fn test_cached_reconciliation_applies_explicit_fill_and_defers_failed_residual(
5432 #[case] status: OrderStatus,
5433 #[case] filled_qty: Quantity,
5434 #[case] residual_qty: Quantity,
5435 #[case] residual_px: Price,
5436 #[case] event_count: usize,
5437 ) {
5438 let (mut manager, _cache, order, mut report, instrument) = cached_commission_fixtures();
5439 report.order_status = status;
5440 report.filled_qty = filled_qty;
5441 report.avg_px = Some(dec!(60.0));
5442
5443 let explicit_fill = FillReport::new(
5444 report.account_id,
5445 report.instrument_id,
5446 report.venue_order_id,
5447 TradeId::from("T-COMMISSION-EXPLICIT"),
5448 OrderSide::Buy,
5449 Quantity::from("4.0"),
5450 Price::from("100.0"),
5451 Money::new(0.25, Currency::USDT()),
5452 LiquiditySide::Taker,
5453 report.client_order_id,
5454 None,
5455 UnixNanos::from(1),
5456 UnixNanos::from(1),
5457 None,
5458 );
5459 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
5460 let mut fill_queue = ReconciliationFillQueue::default();
5461
5462 let first_events = manager.reconcile_order_with_fills(
5463 true,
5464 &order,
5465 &report,
5466 &[&explicit_fill],
5467 Some(&instrument),
5468 &mut fill_queue,
5469 Some(&failing_client),
5470 );
5471 let mut working = order;
5472 for event in &first_events {
5473 working
5474 .apply(event.clone())
5475 .expect("explicit fill projects cleanly");
5476 }
5477
5478 assert_eq!(first_events.len(), 1);
5479
5480 let OrderEventAny::Filled(explicit) = &first_events[0] else {
5481 panic!("expected the valid explicit fill");
5482 };
5483
5484 assert_eq!(explicit.last_qty, Quantity::from("4.0"));
5485 assert_eq!(
5486 explicit.commission,
5487 Some(Money::new(0.25, Currency::USDT()))
5488 );
5489 assert_eq!(working.status(), OrderStatus::PartiallyFilled);
5490
5491 let expected = Money::new(1.5, Currency::USDT());
5492 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5493 let mut reported_residual = explicit_fill.clone();
5494 reported_residual.trade_id = TradeId::from("T-COMMISSION-RESIDUAL");
5495 reported_residual.last_qty = residual_qty;
5496 reported_residual.last_px = residual_px;
5497 reported_residual.commission = expected;
5498
5499 let residual_reports = if status == OrderStatus::Filled {
5500 Vec::new()
5501 } else {
5502 vec![&reported_residual]
5503 };
5504
5505 let retry_events = manager.reconcile_order_with_fills(
5506 true,
5507 &working,
5508 &report,
5509 &residual_reports,
5510 Some(&instrument),
5511 &mut fill_queue,
5512 Some(&retry_client),
5513 );
5514
5515 assert_eq!(retry_events.len(), event_count);
5516
5517 let OrderEventAny::Filled(residual) = &retry_events[0] else {
5518 panic!("expected the residual fill");
5519 };
5520
5521 assert_eq!(residual.last_qty, residual_qty);
5522 assert_eq!(residual.last_px, residual_px);
5523 assert_eq!(residual.commission, Some(expected));
5524 assert_eq!(
5525 retry_client.seen(),
5526 (status == OrderStatus::Filled).then_some((
5527 residual_qty,
5528 residual.last_px,
5529 residual.liquidity_side
5530 )),
5531 "commission must use the exact price and liquidity carried by the residual fill"
5532 );
5533
5534 for event in &retry_events {
5535 working
5536 .apply(event.clone())
5537 .expect("residual precedes terminal status");
5538 }
5539
5540 retry_client.clear_seen();
5541 let replay = manager.reconcile_order_with_fills(
5542 true,
5543 &working,
5544 &report,
5545 &[],
5546 Some(&instrument),
5547 &mut fill_queue,
5548 Some(&retry_client),
5549 );
5550
5551 assert_eq!(working.status(), status);
5552 assert_eq!(working.filled_qty(), filled_qty);
5553 assert_eq!(
5554 working.commissions().get(&Currency::USDT()),
5555 Some(&Money::from("1.75 USDT"))
5556 );
5557 assert!(replay.is_empty());
5558 assert_eq!(retry_client.seen(), None);
5559 }
5560
5561 #[rstest]
5562 fn test_cached_reconciliation_preserves_explicit_fill_side() {
5563 let (mut manager, _cache, order, mut report, instrument) = cached_commission_fixtures();
5564 report.order_status = OrderStatus::PartiallyFilled;
5565 report.filled_qty = Quantity::from("4.0");
5566 let trade_id = TradeId::from("T-CONFLICTING-SIDE");
5567
5568 let explicit_fill = FillReport::new(
5569 report.account_id,
5570 report.instrument_id,
5571 report.venue_order_id,
5572 trade_id,
5573 OrderSide::Sell,
5574 Quantity::from("4.0"),
5575 Price::from("100.0"),
5576 Money::new(0.25, Currency::USDT()),
5577 LiquiditySide::Taker,
5578 report.client_order_id,
5579 None,
5580 UnixNanos::from(1),
5581 UnixNanos::from(1),
5582 None,
5583 );
5584 let mut fill_queue = ReconciliationFillQueue::default();
5585
5586 let events = manager.reconcile_order_with_fills(
5587 true,
5588 &order,
5589 &report,
5590 &[&explicit_fill],
5591 Some(&instrument),
5592 &mut fill_queue,
5593 None,
5594 );
5595
5596 assert_eq!(events.len(), 1);
5597
5598 let OrderEventAny::Filled(fill) = &events[0] else {
5599 panic!("expected the explicit fill");
5600 };
5601
5602 assert_eq!(fill.trade_id, trade_id);
5603 assert_eq!(fill.order_side, OrderSide::Sell);
5604 assert_eq!(fill.last_qty, Quantity::from("4.0"));
5605 assert_eq!(fill.last_px, Price::from("100.0"));
5606 }
5607
5608 #[rstest]
5609 fn test_continuous_report_preserves_newer_fills() {
5610 let (mut manager, _cache, mut order, mut report, instrument) = cached_commission_fixtures();
5611 let fill = TestOrderEventStubs::filled(
5612 &order,
5613 &instrument,
5614 Some(TradeId::from("T-NEWER-STREAM")),
5615 None,
5616 Some(Price::from("100.00")),
5617 Some(Quantity::from("2.0")),
5618 Some(LiquiditySide::Maker),
5619 None,
5620 None,
5621 Some(AccountId::from("TEST-001")),
5622 );
5623 order.apply(fill).unwrap();
5624 report.order_status = OrderStatus::PartiallyFilled;
5625 report.filled_qty = Quantity::from("1.0");
5626 let mut fill_queue = ReconciliationFillQueue::default();
5627
5628 let events = manager.reconcile_order_with_fills(
5629 false,
5630 &order,
5631 &report,
5632 &[],
5633 Some(&instrument),
5634 &mut fill_queue,
5635 None,
5636 );
5637
5638 assert!(events.is_empty());
5639 assert!(fill_queue.pending_fill_keys.is_empty());
5640 }
5641
5642 #[rstest]
5643 #[case::with_fills(true)]
5644 #[case::without_fills(false)]
5645 fn test_cached_snapshot_without_instrument_defers_unaccounted_fills(#[case] has_fills: bool) {
5646 let (mut manager, _cache, order, mut report, _instrument) = cached_commission_fixtures();
5647 report.order_status = OrderStatus::Canceled;
5648
5649 let explicit_fill = FillReport::new(
5650 report.account_id,
5651 report.instrument_id,
5652 report.venue_order_id,
5653 TradeId::from("T-MISSING-INSTRUMENT"),
5654 OrderSide::Buy,
5655 Quantity::from("4.0"),
5656 Price::from("100.0"),
5657 Money::new(0.25, Currency::USDT()),
5658 LiquiditySide::Taker,
5659 report.client_order_id,
5660 None,
5661 UnixNanos::from(1),
5662 UnixNanos::from(1),
5663 None,
5664 );
5665 let mut fill_queue = ReconciliationFillQueue::default();
5666
5667 let fills = if has_fills {
5668 vec![&explicit_fill]
5669 } else {
5670 Vec::new()
5671 };
5672
5673 let events = manager.reconcile_order_with_fills(
5674 true,
5675 &order,
5676 &report,
5677 &fills,
5678 None,
5679 &mut fill_queue,
5680 None,
5681 );
5682
5683 assert!(events.is_empty());
5684 assert_eq!(order.status(), OrderStatus::Accepted);
5685 assert_eq!(order.filled_qty(), Quantity::from("0.0"));
5686 assert!(fill_queue.pending_fill_keys.is_empty());
5687 }
5688
5689 #[rstest]
5690 #[case::canceled(OrderStatus::Canceled)]
5691 #[case::expired(OrderStatus::Expired)]
5692 fn test_terminal_order_report_does_not_void_cached_fills(#[case] status: OrderStatus) {
5693 let (manager, _cache, mut order, mut report, instrument) = cached_commission_fixtures();
5694 let fill = create_inferred_fill_for_qty(
5695 &order,
5696 &report,
5697 &report.account_id,
5698 &instrument,
5699 Quantity::from("4.0"),
5700 UnixNanos::from(1),
5701 None,
5702 )
5703 .unwrap();
5704 order.apply(fill).unwrap();
5705 report.order_status = status;
5706 report.filled_qty = Quantity::from("2.0");
5707 let client = CommissionStubClient::new(CommissionOutcome::Failure);
5708
5709 let events = manager
5710 .reconcile_order_report(&order, &report, Some(&instrument), Some(&client))
5711 .unwrap();
5712 for event in &events {
5713 order.apply(event.clone()).unwrap();
5714 }
5715
5716 assert_eq!(events.len(), 1);
5717 assert_eq!(order.status(), status);
5718 assert_eq!(order.filled_qty(), Quantity::from("4.0"));
5719 assert_eq!(client.seen(), None);
5720 }
5721
5722 #[rstest]
5723 fn test_filled_order_ignores_superseded_cancel_report() {
5724 let (manager, cache, order, mut report, instrument) = cached_commission_fixtures();
5725 let venue_order_id = VenueOrderId::from("V-REPLACEMENT");
5726 let updated = OrderUpdatedSpec::builder()
5727 .trader_id(order.trader_id())
5728 .strategy_id(order.strategy_id())
5729 .instrument_id(order.instrument_id())
5730 .client_order_id(order.client_order_id())
5731 .account_id(report.account_id)
5732 .venue_order_id(venue_order_id)
5733 .quantity(order.quantity())
5734 .build();
5735 let order = cache
5736 .borrow_mut()
5737 .update_order(&OrderEventAny::Updated(updated))
5738 .unwrap();
5739 let mut fill_report = report.clone();
5740 fill_report.venue_order_id = venue_order_id;
5741 let commission = Money::from("1.25 USDT");
5742 let fill = create_inferred_fill_for_qty(
5743 &order,
5744 &fill_report,
5745 &report.account_id,
5746 &instrument,
5747 Quantity::from("10.0"),
5748 UnixNanos::from(2),
5749 Some(commission),
5750 )
5751 .unwrap();
5752 let order = cache.borrow_mut().update_order(&fill).unwrap();
5753 report.order_status = OrderStatus::Canceled;
5754 report.filled_qty = Quantity::from("0.0");
5755 report.avg_px = None;
5756 let client = CommissionStubClient::new(CommissionOutcome::Failure);
5757
5758 let events = manager
5759 .reconcile_order_report(&order, &report, Some(&instrument), Some(&client))
5760 .unwrap();
5761
5762 assert!(events.is_empty());
5763 assert_eq!(order.status(), OrderStatus::Filled);
5764 assert_eq!(order.venue_order_id(), Some(venue_order_id));
5765 assert_eq!(order.filled_qty(), Quantity::from("10.0"));
5766 assert_eq!(order.avg_px(), Some(dec!(100.0)));
5767 assert_eq!(
5768 order.commissions().get(&Currency::USDT()),
5769 Some(&commission)
5770 );
5771 assert_eq!(client.seen(), None);
5772 }
5773
5774 #[rstest]
5775 fn test_continuous_reconciliation_uses_source_client_and_retries_commission() {
5776 let (mut manager, _cache, order, report, _instrument) = cached_commission_fixtures();
5777 let client_id = ClientId::from("STUB");
5778
5779 let check = OpenOrderReportCheck {
5780 command: GenerateOrderStatusReports::new(
5781 UUID4::new(),
5782 UnixNanos::from(1),
5783 true,
5784 None,
5785 None,
5786 None,
5787 None,
5788 None,
5789 ),
5790 filtered_orders: vec![order],
5791 client_coverage: IndexMap::from([(
5792 report.client_order_id.unwrap(),
5793 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
5794 )]),
5795 };
5796
5797 let queried_clients = IndexSet::from([client_id]);
5798 let failed_clients = IndexSet::new();
5799 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
5800
5801 let failed = manager.reconcile_open_order_reports(
5802 &check,
5803 vec![SourcedOrderStatusReport {
5804 client_id,
5805 report: report.clone(),
5806 }],
5807 &queried_clients,
5808 &failed_clients,
5809 &[&failing_client],
5810 );
5811
5812 assert!(failed.events.is_empty());
5813
5814 let expected = Money::new(1.5, Currency::USDT());
5815 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5816 let retry = manager.reconcile_open_order_reports(
5817 &check,
5818 vec![SourcedOrderStatusReport { client_id, report }],
5819 &queried_clients,
5820 &failed_clients,
5821 &[&retry_client],
5822 );
5823
5824 assert_eq!(retry.events.len(), 1);
5825
5826 let OrderEventAny::Filled(fill) = &retry.events[0] else {
5827 panic!("expected inferred fill on valid retry");
5828 };
5829
5830 assert_eq!(fill.last_qty, Quantity::from("10.0"));
5831 assert_eq!(fill.commission, Some(expected));
5832 }
5833
5834 #[rstest]
5835 fn test_open_check_lookback_exclusion_warns_once_without_reconciliation_actions() {
5836 let client_order_id = ClientOrderId::from("O-LOOKBACK-OLD");
5837 let venue_order_id = VenueOrderId::from("V-LOOKBACK-OLD");
5838 let client_id = ClientId::from("BINANCE");
5839 let instrument_id = crypto_perpetual_ethusdt().id();
5840 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5841 let cache = Rc::new(RefCell::new(Cache::default()));
5842 insert_accepted_limit_order(
5843 &cache,
5844 client_order_id,
5845 venue_order_id,
5846 instrument_id,
5847 client_id,
5848 );
5849 let order = cache.borrow().order_owned(&client_order_id).unwrap();
5850 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
5851
5852 let check = OpenOrderReportCheck {
5853 command: GenerateOrderStatusReports::new(
5854 UUID4::new(),
5855 UnixNanos::from(1),
5856 false,
5857 None,
5858 Some(cutoff),
5859 None,
5860 None,
5861 None,
5862 ),
5863 filtered_orders: vec![order],
5864 client_coverage: IndexMap::from([(
5865 client_order_id,
5866 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
5867 )]),
5868 };
5869
5870 let queried_clients = IndexSet::from([client_id]);
5871
5872 let mut manager = ExecutionManager::new(
5873 clock,
5874 cache.clone(),
5875 ExecutionManagerConfig {
5876 open_check_open_only: false,
5877 ..Default::default()
5878 },
5879 )
5880 .expect("valid config");
5881
5882 for _ in 0..2 {
5883 let result = manager.reconcile_open_order_reports(
5884 &check,
5885 Vec::new(),
5886 &queried_clients,
5887 &IndexSet::new(),
5888 &[],
5889 );
5890
5891 assert!(result.events.is_empty());
5892 assert!(result.targeted_queries.is_empty());
5893 assert_eq!(
5894 cache.borrow().order(&client_order_id).unwrap().status(),
5895 OrderStatus::Accepted
5896 );
5897 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
5898 assert!(!manager.order_query_recency.contains_key(&client_order_id));
5899 assert!(!manager.order_query_pending.contains(&client_order_id));
5900 assert_eq!(
5901 manager.order_lookback_warnings,
5902 IndexSet::from([client_order_id])
5903 );
5904 assert_eq!(manager.order_lookback_warnings.len(), 1);
5905 }
5906 }
5907
5908 #[rstest]
5909 fn test_open_check_lookback_warning_clears_at_boundary_and_rearms() {
5910 let client_order_id = ClientOrderId::from("O-LOOKBACK-REARM");
5911 let client_id = ClientId::from("BINANCE");
5912 let cache = Rc::new(RefCell::new(Cache::default()));
5913 insert_accepted_limit_order(
5914 &cache,
5915 client_order_id,
5916 VenueOrderId::from("V-LOOKBACK-REARM"),
5917 crypto_perpetual_ethusdt().id(),
5918 client_id,
5919 );
5920 let order = cache.borrow().order_owned(&client_order_id).unwrap();
5921 let old_cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
5922
5923 let make_check = |start| OpenOrderReportCheck {
5924 command: GenerateOrderStatusReports::new(
5925 UUID4::new(),
5926 UnixNanos::from(1),
5927 false,
5928 None,
5929 Some(start),
5930 None,
5931 None,
5932 None,
5933 ),
5934 filtered_orders: vec![order.clone()],
5935 client_coverage: IndexMap::from([(
5936 client_order_id,
5937 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
5938 )]),
5939 };
5940
5941 let queried_clients = IndexSet::new();
5942
5943 let mut manager = ExecutionManager::new(
5944 Rc::new(RefCell::new(VirtualClock::new())),
5945 cache,
5946 ExecutionManagerConfig {
5947 open_check_open_only: false,
5948 ..Default::default()
5949 },
5950 )
5951 .expect("valid config");
5952
5953 manager.reconcile_open_order_reports(
5954 &make_check(old_cutoff),
5955 Vec::new(),
5956 &queried_clients,
5957 &IndexSet::new(),
5958 &[],
5959 );
5960 assert!(manager.order_lookback_warnings.contains(&client_order_id));
5961
5962 let boundary = order.ts_last();
5963 let boundary_result = manager.reconcile_open_order_reports(
5964 &make_check(boundary),
5965 Vec::new(),
5966 &queried_clients,
5967 &IndexSet::new(),
5968 &[],
5969 );
5970 assert!(boundary_result.targeted_queries.is_empty());
5971 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
5972 assert!(manager.order_coverage_warnings.contains(&client_order_id));
5973
5974 manager.reconcile_open_order_reports(
5975 &make_check(old_cutoff),
5976 Vec::new(),
5977 &queried_clients,
5978 &IndexSet::new(),
5979 &[],
5980 );
5981 assert_eq!(
5982 manager.order_lookback_warnings,
5983 IndexSet::from([client_order_id])
5984 );
5985 }
5986
5987 #[rstest]
5988 fn test_venue_order_id_mapped_report_clears_old_order_lookback_warning() {
5989 let client_order_id = ClientOrderId::from("O-LOOKBACK-MAPPED");
5990 let venue_order_id = VenueOrderId::from("V-LOOKBACK-MAPPED");
5991 let client_id = ClientId::from("BINANCE");
5992 let instrument_id = crypto_perpetual_ethusdt().id();
5993 let cache = Rc::new(RefCell::new(Cache::default()));
5994 insert_accepted_limit_order(
5995 &cache,
5996 client_order_id,
5997 venue_order_id,
5998 instrument_id,
5999 client_id,
6000 );
6001 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6002 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6003
6004 let check = OpenOrderReportCheck {
6005 command: GenerateOrderStatusReports::new(
6006 UUID4::new(),
6007 UnixNanos::from(1),
6008 false,
6009 None,
6010 Some(cutoff),
6011 None,
6012 None,
6013 None,
6014 ),
6015 filtered_orders: vec![order],
6016 client_coverage: IndexMap::from([(
6017 client_order_id,
6018 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6019 )]),
6020 };
6021
6022 let mut manager = ExecutionManager::new(
6023 Rc::new(RefCell::new(VirtualClock::new())),
6024 cache,
6025 ExecutionManagerConfig {
6026 open_check_open_only: false,
6027 ..Default::default()
6028 },
6029 )
6030 .expect("valid config");
6031
6032 manager.order_lookback_warnings.insert(client_order_id);
6033
6034 let report = OrderStatusReport::new(
6037 AccountId::from("TEST-001"),
6038 instrument_id,
6039 None,
6040 venue_order_id,
6041 OrderSide::Buy.into(),
6042 OrderType::Limit,
6043 TimeInForce::Gtc,
6044 OrderStatus::Accepted,
6045 Quantity::from("10.0"),
6046 Quantity::from("0.0"),
6047 UnixNanos::from(0),
6048 UnixNanos::from(0),
6049 UnixNanos::from(0),
6050 None,
6051 );
6052
6053 let result = manager.reconcile_open_order_reports(
6054 &check,
6055 vec![SourcedOrderStatusReport { client_id, report }],
6056 &IndexSet::from([client_id]),
6057 &IndexSet::new(),
6058 &[],
6059 );
6060
6061 assert!(result.targeted_queries.is_empty());
6062 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
6063 }
6064
6065 #[rstest]
6066 fn test_positive_report_clears_old_order_lookback_warning_without_reinserting_it() {
6067 let client_order_id = ClientOrderId::from("O-LOOKBACK-REPORTED");
6068 let venue_order_id = VenueOrderId::from("V-LOOKBACK-REPORTED");
6069 let client_id = ClientId::from("BINANCE");
6070 let instrument_id = crypto_perpetual_ethusdt().id();
6071 let cache = Rc::new(RefCell::new(Cache::default()));
6072 insert_accepted_limit_order(
6073 &cache,
6074 client_order_id,
6075 venue_order_id,
6076 instrument_id,
6077 client_id,
6078 );
6079 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6080 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6081
6082 let check = OpenOrderReportCheck {
6083 command: GenerateOrderStatusReports::new(
6084 UUID4::new(),
6085 UnixNanos::from(1),
6086 false,
6087 None,
6088 Some(cutoff),
6089 None,
6090 None,
6091 None,
6092 ),
6093 filtered_orders: vec![order],
6094 client_coverage: IndexMap::from([(
6095 client_order_id,
6096 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6097 )]),
6098 };
6099
6100 let mut manager = ExecutionManager::new(
6101 Rc::new(RefCell::new(VirtualClock::new())),
6102 cache,
6103 ExecutionManagerConfig {
6104 open_check_open_only: false,
6105 ..Default::default()
6106 },
6107 )
6108 .expect("valid config");
6109
6110 manager.order_lookback_warnings.insert(client_order_id);
6111
6112 let report = OrderStatusReport::new(
6113 AccountId::from("TEST-001"),
6114 instrument_id,
6115 Some(client_order_id),
6116 venue_order_id,
6117 OrderSide::Buy.into(),
6118 OrderType::Limit,
6119 TimeInForce::Gtc,
6120 OrderStatus::Accepted,
6121 Quantity::from("10.0"),
6122 Quantity::from("0.0"),
6123 UnixNanos::from(0),
6124 UnixNanos::from(0),
6125 UnixNanos::from(0),
6126 None,
6127 );
6128
6129 let result = manager.reconcile_open_order_reports(
6130 &check,
6131 vec![SourcedOrderStatusReport { client_id, report }],
6132 &IndexSet::from([client_id]),
6133 &IndexSet::new(),
6134 &[],
6135 );
6136
6137 assert!(result.targeted_queries.is_empty());
6138 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
6139 }
6140
6141 #[rstest]
6142 fn test_targeted_reconciliation_uses_source_client_and_retries_commission() {
6143 let (mut manager, _cache, _order, report, _instrument) = cached_commission_fixtures();
6144 let client_order_id = report.client_order_id.unwrap();
6145 let client_id = ClientId::from("STUB");
6146 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
6147
6148 let failed = manager.reconcile_targeted_order_reports(
6149 vec![TargetedOrderReportResult {
6150 client_order_id,
6151 client_id: Some(client_id),
6152 report: Some(report.clone()),
6153 fills: Vec::new(),
6154 coverage_complete: true,
6155 }],
6156 &[&failing_client],
6157 );
6158
6159 assert!(failed.is_empty());
6160
6161 let expected = Money::new(1.5, Currency::USDT());
6162 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
6163 let retry = manager.reconcile_targeted_order_reports(
6164 vec![TargetedOrderReportResult {
6165 client_order_id,
6166 client_id: Some(client_id),
6167 report: Some(report),
6168 fills: Vec::new(),
6169 coverage_complete: true,
6170 }],
6171 &[&retry_client],
6172 );
6173
6174 assert_eq!(retry.len(), 1);
6175
6176 let OrderEventAny::Filled(fill) = &retry[0] else {
6177 panic!("expected inferred fill on valid targeted retry");
6178 };
6179
6180 assert_eq!(fill.last_qty, Quantity::from("10.0"));
6181 assert_eq!(fill.commission, Some(expected));
6182 }
6183
6184 #[rstest]
6185 fn test_clear_recon_tracking_removes_targeted_query() {
6186 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6187 let cache = Rc::new(RefCell::new(Cache::default()));
6188 let mut manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
6189 .expect("valid config");
6190 let client_order_id = ClientOrderId::from("O-TARGETED-CLEAR");
6191 manager.order_query_pending.insert(client_order_id);
6192 manager.order_lookback_warnings.insert(client_order_id);
6193
6194 manager.clear_recon_tracking(&client_order_id, true);
6195
6196 assert!(manager.order_query_pending.is_empty());
6197 assert!(manager.order_lookback_warnings.is_empty());
6198 }
6199
6200 #[rstest]
6201 fn test_register_inflight_skips_filtered_order() {
6202 let client_order_id = ClientOrderId::from("O-FILTERED-REGISTER");
6203 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6204 let cache = Rc::new(RefCell::new(Cache::default()));
6205
6206 let mut manager = ExecutionManager::new(
6207 clock,
6208 cache,
6209 ExecutionManagerConfig {
6210 filtered_client_order_ids: IndexSet::from([client_order_id]),
6211 ..Default::default()
6212 },
6213 )
6214 .expect("valid config");
6215
6216 manager.register_inflight(client_order_id);
6217
6218 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6219 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
6220 }
6221
6222 #[rstest]
6223 #[cfg_attr(
6224 not(all(feature = "simulation", madsim)),
6225 tokio::test(start_paused = true)
6226 )]
6227 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6228 async fn test_inflight_check_retires_order_filtered_after_registration() {
6229 let client_order_id = ClientOrderId::from("O-FILTERED-LATE");
6230 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6231 let cache = Rc::new(RefCell::new(Cache::default()));
6232
6233 let mut manager = ExecutionManager::new(
6234 clock,
6235 cache,
6236 ExecutionManagerConfig {
6237 inflight_threshold_ms: 100,
6238 ..Default::default()
6239 },
6240 )
6241 .expect("valid config");
6242
6243 manager.register_inflight(client_order_id);
6244 manager
6245 .config
6246 .filtered_client_order_ids
6247 .insert(client_order_id);
6248 dst::time::sleep(Duration::from_millis(101)).await;
6249
6250 let first = manager.check_inflight_orders();
6251
6252 assert!(first.events.is_empty());
6253 assert!(first.queries.is_empty());
6254 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6255 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
6256
6257 dst::time::sleep(Duration::from_millis(101)).await;
6258 let second = manager.check_inflight_orders();
6259 assert!(second.events.is_empty());
6260 assert!(second.queries.is_empty());
6261 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6262 }
6263
6264 #[rstest]
6265 #[case(false, OrderStatus::PendingUpdate, true, true, true)]
6266 #[case(false, OrderStatus::Accepted, false, true, true)]
6267 #[case(false, OrderStatus::Canceled, false, true, false)]
6268 #[case(true, OrderStatus::PendingCancel, true, true, true)]
6269 #[case(true, OrderStatus::Accepted, false, true, true)]
6270 #[case(true, OrderStatus::Filled, false, true, false)]
6271 fn test_observe_order_status_report_tracking_matrix(
6272 #[case] with_fills: bool,
6273 #[case] status: OrderStatus,
6274 #[case] expect_inflight: bool,
6275 #[case] expect_activity: bool,
6276 #[case] expect_last_query: bool,
6277 ) {
6278 let client_order_id = ClientOrderId::from("O-STATUS-MATRIX");
6279 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6280 let cache = Rc::new(RefCell::new(Cache::default()));
6281 let mut manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
6282 .expect("valid config");
6283 manager.register_inflight(client_order_id);
6284 manager.order_query_recency.mark(client_order_id);
6285 manager.order_coverage_warnings.insert(client_order_id);
6286 manager.order_coverage_unresolved.insert(client_order_id);
6287 manager.order_query_pending.insert(client_order_id);
6288
6289 let order_report = OrderStatusReport::new(
6290 AccountId::from("TEST-001"),
6291 crypto_perpetual_ethusdt().id(),
6292 Some(client_order_id),
6293 VenueOrderId::from("V-STATUS-MATRIX"),
6294 OrderSide::Buy.into(),
6295 OrderType::Limit,
6296 TimeInForce::Gtc,
6297 status,
6298 Quantity::from("10.0"),
6299 Quantity::from("0.0"),
6300 UnixNanos::from(1_000),
6301 UnixNanos::from(1_000),
6302 UnixNanos::from(1_000),
6303 None,
6304 );
6305
6306 let report = if with_fills {
6307 ExecutionReport::OrderWithFills(Box::new(order_report), Vec::new())
6308 } else {
6309 ExecutionReport::Order(Box::new(order_report))
6310 };
6311
6312 manager.observe_execution_report(&report);
6313
6314 assert_eq!(
6315 manager.order_inflight_checks.contains_key(&client_order_id),
6316 expect_inflight,
6317 );
6318 assert_eq!(
6319 manager.order_recon_retries.contains_key(&client_order_id),
6320 expect_inflight,
6321 );
6322 assert_eq!(
6323 manager.order_activity.contains_key(&client_order_id),
6324 expect_activity,
6325 );
6326 assert_eq!(
6327 manager.order_query_recency.contains_key(&client_order_id),
6328 expect_last_query,
6329 );
6330 assert_eq!(
6331 manager.order_coverage_warnings.contains(&client_order_id),
6332 expect_inflight,
6333 );
6334 assert_eq!(
6335 manager.order_coverage_unresolved.contains(&client_order_id),
6336 expect_inflight,
6337 );
6338 assert_eq!(
6339 manager.order_query_pending.contains(&client_order_id),
6340 expect_inflight,
6341 );
6342 }
6343
6344 #[rstest]
6345 #[case(OrderStatus::PendingUpdate)]
6346 #[case(OrderStatus::PendingCancel)]
6347 fn test_accepted_report_during_pending_command_preserves_inflight_tracking(
6348 #[case] pending_status: OrderStatus,
6349 ) {
6350 let client_order_id = ClientOrderId::from("O-PENDING-COMMAND");
6351 let venue_order_id = VenueOrderId::from("V-PENDING-COMMAND");
6352 let account_id = AccountId::from("TEST-001");
6353 let client_id = ClientId::from("TEST");
6354 let instrument_id = crypto_perpetual_ethusdt().id();
6355 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6356 let cache = Rc::new(RefCell::new(Cache::default()));
6357 insert_accepted_limit_order(
6358 &cache,
6359 client_order_id,
6360 venue_order_id,
6361 instrument_id,
6362 client_id,
6363 );
6364
6365 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6366
6367 let event = match pending_status {
6368 OrderStatus::PendingUpdate => OrderEventAny::PendingUpdate(
6369 OrderPendingUpdateSpec::builder()
6370 .trader_id(order.trader_id())
6371 .strategy_id(order.strategy_id())
6372 .instrument_id(instrument_id)
6373 .client_order_id(client_order_id)
6374 .account_id(account_id)
6375 .venue_order_id(venue_order_id)
6376 .build(),
6377 ),
6378 OrderStatus::PendingCancel => OrderEventAny::PendingCancel(
6379 OrderPendingCancelSpec::builder()
6380 .trader_id(order.trader_id())
6381 .strategy_id(order.strategy_id())
6382 .instrument_id(instrument_id)
6383 .client_order_id(client_order_id)
6384 .account_id(account_id)
6385 .venue_order_id(venue_order_id)
6386 .build(),
6387 ),
6388 _ => unreachable!(),
6389 };
6390
6391 cache.borrow_mut().update_order(&event).unwrap();
6392
6393 let mut manager =
6394 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
6395 .expect("valid config");
6396 manager.register_inflight(client_order_id);
6397 manager.order_query_recency.mark(client_order_id);
6398 manager.order_coverage_warnings.insert(client_order_id);
6399 manager.order_coverage_unresolved.insert(client_order_id);
6400 manager.order_query_pending.insert(client_order_id);
6401 let report = OrderStatusReport::new(
6402 account_id,
6403 instrument_id,
6404 Some(client_order_id),
6405 venue_order_id,
6406 OrderSide::Buy.into(),
6407 OrderType::Limit,
6408 TimeInForce::Gtc,
6409 OrderStatus::Accepted,
6410 Quantity::from("10.0"),
6411 Quantity::from("0.0"),
6412 UnixNanos::from(1_000),
6413 UnixNanos::from(1_000),
6414 UnixNanos::from(1_000),
6415 None,
6416 )
6417 .with_price(Price::from("100.0"));
6418
6419 manager.observe_execution_report(&ExecutionReport::Order(Box::new(report.clone())));
6420 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6421 let events =
6422 generate_reconciliation_order_events(&order, &report, None, UnixNanos::from(1_000));
6423
6424 assert!(events.is_empty());
6425 assert_eq!(order.status(), pending_status);
6426 assert!(manager.order_inflight_checks.contains_key(&client_order_id));
6427 assert!(manager.order_recon_retries.contains_key(&client_order_id));
6428 assert!(manager.order_query_recency.contains_key(&client_order_id));
6429 assert!(manager.order_activity.contains_key(&client_order_id));
6430 assert!(manager.order_coverage_warnings.contains(&client_order_id));
6431 assert!(manager.order_coverage_unresolved.contains(&client_order_id));
6432 assert!(manager.order_query_pending.contains(&client_order_id));
6433 }
6434
6435 #[rstest]
6436 fn test_superseded_cancel_report_preserves_missing_order_grace() {
6437 let client_order_id = ClientOrderId::from("O-CANCEL-REPLACE");
6438 let old_venue_order_id = VenueOrderId::from("V-CANCEL-REPLACE-OLD");
6439 let new_venue_order_id = VenueOrderId::from("V-CANCEL-REPLACE-NEW");
6440 let account_id = AccountId::from("TEST-001");
6441 let client_id = ClientId::from("TEST");
6442 let instrument_id = crypto_perpetual_ethusdt().id();
6443 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6444 let cache = Rc::new(RefCell::new(Cache::default()));
6445 insert_accepted_limit_order(
6446 &cache,
6447 client_order_id,
6448 old_venue_order_id,
6449 instrument_id,
6450 client_id,
6451 );
6452
6453 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6454 let pending_update = OrderPendingUpdateSpec::builder()
6455 .trader_id(order.trader_id())
6456 .strategy_id(order.strategy_id())
6457 .instrument_id(order.instrument_id())
6458 .client_order_id(client_order_id)
6459 .account_id(account_id)
6460 .venue_order_id(old_venue_order_id)
6461 .build();
6462 cache
6463 .borrow_mut()
6464 .update_order(&OrderEventAny::PendingUpdate(pending_update))
6465 .unwrap();
6466 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6467 let updated = OrderUpdatedSpec::builder()
6468 .trader_id(order.trader_id())
6469 .strategy_id(order.strategy_id())
6470 .instrument_id(order.instrument_id())
6471 .client_order_id(client_order_id)
6472 .quantity(order.quantity())
6473 .venue_order_id(new_venue_order_id)
6474 .account_id(account_id)
6475 .build();
6476 cache
6477 .borrow_mut()
6478 .update_order(&OrderEventAny::Updated(updated))
6479 .unwrap();
6480
6481 let mut manager = ExecutionManager::new(
6482 clock,
6483 cache.clone(),
6484 ExecutionManagerConfig {
6485 open_check_missing_retries: 1,
6486 ..Default::default()
6487 },
6488 )
6489 .expect("valid config");
6490
6491 manager.record_local_activity(client_order_id);
6492 assert!(
6493 manager
6494 .prepare_missing_order_query(client_order_id)
6495 .is_none()
6496 );
6497
6498 let report = OrderStatusReport::new(
6499 account_id,
6500 instrument_id,
6501 Some(client_order_id),
6502 old_venue_order_id,
6503 OrderSide::Buy.into(),
6504 OrderType::Limit,
6505 TimeInForce::Gtc,
6506 OrderStatus::Canceled,
6507 Quantity::from("10.0"),
6508 Quantity::from("0.0"),
6509 UnixNanos::from(1_000),
6510 UnixNanos::from(2_000),
6511 UnixNanos::from(3_000),
6512 None,
6513 );
6514
6515 manager.observe_execution_report(&ExecutionReport::Order(Box::new(report.clone())));
6516 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6517 let events =
6518 generate_reconciliation_order_events(&order, &report, None, UnixNanos::from(1_000));
6519
6520 assert!(events.is_empty());
6521 assert_eq!(order.status(), OrderStatus::Accepted);
6522 assert_eq!(order.venue_order_id(), Some(new_venue_order_id));
6523 assert!(manager.order_activity.contains_key(&client_order_id));
6524 assert!(
6525 manager
6526 .prepare_missing_order_query(client_order_id)
6527 .is_none()
6528 );
6529 assert_eq!(manager.recon_check_retry_count(&client_order_id), 0);
6530 }
6531
6532 #[rstest]
6533 #[cfg_attr(
6534 not(all(feature = "simulation", madsim)),
6535 tokio::test(start_paused = true)
6536 )]
6537 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6538 async fn test_prune_order_local_activity_uses_open_check_threshold() {
6539 let old_id = ClientOrderId::from("O-ACTIVITY-OLD");
6540 let fresh_id = ClientOrderId::from("O-ACTIVITY-FRESH");
6541 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6542 let cache = Rc::new(RefCell::new(Cache::default()));
6543
6544 let mut manager = ExecutionManager::new(
6545 clock,
6546 cache,
6547 ExecutionManagerConfig {
6548 open_check_threshold_ns: DurationNanos::from_millis(100),
6549 ..Default::default()
6550 },
6551 )
6552 .expect("valid config");
6553
6554 manager.record_local_activity(old_id);
6555 dst::time::sleep(Duration::from_millis(101)).await;
6556 manager.record_local_activity(fresh_id);
6557
6558 manager.prune_order_local_activity();
6559
6560 assert!(!manager.order_activity.contains_key(&old_id));
6561 assert!(manager.order_activity.contains_key(&fresh_id));
6562 }
6563
6564 #[rstest]
6565 fn test_prepare_open_order_report_check_builds_bulk_command_with_config() {
6566 let lookback_mins = 5_u64;
6567 let lookback = DurationNanos::from_mins(lookback_mins);
6568 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6569 let cache = Rc::new(RefCell::new(Cache::default()));
6570
6571 let mut manager = ExecutionManager::new(
6572 clock.clone(),
6573 cache.clone(),
6574 ExecutionManagerConfig {
6575 open_check_lookback_mins: Some(lookback_mins),
6576 open_check_open_only: false,
6577 reconciliation_instrument_ids: IndexSet::from([crypto_perpetual_ethusdt().id()]),
6578 ..Default::default()
6579 },
6580 )
6581 .expect("valid config");
6582
6583 let included_id = ClientOrderId::from("O-REPORT-001");
6584 let excluded_id = ClientOrderId::from("O-REPORT-002");
6585 let included_instrument_id = crypto_perpetual_ethusdt().id();
6586 let excluded_instrument_id = xbtusd_bitmex().id();
6587
6588 cache
6589 .borrow_mut()
6590 .add_instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
6591 .unwrap();
6592 cache
6593 .borrow_mut()
6594 .add_instrument(InstrumentAny::CryptoPerpetual(xbtusd_bitmex()))
6595 .unwrap();
6596 insert_accepted_limit_order(
6597 &cache,
6598 included_id,
6599 VenueOrderId::from("V-REPORT-001"),
6600 included_instrument_id,
6601 ClientId::from("BINANCE"),
6602 );
6603 insert_accepted_limit_order(
6604 &cache,
6605 excluded_id,
6606 VenueOrderId::from("V-REPORT-002"),
6607 excluded_instrument_id,
6608 ClientId::from("BITMEX"),
6609 );
6610 clock
6611 .borrow_mut()
6612 .advance_time(UnixNanos::default().saturating_add(lookback * 2), true);
6613
6614 let ts_now = clock.borrow().timestamp_ns();
6615 let command_id = UUID4::new();
6616 let check = manager.prepare_open_order_report_check(command_id, &[]);
6617
6618 assert_eq!(check.command.command_id, command_id);
6619 assert_eq!(check.command.ts_init, ts_now);
6620 assert!(!check.command.open_only);
6621 assert_eq!(check.command.instrument_id, None);
6622 assert_eq!(check.command.start, Some(ts_now.saturating_sub(lookback)));
6623 assert_eq!(check.command.end, None);
6624 assert_eq!(check.command.log_receipt_level, LogLevel::Debug);
6625 assert_eq!(check.filtered_orders.len(), 1);
6626 assert_eq!(check.filtered_orders[0].client_order_id(), included_id);
6627 }
6628
6629 #[rstest]
6630 fn test_prepare_position_report_check_builds_bulk_command_with_coverage() {
6631 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6632 let cache = Rc::new(RefCell::new(Cache::default()));
6633
6634 let manager = ExecutionManager::new(
6635 clock.clone(),
6636 cache.clone(),
6637 ExecutionManagerConfig {
6638 reconciliation_instrument_ids: IndexSet::from([crypto_perpetual_ethusdt().id()]),
6639 ..Default::default()
6640 },
6641 )
6642 .expect("valid config");
6643
6644 let included_instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6645 let excluded_instrument = InstrumentAny::CryptoPerpetual(xbtusd_bitmex());
6646
6647 cache
6648 .borrow_mut()
6649 .add_instrument(included_instrument.clone())
6650 .unwrap();
6651 cache
6652 .borrow_mut()
6653 .add_instrument(excluded_instrument.clone())
6654 .unwrap();
6655 let included_position = insert_open_position(
6656 &cache,
6657 &included_instrument,
6658 PositionId::from("P-REPORT-001"),
6659 OrderSide::Buy,
6660 "5.0",
6661 "3000.00",
6662 );
6663 insert_open_position(
6664 &cache,
6665 &excluded_instrument,
6666 PositionId::from("P-REPORT-002"),
6667 OrderSide::Buy,
6668 "2.0",
6669 "40000.00",
6670 );
6671
6672 let ts_now = clock.borrow().timestamp_ns();
6673 let command_id = UUID4::new();
6674 let check = manager.prepare_position_report_check(command_id, &[]);
6675 let key = (
6676 included_position.instrument_id,
6677 included_position.account_id,
6678 );
6679
6680 assert_eq!(check.command.command_id, command_id);
6681 assert_eq!(check.command.ts_init, ts_now);
6682 assert_eq!(check.command.instrument_id, None);
6683 assert_eq!(check.command.start, None);
6684 assert_eq!(check.command.end, None);
6685 assert_eq!(check.command.log_receipt_level, LogLevel::Debug);
6686 assert_eq!(check.client_coverage.len(), 1);
6687 assert_eq!(
6688 check.client_coverage.get(&key),
6689 Some(&ReportClientCoverage::Unresolved)
6690 );
6691 assert_eq!(check.activity_revisions.get(&key), Some(&0));
6692 }
6693
6694 #[rstest]
6695 fn test_position_reconciliation_preserves_unavailable_spot_coverage() {
6696 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6697 let cache = Rc::new(RefCell::new(Cache::default()));
6698 let mut manager =
6699 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
6700 .expect("valid config");
6701 let derivative = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6702 let spot = test_bybit_spot_instrument();
6703 cache
6704 .borrow_mut()
6705 .add_instrument(derivative.clone())
6706 .unwrap();
6707 cache.borrow_mut().add_instrument(spot.clone()).unwrap();
6708 let derivative_position = insert_open_position(
6709 &cache,
6710 &derivative,
6711 PositionId::from("P-DERIVATIVE-RECONCILE"),
6712 OrderSide::Buy,
6713 "5.0",
6714 "3000.00",
6715 );
6716 let spot_position = insert_open_position(
6717 &cache,
6718 &spot,
6719 PositionId::from("P-SPOT-PRESERVED"),
6720 OrderSide::Buy,
6721 "2.0",
6722 "2000.00",
6723 );
6724 let client = PositionCoverageStubClient;
6725 let check = manager.prepare_position_report_check(UUID4::new(), &[&client]);
6726 let queried_clients = IndexSet::from([client.client_id()]);
6727
6728 let events = manager.reconcile_position_reports(
6729 &check,
6730 Vec::new(),
6731 &queried_clients,
6732 &IndexSet::new(),
6733 );
6734
6735 assert!(events.iter().any(|event| {
6736 matches!(event, OrderEventAny::Filled(fill) if fill.instrument_id == derivative_position.instrument_id)
6737 }));
6738 assert!(!events.iter().any(|event| {
6739 matches!(event, OrderEventAny::Filled(fill) if fill.instrument_id == spot_position.instrument_id)
6740 }));
6741 }
6742
6743 #[rstest]
6744 fn test_position_reconciliation_preserves_spot_position_when_client_query_fails() {
6745 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6746 let cache = Rc::new(RefCell::new(Cache::default()));
6747 let mut manager =
6748 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
6749 .expect("valid config");
6750 let instrument = test_bybit_spot_instrument();
6751 cache
6752 .borrow_mut()
6753 .add_instrument(instrument.clone())
6754 .unwrap();
6755 let position = insert_open_position(
6756 &cache,
6757 &instrument,
6758 PositionId::from("P-SPOT-QUERY-FAILED"),
6759 OrderSide::Buy,
6760 "5.0",
6761 "3000.00",
6762 );
6763 let key = (position.instrument_id, position.account_id);
6764 let client_id = ClientId::from("BYBIT");
6765 let mut check = manager.prepare_position_report_check(UUID4::new(), &[]);
6766 check.client_coverage.insert(
6767 key,
6768 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6769 );
6770 let queried_clients = IndexSet::from([client_id]);
6771 let failed_clients = IndexSet::from([client_id]);
6772
6773 let events = manager.reconcile_position_reports(
6774 &check,
6775 Vec::new(),
6776 &queried_clients,
6777 &failed_clients,
6778 );
6779
6780 assert!(
6781 !events
6782 .iter()
6783 .any(|event| matches!(event, OrderEventAny::Filled(_))),
6784 "a failed bulk query must not generate a synthetic closing fill",
6785 );
6786 let cached_position = cache.borrow().position(&position.id).unwrap().clone();
6787 assert!(cached_position.is_open());
6788 assert_eq!(cached_position.quantity, Quantity::from("5.0"));
6789 }
6790
6791 #[rstest]
6792 #[cfg_attr(
6793 not(all(feature = "simulation", madsim)),
6794 tokio::test(start_paused = true)
6795 )]
6796 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6797 async fn test_position_report_check_defers_activity_recorded_during_delayed_request() {
6798 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6799 let cache = Rc::new(RefCell::new(Cache::default()));
6800
6801 let mut manager = ExecutionManager::new(
6802 clock,
6803 cache.clone(),
6804 ExecutionManagerConfig {
6805 position_check_threshold_ns: DurationNanos::from_secs(5),
6806 ..Default::default()
6807 },
6808 )
6809 .expect("valid config");
6810
6811 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6812 let instrument_id = instrument.id();
6813 let position = insert_open_position(
6814 &cache,
6815 &instrument,
6816 PositionId::from("P-ACTIVITY-DURING-REQUEST"),
6817 OrderSide::Buy,
6818 "5.0",
6819 "3000.00",
6820 );
6821 cache
6822 .borrow_mut()
6823 .add_instrument(instrument.clone())
6824 .unwrap();
6825 let account_id = position.account_id;
6826 let check = manager.prepare_position_report_check(UUID4::new(), &[]);
6827
6828 let report = PositionStatusReport::new(
6829 account_id,
6830 instrument_id,
6831 PositionSide::Long,
6832 Quantity::from("5.0"),
6833 UnixNanos::from(1_000_000),
6834 UnixNanos::from(1_000_000),
6835 None,
6836 None,
6837 Some(Decimal::from(3000)),
6838 );
6839
6840 let closed_position = close_long_position(
6841 position,
6842 &instrument,
6843 TradeId::from("T-ACTIVITY-DURING-REQUEST"),
6844 );
6845 cache
6846 .borrow_mut()
6847 .update_position(&closed_position)
6848 .unwrap();
6849 manager.record_position_activity(instrument_id, account_id);
6850
6851 dst::time::sleep(Duration::from_secs(6)).await;
6853
6854 let events = manager.reconcile_position_reports(
6855 &check,
6856 vec![report],
6857 &IndexSet::new(),
6858 &IndexSet::new(),
6859 );
6860
6861 assert!(
6862 !events.iter().any(|event| {
6863 matches!(
6864 event,
6865 OrderEventAny::Filled(fill)
6866 if fill.order_side == OrderSide::Buy
6867 && fill.last_qty == Quantity::from("5.0")
6868 )
6869 }),
6870 "activity recorded after the request started must defer A's stale report",
6871 );
6872 }
6873
6874 #[rstest]
6875 fn test_position_report_check_does_not_defer_activity_recorded_before_request() {
6876 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6877 let cache = Rc::new(RefCell::new(Cache::default()));
6878
6879 let mut manager = ExecutionManager::new(
6880 clock,
6881 cache.clone(),
6882 ExecutionManagerConfig {
6883 position_check_threshold_ns: DurationNanos::ZERO,
6884 ..Default::default()
6885 },
6886 )
6887 .expect("valid config");
6888
6889 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6890 let instrument_id = instrument.id();
6891 let position = insert_open_position(
6892 &cache,
6893 &instrument,
6894 PositionId::from("P-ACTIVITY-BEFORE-REQUEST"),
6895 OrderSide::Buy,
6896 "5.0",
6897 "3000.00",
6898 );
6899 cache.borrow_mut().add_instrument(instrument).unwrap();
6900 let account_id = position.account_id;
6901 manager.record_position_activity(instrument_id, account_id);
6902 let check = manager.prepare_position_report_check(UUID4::new(), &[]);
6903
6904 let report = PositionStatusReport::new(
6905 account_id,
6906 instrument_id,
6907 PositionSide::Long,
6908 Quantity::from("10.0"),
6909 UnixNanos::from(1_000_000),
6910 UnixNanos::from(1_000_000),
6911 None,
6912 None,
6913 Some(Decimal::from(3000)),
6914 );
6915
6916 let events = manager.reconcile_position_reports(
6917 &check,
6918 vec![report],
6919 &IndexSet::new(),
6920 &IndexSet::new(),
6921 );
6922
6923 let fills: Vec<_> = events
6924 .iter()
6925 .filter_map(|event| match event {
6926 OrderEventAny::Filled(fill) => Some(fill),
6927 _ => None,
6928 })
6929 .collect();
6930
6931 assert_eq!(fills.len(), 1);
6932 assert_eq!(fills[0].order_side, OrderSide::Buy);
6933 assert_eq!(fills[0].last_qty, Quantity::from("5.0"));
6934 assert_eq!(fills[0].commission, None);
6935 }
6936
6937 #[rstest]
6938 fn test_mass_status_projects_companion_fill_before_void_correction() {
6939 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6940 let cache = Rc::new(RefCell::new(Cache::default()));
6941 let mut manager =
6942 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
6943 .expect("valid config");
6944 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6945 let client_order_id = ClientOrderId::from("O-MASS-VOID-001");
6946 let venue_order_id = VenueOrderId::from("V-MASS-VOID-001");
6947 let account_id = AccountId::from("TEST-001");
6948 cache
6949 .borrow_mut()
6950 .add_instrument(instrument.clone())
6951 .unwrap();
6952 insert_accepted_limit_order(
6953 &cache,
6954 client_order_id,
6955 venue_order_id,
6956 instrument.id(),
6957 ClientId::from("BINANCE"),
6958 );
6959 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6960 let initial_fill = TestOrderEventStubs::filled(
6961 &order,
6962 &instrument,
6963 Some(TradeId::from("T-MASS-VOID-INITIAL")),
6964 None,
6965 Some(Price::from("100.0")),
6966 Some(Quantity::from("6.0")),
6967 Some(LiquiditySide::Taker),
6968 None,
6969 None,
6970 Some(account_id),
6971 );
6972 cache.borrow_mut().update_order(&initial_fill).unwrap();
6973 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6974
6975 let report = OrderStatusReport::new(
6976 account_id,
6977 instrument.id(),
6978 Some(client_order_id),
6979 venue_order_id,
6980 OrderSide::Buy.into(),
6981 OrderType::Limit,
6982 TimeInForce::Gtc,
6983 OrderStatus::Canceled,
6984 Quantity::from("10.0"),
6985 Quantity::from("5.0"),
6986 UnixNanos::from(1_000),
6987 UnixNanos::from(1_000),
6988 UnixNanos::from(1_000),
6989 None,
6990 );
6991
6992 let companion_fill = FillReport::new(
6993 account_id,
6994 instrument.id(),
6995 venue_order_id,
6996 TradeId::from("T-MASS-VOID-COMPANION"),
6997 OrderSide::Buy,
6998 Quantity::from("1.0"),
6999 Price::from("100.0"),
7000 Money::zero(instrument.quote_currency()),
7001 LiquiditySide::Taker,
7002 Some(client_order_id),
7003 None,
7004 UnixNanos::from(900),
7005 UnixNanos::from(1_000),
7006 None,
7007 );
7008
7009 let mut fill_queue = ReconciliationFillQueue::default();
7010 let events = manager.reconcile_order_with_fills(
7011 true,
7012 &order,
7013 &report,
7014 &[&companion_fill],
7015 Some(&instrument),
7016 &mut fill_queue,
7017 None,
7018 );
7019 let mut projected = order;
7020 for event in &events {
7021 projected.apply(event.clone()).unwrap();
7022 }
7023
7024 assert!(matches!(events[0], OrderEventAny::Filled(_)));
7025 assert_eq!(
7026 events
7027 .iter()
7028 .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
7029 .count(),
7030 2
7031 );
7032 assert_eq!(projected.status(), OrderStatus::Canceled);
7033 assert_eq!(projected.filled_qty(), Quantity::from("5.0"));
7034 assert_eq!(projected.voided_qty(), Quantity::from("2.0"));
7035 }
7036
7037 fn insert_accepted_limit_order(
7038 cache: &Rc<RefCell<Cache>>,
7039 client_order_id: ClientOrderId,
7040 venue_order_id: VenueOrderId,
7041 instrument_id: InstrumentId,
7042 client_id: ClientId,
7043 ) {
7044 let account_id = AccountId::from("TEST-001");
7045 let order = OrderTestBuilder::new(OrderType::Limit)
7046 .client_order_id(client_order_id)
7047 .instrument_id(instrument_id)
7048 .quantity(Quantity::from("10.0"))
7049 .price(Price::from("100.0"))
7050 .build();
7051 let submitted = TestOrderEventStubs::submitted(&order, account_id);
7052 cache
7053 .borrow_mut()
7054 .add_order(order, None, Some(client_id), false)
7055 .unwrap();
7056 let order = cache.borrow_mut().update_order(&submitted).unwrap();
7057 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
7058 cache.borrow_mut().update_order(&accepted).unwrap();
7059 }
7060
7061 fn test_bybit_spot_instrument() -> InstrumentAny {
7062 InstrumentAny::CurrencyPair(
7063 CurrencyPair::builder()
7064 .instrument_id(InstrumentId::from("ETHUSDT-SPOT.BYBIT"))
7065 .raw_symbol(Symbol::from("ETHUSDT"))
7066 .base_currency(Currency::from("ETH"))
7067 .quote_currency(Currency::from("USDT"))
7068 .price_precision(2)
7069 .size_precision(5)
7070 .price_increment(Price::from("0.01"))
7071 .size_increment(Quantity::from("0.00001"))
7072 .ts_event(UnixNanos::default())
7073 .ts_init(UnixNanos::default())
7074 .build()
7075 .unwrap(),
7076 )
7077 }
7078
7079 fn insert_open_position(
7080 cache: &Rc<RefCell<Cache>>,
7081 instrument: &InstrumentAny,
7082 position_id: PositionId,
7083 side: OrderSide,
7084 quantity: &str,
7085 price: &str,
7086 ) -> Position {
7087 let order = OrderTestBuilder::new(OrderType::Market)
7088 .instrument_id(instrument.id())
7089 .side(side)
7090 .quantity(Quantity::from(quantity))
7091 .build();
7092 let fill = TestOrderEventStubs::filled(
7093 &order,
7094 instrument,
7095 Some(TradeId::new("T-REPORT-001")),
7096 Some(position_id),
7097 Some(Price::from(price)),
7098 Some(Quantity::from(quantity)),
7099 None,
7100 None,
7101 None,
7102 Some(AccountId::from("TEST-001")),
7103 );
7104 let order_filled: OrderFilled = fill.into();
7105 let position = Position::new(instrument, order_filled);
7106 cache
7107 .borrow_mut()
7108 .add_position(&position, OmsType::Hedging)
7109 .unwrap();
7110 position
7111 }
7112
7113 fn close_long_position(
7114 mut position: Position,
7115 instrument: &InstrumentAny,
7116 trade_id: TradeId,
7117 ) -> Position {
7118 let order = OrderTestBuilder::new(OrderType::Market)
7119 .instrument_id(instrument.id())
7120 .side(OrderSide::Sell)
7121 .quantity(position.quantity)
7122 .build();
7123 let fill = TestOrderEventStubs::filled(
7124 &order,
7125 instrument,
7126 Some(trade_id),
7127 Some(position.id),
7128 Some(Price::from("3000.00")),
7129 Some(position.quantity),
7130 None,
7131 None,
7132 None,
7133 Some(position.account_id),
7134 );
7135 let order_filled: OrderFilled = fill.into();
7136 position.apply(&order_filled);
7137 position
7138 }
7139
7140 #[cfg(feature = "node")]
7141 mod node {
7142 use super::*;
7143 use crate::execution::client::LiveExecutionClient;
7144
7145 #[rstest]
7146 fn test_plan_position_fill_reports_uses_configured_lookback() {
7147 let lookback_mins = 7_u64;
7148 let lookback = DurationNanos::from_mins(lookback_mins);
7149 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7150 let cache = Rc::new(RefCell::new(Cache::default()));
7151
7152 let mut manager = ExecutionManager::new(
7153 clock.clone(),
7154 cache.clone(),
7155 ExecutionManagerConfig {
7156 position_check_lookback_mins: lookback_mins,
7157 position_check_threshold_ns: DurationNanos::ZERO,
7158 ..Default::default()
7159 },
7160 )
7161 .expect("valid config");
7162
7163 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7164 cache
7165 .borrow_mut()
7166 .add_instrument(instrument.clone())
7167 .unwrap();
7168 let position = insert_open_position(
7169 &cache,
7170 &instrument,
7171 PositionId::from("P-FILL-LOOKBACK"),
7172 OrderSide::Buy,
7173 "1.0",
7174 "3000.00",
7175 );
7176 clock
7177 .borrow_mut()
7178 .advance_time(UnixNanos::default().saturating_add(lookback * 2), true);
7179 let client = PositionCoverageStubClient;
7180 let clients: [&dyn ExecutionClient; 1] = [&client];
7181 let mut check = manager.prepare_position_report_check(UUID4::new(), &clients);
7182 let query_end = clock.borrow().timestamp_ns();
7183
7184 let report = PositionStatusReport::new(
7185 position.account_id,
7186 position.instrument_id,
7187 PositionSide::Long,
7188 Quantity::from("2.0"),
7189 query_end,
7190 query_end,
7191 None,
7192 None,
7193 Some(dec!(3000.00)),
7194 );
7195 let queried_clients = IndexSet::from([client.client_id()]);
7196
7197 let plan = manager.plan_position_fill_reports(
7198 &mut check,
7199 &[report],
7200 &queried_clients,
7201 &IndexSet::new(),
7202 &clients,
7203 );
7204
7205 assert_eq!(
7206 plan.discrepancy_keys,
7207 IndexSet::from([(position.instrument_id, position.account_id)])
7208 );
7209 assert_eq!(plan.queries.len(), 1);
7210 let query = &plan.queries[0];
7211 assert_eq!(
7212 (query.key, query.client_id),
7213 (
7214 (position.instrument_id, position.account_id),
7215 client.client_id()
7216 )
7217 );
7218 assert_eq!(query.command.instrument_id, Some(position.instrument_id));
7219 assert_eq!(query.command.venue_order_id, None);
7220 assert_eq!(
7221 query.command.start,
7222 Some(query_end.saturating_sub(lookback))
7223 );
7224 assert_eq!(query.command.end, Some(query_end));
7225 assert_eq!(query.command.correlation_id, Some(check.command.command_id));
7226 assert_eq!(query.command.log_receipt_level, LogLevel::Debug);
7227 }
7228
7229 #[rstest]
7230 fn test_plan_position_fill_reports_defers_position_opened_during_request() {
7231 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7232 let cache = Rc::new(RefCell::new(Cache::default()));
7233
7234 let mut manager = ExecutionManager::new(
7235 clock,
7236 cache.clone(),
7237 ExecutionManagerConfig {
7238 position_check_threshold_ns: DurationNanos::ZERO,
7239 ..Default::default()
7240 },
7241 )
7242 .expect("valid config");
7243
7244 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7245 cache
7246 .borrow_mut()
7247 .add_instrument(instrument.clone())
7248 .unwrap();
7249 let client = PositionCoverageStubClient;
7250 let clients: [&dyn ExecutionClient; 1] = [&client];
7251 let mut check = manager.prepare_position_report_check(UUID4::new(), &clients);
7252 let position = insert_open_position(
7253 &cache,
7254 &instrument,
7255 PositionId::from("P-FILL-DURING-REQUEST"),
7256 OrderSide::Buy,
7257 "1.0",
7258 "3000.00",
7259 );
7260 manager.record_position_activity(position.instrument_id, position.account_id);
7261
7262 let report = PositionStatusReport::new(
7263 position.account_id,
7264 position.instrument_id,
7265 PositionSide::Long,
7266 Quantity::from("2.0"),
7267 UnixNanos::from(1_000_000),
7268 UnixNanos::from(1_000_000),
7269 None,
7270 None,
7271 Some(dec!(3000.00)),
7272 );
7273
7274 let plan = manager.plan_position_fill_reports(
7275 &mut check,
7276 &[report],
7277 &IndexSet::from([client.client_id()]),
7278 &IndexSet::new(),
7279 &clients,
7280 );
7281
7282 assert_eq!(
7283 plan.discrepancy_keys,
7284 IndexSet::from([(position.instrument_id, position.account_id)])
7285 );
7286 assert!(plan.queries.is_empty());
7287 assert_eq!(
7288 check
7289 .activity_revisions
7290 .get(&(position.instrument_id, position.account_id)),
7291 Some(&0)
7292 );
7293 }
7294
7295 #[rstest]
7296 fn test_prepare_position_report_check_uses_live_client_bulk_coverage() {
7297 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7298 let cache = Rc::new(RefCell::new(Cache::default()));
7299 let manager =
7300 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
7301 .expect("valid config");
7302 let derivative = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7303 let spot = test_bybit_spot_instrument();
7304 cache
7305 .borrow_mut()
7306 .add_instrument(derivative.clone())
7307 .unwrap();
7308 cache.borrow_mut().add_instrument(spot.clone()).unwrap();
7309 let derivative_position = insert_open_position(
7310 &cache,
7311 &derivative,
7312 PositionId::from("P-DERIVATIVE-COVERAGE"),
7313 OrderSide::Buy,
7314 "5.0",
7315 "3000.00",
7316 );
7317 let spot_position = insert_open_position(
7318 &cache,
7319 &spot,
7320 PositionId::from("P-SPOT-COVERAGE"),
7321 OrderSide::Buy,
7322 "2.0",
7323 "2000.00",
7324 );
7325 let client = LiveExecutionClient::new(Box::new(PositionCoverageStubClient));
7326 let client: &dyn ExecutionClient = &client;
7327
7328 let check = manager.prepare_position_report_check(UUID4::new(), &[client]);
7329 let client_id = ClientId::from("BYBIT");
7330
7331 assert_eq!(
7332 check.client_coverage.get(&(
7333 derivative_position.instrument_id,
7334 derivative_position.account_id
7335 )),
7336 Some(&ReportClientCoverage::Resolved(IndexSet::from([client_id])))
7337 );
7338 assert_eq!(
7339 check
7340 .client_coverage
7341 .get(&(spot_position.instrument_id, spot_position.account_id)),
7342 Some(&ReportClientCoverage::Unavailable(IndexSet::from([
7343 client_id
7344 ])))
7345 );
7346 }
7347 }
7348}