1use std::{
29 cell::{Ref, RefCell},
30 fmt::Debug,
31 rc::Rc,
32 str::FromStr,
33 sync::LazyLock,
34 time::Duration,
35};
36
37use indexmap::{IndexMap, IndexSet};
38use nautilus_common::{
39 cache::Cache,
40 clients::{DEFAULT_POSITION_RECONCILIATION_TOLERANCE, ExecutionClient},
41 clock::Clock,
42 config::ConfigResult,
43 enums::{LogColor, LogLevel},
44 live::dst,
45 log_info,
46 messages::{
47 ExecutionReport,
48 execution::{
49 QueryOrder, TradingCommand,
50 report::{
51 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
52 GeneratePositionStatusReports,
53 },
54 },
55 },
56 msgbus::{self, MessagingSwitchboard, switchboard},
57};
58use nautilus_core::{DurationNanos, UUID4, UnixNanos, datetime::mins_to_secs};
59use nautilus_execution::{
60 engine::ExecutionEngine,
61 reconciliation::{
62 RECONCILIATION_ORDER_TAG, ReconciliationResult as AdjustedReports,
63 calculate_reconciliation_price, create_inferred_fill_for_qty,
64 create_reconciliation_rejected, create_reconciliation_triggered,
65 fill_precedes_snapshot_reconciled_position,
66 generate_external_order_status_events_with_commission,
67 generate_reconciliation_order_pre_fill_events,
68 generate_reconciliation_order_snapshot_events_with_commission,
69 incremental_inferred_fill_price_and_liquidity, inferred_fill_price_and_liquidity,
70 position_prices_match, process_mass_status_for_reconciliation,
71 process_mass_status_for_reconciliation_without_synthetic_reports,
72 reconcile_order_report_with_commission,
73 },
74};
75use nautilus_model::{
76 enums::{AvgPxReconciliation, OmsType, OrderSide, OrderStatus},
77 events::{OrderEventAny, OrderFilled, OrderInitialized, OrderUpdated},
78 identifiers::{
79 AccountId, ClientId, ClientOrderId, InstrumentId, PositionId, StrategyId, TradeId,
80 TraderId, VenueOrderId,
81 },
82 instruments::{Instrument, InstrumentAny},
83 orders::{Order, OrderAny, TRIGGERABLE_ORDER_TYPES},
84 position::{Position, PositionReplayEvent},
85 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
86 types::{Money, Quantity},
87};
88use rust_decimal::Decimal;
89use ustr::Ustr;
90
91pub(crate) use super::reconciliation::{
92 OpenOrderReconciliationResult, OpenOrderReportCheck, SourcedOrderStatusReport,
93 TargetedOrderQuery, TargetedOrderReportResult, request_targeted_order_reports,
94 resolve_position_report_client_coverage,
95};
96pub use super::{
97 config::ExecutionManagerConfig,
98 reconciliation::{
99 ExternalOrderMetadata, InflightCheckResult, InstrumentAccountKey, PositionFillReportPlan,
100 PositionFillReportPreparation, PositionFillReportQuery, PositionReportCheck,
101 ReconciliationResult, ReportClientCoverage,
102 },
103};
104use super::{
105 recency::RecencyMap,
106 reconciliation::{
107 FillKey, InflightCheck, PositionQuantityComparison, PositionReconciliationState,
108 PositionReportShape, ReconciliationFillQueue, RetainedFillState,
109 create_orphan_fill_order_report, create_position_reconciliation_report,
110 create_reconciliation_canceled, distinct_position_reports, has_active_inferred_fill,
111 is_exact_order_match, position_avg_px, position_qty_aggregates,
112 resolve_inferred_fill_commission, responsible_client_ids, should_project_fill,
113 sort_reconciliation_events, terminal_report_has_missing_fills,
114 },
115 submission::{SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource},
116};
117
118static TAG_VENUE: LazyLock<Ustr> = LazyLock::new(|| Ustr::from("VENUE"));
120
121static TAG_RECONCILIATION: LazyLock<Ustr> = LazyLock::new(|| Ustr::from(RECONCILIATION_ORDER_TAG));
123
124const LOG_SAMPLES_MAX: usize = 5;
125
126#[derive(Clone, Debug)]
127struct SubmissionRecoveryState {
128 trader_id: TraderId,
129 client_id: Option<ClientId>,
130 strategy_id: StrategyId,
131 instrument_id: InstrumentId,
132 exhausted: bool,
133 venue_confirmed: bool,
134}
135
136#[derive(Clone)]
150pub struct ExecutionManager {
151 clock: Rc<RefCell<dyn Clock>>,
152 cache: Rc<RefCell<Cache>>,
153 config: ExecutionManagerConfig,
154
155 order_activity: RecencyMap<ClientOrderId>,
156 order_inflight_checks: IndexMap<ClientOrderId, InflightCheck>,
157 submissions: IndexMap<ClientOrderId, SubmissionRecoveryState>,
158 submission_exhaustions: Vec<SubmissionRecoveryExhausted>,
159 submission_retention: IndexMap<ClientId, AccountId>,
160 order_query_recency: RecencyMap<ClientOrderId>,
161 order_query_pending: IndexSet<ClientOrderId>,
162 order_recon_retries: IndexMap<ClientOrderId, u32>,
163 order_coverage_unresolved: IndexSet<ClientOrderId>,
164 order_coverage_warnings: IndexSet<ClientOrderId>,
165 order_lookback_warnings: IndexSet<ClientOrderId>,
166
167 fills_processed: RecencyMap<FillKey>,
168 fills_recent: RecencyMap<FillKey>,
169 fills_unapplied: RecencyMap<FillKey>,
170
171 position_activity: RecencyMap<InstrumentAccountKey>,
172 position_activity_revisions: IndexMap<InstrumentAccountKey, u64>,
173 position_recon: IndexMap<InstrumentAccountKey, PositionReconciliationState>,
174 position_recon_tolerances: IndexMap<AccountId, Decimal>,
175}
176
177impl Debug for ExecutionManager {
178 #[rustfmt::skip]
179 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
180 f.debug_struct(stringify!(ExecutionManager))
181 .field("clock", &self.clock)
182 .field("cache", &self.cache)
183 .field("config", &self.config)
184 .field("order_activity", &self.order_activity)
185 .field("order_inflight_checks", &self.order_inflight_checks)
186 .field("submissions", &self.submissions)
187 .field("submission_exhaustions", &self.submission_exhaustions)
188 .field("submission_retention", &self.submission_retention)
189 .field("order_query_recency", &self.order_query_recency)
190 .field("order_query_pending", &self.order_query_pending)
191 .field("order_recon_retries", &self.order_recon_retries)
192 .field("order_coverage_unresolved", &self.order_coverage_unresolved)
193 .field("order_coverage_warnings", &self.order_coverage_warnings)
194 .field("order_lookback_warnings", &self.order_lookback_warnings)
195 .field("fills_processed", &self.fills_processed)
196 .field("fills_recent", &self.fills_recent)
197 .field("fills_unapplied", &self.fills_unapplied)
198 .field("position_activity", &self.position_activity)
199 .field("position_activity_revisions", &self.position_activity_revisions)
200 .field("position_recon", &self.position_recon)
201 .field("position_recon_tolerances", &self.position_recon_tolerances)
202 .finish()
203 }
204}
205
206impl ExecutionManager {
207 pub fn new(
213 clock: Rc<RefCell<dyn Clock>>,
214 cache: Rc<RefCell<Cache>>,
215 config: ExecutionManagerConfig,
216 ) -> ConfigResult<Self> {
217 config.validate()?;
218
219 Ok(Self {
220 clock,
221 cache,
222 config,
223 order_activity: RecencyMap::default(),
224 order_inflight_checks: IndexMap::new(),
225 submissions: IndexMap::new(),
226 submission_exhaustions: Vec::new(),
227 submission_retention: IndexMap::new(),
228 order_query_recency: RecencyMap::default(),
229 order_query_pending: IndexSet::new(),
230 order_recon_retries: IndexMap::new(),
231 order_coverage_unresolved: IndexSet::new(),
232 order_coverage_warnings: IndexSet::new(),
233 order_lookback_warnings: IndexSet::new(),
234 fills_processed: RecencyMap::default(),
235 fills_recent: RecencyMap::default(),
236 fills_unapplied: RecencyMap::default(),
237 position_activity: RecencyMap::default(),
238 position_activity_revisions: IndexMap::new(),
239 position_recon: IndexMap::new(),
240 position_recon_tolerances: IndexMap::new(),
241 })
242 }
243
244 pub(crate) const fn config(&self) -> &ExecutionManagerConfig {
246 &self.config
247 }
248
249 pub(crate) fn timestamp_ns(&self) -> UnixNanos {
251 self.clock.borrow().timestamp_ns()
252 }
253
254 pub(crate) fn cache(&self) -> Ref<'_, Cache> {
256 self.cache.borrow()
257 }
258
259 pub fn register_inflight(&mut self, client_order_id: ClientOrderId) {
264 self.confirm_submission_outcome(&client_order_id);
265
266 if self.submission_recovery_pending(client_order_id)
267 && (self.order_inflight_checks.contains_key(&client_order_id)
268 || self.submission_is_retained(client_order_id))
269 {
270 return;
271 }
272
273 self.start_inflight_check(client_order_id);
274 }
275
276 pub fn register_submission(
282 &mut self,
283 initialized: &OrderInitialized,
284 client_id: Option<ClientId>,
285 ) {
286 let client_order_id = initialized.client_order_id;
287
288 if self.submissions.contains_key(&client_order_id)
289 || self.submission_retention_enabled(client_order_id, client_id)
290 {
291 self.confirm_submission_outcome(&client_order_id);
292
293 if self.submissions.contains_key(&client_order_id)
294 || self.submission_is_acknowledged(client_order_id)
295 {
296 return;
297 }
298
299 self.track_submission(initialized, client_id);
300 }
301
302 self.start_inflight_check(client_order_id);
303 }
304
305 fn start_inflight_check(&mut self, client_order_id: ClientOrderId) {
306 if self
307 .config
308 .filtered_client_order_ids
309 .contains(&client_order_id)
310 {
311 return;
312 }
313
314 self.order_inflight_checks.insert(
315 client_order_id,
316 InflightCheck {
317 submitted_at: dst::time::Instant::now(),
318 retry_count: 0,
319 last_query_at: None,
320 },
321 );
322
323 self.order_recon_retries.insert(client_order_id, 0);
324 self.order_query_recency.remove(&client_order_id);
325 self.order_activity.remove(&client_order_id);
326 }
327
328 fn track_submission(&mut self, initialized: &OrderInitialized, client_id: Option<ClientId>) {
329 if !self.submission_retention_enabled(initialized.client_order_id, client_id)
330 || self
331 .config
332 .filtered_client_order_ids
333 .contains(&initialized.client_order_id)
334 {
335 return;
336 }
337
338 self.submissions
339 .entry(initialized.client_order_id)
340 .or_insert(SubmissionRecoveryState {
341 trader_id: initialized.trader_id,
342 client_id,
343 strategy_id: initialized.strategy_id,
344 instrument_id: initialized.instrument_id,
345 exhausted: false,
346 venue_confirmed: false,
347 });
348 }
349
350 fn submission_is_unacknowledged(order: &OrderAny) -> bool {
351 !order.events().iter().any(|event| {
352 matches!(
353 event,
354 OrderEventAny::Accepted(_)
355 | OrderEventAny::Triggered(_)
356 | OrderEventAny::Filled(_)
357 | OrderEventAny::FillVoided(_)
358 | OrderEventAny::Rejected(_)
359 | OrderEventAny::Denied(_)
360 | OrderEventAny::Canceled(_)
361 | OrderEventAny::Expired(_)
362 ) || matches!(event, OrderEventAny::Updated(updated) if updated.venue_order_id.is_some())
363 })
364 }
365
366 fn submission_is_acknowledged(&self, client_order_id: ClientOrderId) -> bool {
367 self.cache
368 .borrow()
369 .order_ref(&client_order_id)
370 .is_some_and(|order| !Self::submission_is_unacknowledged(&order))
371 }
372
373 fn submission_recovery_pending(&self, client_order_id: ClientOrderId) -> bool {
374 self.submissions
375 .get(&client_order_id)
376 .is_some_and(|submission| !submission.venue_confirmed)
377 && !self.submission_is_acknowledged(client_order_id)
378 }
379
380 #[cfg(feature = "node")]
386 pub(crate) fn unresolved_submission_ids(&self) -> Vec<ClientOrderId> {
387 self.submissions
388 .keys()
389 .copied()
390 .filter(|client_order_id| self.submission_recovery_pending(*client_order_id))
391 .collect()
392 }
393
394 pub fn confirm_submission_outcome(&mut self, client_order_id: &ClientOrderId) {
399 let Some(submission) = self.submissions.get(client_order_id) else {
400 return;
401 };
402
403 let venue_confirmed = submission.venue_confirmed;
404
405 if !self.submission_is_acknowledged(*client_order_id) {
406 return;
407 }
408
409 if venue_confirmed {
410 self.submissions.shift_remove(client_order_id);
412 return;
413 }
414
415 self.clear_recon_tracking(client_order_id, true);
416 self.record_local_activity(*client_order_id);
417 }
418
419 fn exhaust_submission(
420 &mut self,
421 client_order_id: ClientOrderId,
422 source: SubmissionRecoverySource,
423 retry_count: u32,
424 ) -> bool {
425 if !self.submissions.contains_key(&client_order_id)
426 && !self.submission_retention_enabled(client_order_id, None)
427 {
428 return false;
429 }
430
431 let budget = match source {
432 SubmissionRecoverySource::Inflight => self.config.inflight_max_retries,
433 SubmissionRecoverySource::MissingOrder => self.config.open_check_missing_retries,
434 };
435
436 if retry_count < budget {
437 return false;
438 }
439
440 if let Some(order) = self.get_order(client_order_id) {
441 if !Self::submission_is_unacknowledged(&order) {
442 return false;
443 }
444
445 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
446 self.track_submission(order.init_event(), client_id);
447 }
448
449 let Some(submission) = self.submissions.get_mut(&client_order_id) else {
450 return false;
451 };
452
453 if submission.venue_confirmed {
454 return false;
455 }
456
457 if !submission.exhausted {
458 submission.exhausted = true;
459 self.submission_exhaustions
460 .push(SubmissionRecoveryExhausted {
461 trader_id: submission.trader_id,
462 client_id: submission
463 .client_id
464 .or_else(|| self.cache.borrow().client_id(&client_order_id).copied()),
465 strategy_id: submission.strategy_id,
466 instrument_id: submission.instrument_id,
467 client_order_id,
468 source,
469 retry_count,
470 ts_event: self.clock.borrow().timestamp_ns(),
471 });
472
473 log::warn!(
474 "Submission recovery exhausted for {client_order_id}, retaining unacknowledged order pending venue evidence"
475 );
476 }
477
478 self.clear_recon_tracking(&client_order_id, true);
479 true
480 }
481
482 pub fn take_submission_recovery_exhaustions(&mut self) -> Vec<SubmissionRecoveryExhausted> {
489 std::mem::take(&mut self.submission_exhaustions)
490 }
491
492 pub fn record_local_activity(&mut self, client_order_id: ClientOrderId) {
499 self.order_activity.mark(client_order_id);
500 }
501
502 #[must_use]
506 pub fn recon_check_retry_count(&self, client_order_id: &ClientOrderId) -> u32 {
507 self.order_recon_retries
508 .get(client_order_id)
509 .copied()
510 .unwrap_or(0)
511 }
512
513 pub(crate) fn remove_targeted_order_queries(&mut self, client_order_ids: &[ClientOrderId]) {
515 for client_order_id in client_order_ids {
516 self.order_query_pending.shift_remove(client_order_id);
517 }
518 }
519
520 pub fn clear_recon_tracking(&mut self, client_order_id: &ClientOrderId, drop_last_query: bool) {
526 let pending = self.submission_recovery_pending(*client_order_id);
527 if pending && !self.submission_is_retained(*client_order_id) {
528 return;
529 }
530
531 self.order_inflight_checks.shift_remove(client_order_id);
532
533 let keep_confirmation = self
534 .submissions
535 .get(client_order_id)
536 .is_some_and(|submission| submission.venue_confirmed)
537 && self
538 .cache
539 .borrow()
540 .order_ref(client_order_id)
541 .is_some_and(|order| Self::submission_is_unacknowledged(&order));
542
543 if !pending && !keep_confirmation {
544 self.submissions.shift_remove(client_order_id);
545 }
546
547 self.order_recon_retries.shift_remove(client_order_id);
548 self.order_coverage_warnings.shift_remove(client_order_id);
549 self.order_lookback_warnings.shift_remove(client_order_id);
550 self.order_coverage_unresolved.shift_remove(client_order_id);
551 self.remove_targeted_order_queries(&[*client_order_id]);
552
553 if drop_last_query {
554 self.order_query_recency.remove(client_order_id);
555 }
556
557 self.order_activity.remove(client_order_id);
558 }
559
560 pub fn prune_order_local_activity(&mut self) {
562 self.order_activity
563 .prune_older_than(Duration::from(self.config.open_check_threshold_ns));
564 }
565
566 #[must_use]
568 pub fn is_fill_recently_processed(
569 &self,
570 account_id: AccountId,
571 instrument_id: InstrumentId,
572 trade_id: TradeId,
573 ) -> bool {
574 self.fills_recent
575 .contains_key(&(account_id, instrument_id, trade_id))
576 }
577
578 pub fn commit_recent_fill_if_applied(&mut self, fill: &OrderFilled) {
580 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
581 if self.is_fill_applied(fill, fill_key) {
582 self.mark_fill_processed(fill_key.0, fill_key.1, fill_key.2);
583 }
584 }
585
586 pub fn mark_fill_processed(
588 &mut self,
589 account_id: AccountId,
590 instrument_id: InstrumentId,
591 trade_id: TradeId,
592 ) {
593 self.fills_recent
594 .mark((account_id, instrument_id, trade_id));
595 }
596
597 pub fn prune_recent_fills_cache(&mut self, ttl_secs: f64) {
601 let ttl = match Duration::try_from_secs_f64(ttl_secs) {
609 Ok(ttl) => ttl,
610 Err(_) if ttl_secs > 0.0 => Duration::MAX,
611 Err(_) => Duration::ZERO,
612 };
613
614 self.fills_recent.prune_older_than(ttl);
615 }
616
617 pub fn prune_processed_fills(&mut self) {
622 let Some(lookback_mins) = self.config.lookback_mins else {
623 return;
624 };
625
626 let ttl = Duration::from_mins(lookback_mins).max(Duration::from_mins(1));
627 self.fills_processed.prune_older_than(ttl);
628 }
629
630 pub(crate) fn set_position_reconciliation_tolerance(
632 &mut self,
633 account_id: AccountId,
634 tolerance: Decimal,
635 ) {
636 let tolerance = if tolerance < Decimal::ZERO {
637 log::error!(
638 "Invalid negative position reconciliation tolerance {tolerance} for \
639 {account_id}; using the default"
640 );
641 DEFAULT_POSITION_RECONCILIATION_TOLERANCE
642 } else {
643 tolerance
644 };
645
646 self.position_recon_tolerances.insert(account_id, tolerance);
647 }
648
649 pub(crate) fn position_reconciliation_tolerance(&self, account_id: AccountId) -> Decimal {
651 self.position_recon_tolerances
652 .get(&account_id)
653 .copied()
654 .unwrap_or(DEFAULT_POSITION_RECONCILIATION_TOLERANCE)
655 }
656
657 pub fn record_position_activity(&mut self, instrument_id: InstrumentId, account_id: AccountId) {
663 let key = (instrument_id, account_id);
664 self.position_activity.mark(key);
665 let revision = self.position_activity_revisions.entry(key).or_default();
666 *revision = revision.saturating_add(1);
667 }
668
669 pub(crate) fn position_activity_is_recent(&self, key: &InstrumentAccountKey) -> bool {
671 self.position_activity
672 .within(key, Duration::from(self.config.position_check_threshold_ns))
673 }
674
675 pub(crate) fn position_activity_revision(&self, key: &InstrumentAccountKey) -> u64 {
677 self.position_activity_revisions
678 .get(key)
679 .copied()
680 .unwrap_or_default()
681 }
682
683 fn set_position_reconciliation_retries(
684 &mut self,
685 key: InstrumentAccountKey,
686 report_shape: PositionReportShape,
687 retries: u32,
688 ) {
689 self.position_recon.insert(
690 key,
691 PositionReconciliationState {
692 report_shape,
693 retries,
694 },
695 );
696 }
697
698 fn position_reconciliation_retries(
700 &self,
701 key: &InstrumentAccountKey,
702 report_shape: PositionReportShape,
703 ) -> u32 {
704 self.position_recon
705 .get(key)
706 .filter(|state| state.report_shape == report_shape)
707 .map_or(0, |state| state.retries)
708 }
709
710 #[must_use]
713 pub fn position_recon_retry_count(&self, key: &InstrumentAccountKey) -> u32 {
714 self.position_recon
715 .get(key)
716 .map_or(0, |state| state.retries)
717 }
718
719 pub(crate) fn clear_position_reconciliation(&mut self, key: &InstrumentAccountKey) {
721 self.position_recon.shift_remove(key);
722 }
723
724 pub(crate) fn retain_position_reconciliation(
726 &mut self,
727 active_keys: &IndexSet<InstrumentAccountKey>,
728 ) {
729 self.position_recon
730 .retain(|key, _| active_keys.contains(key));
731 }
732
733 pub fn reconcile_execution_mass_status(
739 &mut self,
740 mass_status: &ExecutionMassStatus,
741 exec_engine: &RefCell<ExecutionEngine>,
742 ) -> ReconciliationResult {
743 if exec_engine
744 .borrow()
745 .get_client(&mass_status.client_id)
746 .is_none()
747 {
748 log::error!(
749 "Cannot reconcile ExecutionMassStatus from unknown client {}",
750 mass_status.client_id
751 );
752 return ReconciliationResult::default();
753 }
754
755 self.validate_mass_status_order_sources(mass_status);
756
757 let raw_order_status_topic =
762 MessagingSwitchboard::reconciliation_raw_order_status_report_topic();
763
764 for report in mass_status.order_reports().values() {
765 msgbus::publish_any(raw_order_status_topic, report);
766 }
767
768 let raw_fill_topic = MessagingSwitchboard::reconciliation_raw_fill_report_topic();
769
770 for fills in mass_status.fill_reports().values() {
771 for fill in fills {
772 msgbus::publish_any(raw_fill_topic, fill);
773 }
774 }
775
776 let raw_position_topic =
777 MessagingSwitchboard::reconciliation_raw_position_status_report_topic();
778
779 for reports in mass_status.position_reports().values() {
780 for report in reports {
781 msgbus::publish_any(raw_position_topic, report);
782 }
783 }
784
785 if exec_engine
786 .borrow()
787 .get_client(&mass_status.client_id)
788 .is_none()
789 {
790 log::error!(
791 "Execution client {} disappeared while publishing raw mass status reports",
792 mass_status.client_id
793 );
794 return ReconciliationResult::default();
795 }
796
797 let venue = mass_status.venue;
798 let order_count = mass_status.order_reports().len();
799 let fill_count: usize = mass_status.fill_reports().values().map(Vec::len).sum();
800 let position_count: usize = mass_status.position_reports().values().map(Vec::len).sum();
801
802 log_info!(
803 "Reconciling ExecutionMassStatus for {venue}",
804 color = LogColor::Blue
805 );
806 log_info!(
807 "Received {order_count} order(s), {fill_count} fill(s), {position_count} position(s)",
808 color = LogColor::Blue
809 );
810
811 if mass_status.lookback_start().is_some() && !mass_status.reports_complete() {
812 log::warn!("Received incomplete bounded reconciliation reports for {venue}");
813 }
814
815 let retained_fill_state = self.retained_fill_state();
816 let reported_fill_keys: IndexSet<FillKey> = mass_status
817 .fill_reports()
818 .values()
819 .flatten()
820 .filter(|fill| !fill.last_qty.is_zero())
821 .map(|fill| (fill.account_id, fill.instrument_id, fill.trade_id))
822 .collect();
823
824 let order_venue_ids = exec_engine
825 .borrow()
826 .get_client(&mass_status.client_id)
827 .and_then(|client| client.venue_order_ids(mass_status));
828 let mut adjusted_reports = self.adjust_mass_status_fills(mass_status);
829 Self::preserve_external_order_history(
830 mass_status,
831 order_venue_ids.as_ref(),
832 &mut adjusted_reports,
833 );
834 let AdjustedReports {
835 orders: adjusted_order_reports,
836 fills: adjusted_fill_reports,
837 mut order_only_ids,
838 order_only_fill_keys,
839 } = adjusted_reports;
840
841 order_only_ids.extend(self.order_only_ids(
842 mass_status,
843 &adjusted_order_reports,
844 &adjusted_fill_reports,
845 ));
846
847 let mut events = Vec::new();
848 let mut external_orders = Vec::new();
849 let mut orders_reconciled = 0usize;
850 let mut external_orders_created = 0usize;
851 let mut open_orders_initialized = 0usize;
852 let mut orders_skipped_no_instrument = 0usize;
853 let mut orders_skipped_duplicate = 0usize;
854 let mut fills_applied = 0usize;
855 let mut fill_queue = ReconciliationFillQueue::default();
856
857 let fill_reports = &adjusted_fill_reports;
858 let mut seen_fill_keys: IndexSet<FillKey> = IndexSet::new();
859
860 for fills in fill_reports.values() {
861 for fill in fills {
862 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
863 if !seen_fill_keys.insert(fill_key) {
864 log::warn!(
865 "Duplicate trade_id {} for {} in mass status",
866 fill.trade_id,
867 fill.instrument_id
868 );
869 }
870 }
871 }
872
873 let order_reports = &adjusted_order_reports;
874 let mut orders_skipped_filtered = 0usize;
875 let mut orders_projected: IndexMap<ClientOrderId, OrderAny> = IndexMap::new();
876
877 for report in order_reports.values().filter(|report| {
878 order_venue_ids
879 .as_ref()
880 .and_then(|histories| report.client_order_id.and_then(|id| histories.get(&id)))
881 .is_none_or(|ids| ids.last() == Some(&report.venue_order_id))
882 }) {
883 if self.should_skip_order_report(report) {
884 orders_skipped_filtered += 1;
885 continue;
886 }
887
888 let order = report
889 .client_order_id
890 .and_then(|id| {
891 orders_projected
892 .get(&id)
893 .cloned()
894 .or_else(|| self.get_order(id))
895 })
896 .or_else(|| {
897 self.get_order_by_venue_order_id(report.venue_order_id)
898 .map(|order| {
899 orders_projected
900 .get(&order.client_order_id())
901 .cloned()
902 .unwrap_or(order)
903 })
904 });
905 let order_client = order
906 .as_ref()
907 .filter(|order| Some(order.client_order_id()) == report.client_order_id);
908 let events_start = events.len();
909
910 if let Some(client_order_id) = &report.client_order_id {
911 if let Some(cached_order) = order_client
912 && is_exact_order_match(cached_order, report)
913 {
914 log::debug!("Skipping order {client_order_id}: already in sync with venue");
915 orders_skipped_duplicate += 1;
916
917 if let Err(e) = self
919 .cache
920 .borrow_mut()
921 .index_venue_order_id(client_order_id, &report.venue_order_id)
922 {
923 log::warn!("Failed to index venue order ID: {e}");
924 }
925
926 continue;
927 }
928
929 if let Some(cached_order) = order_client
931 && cached_order.is_closed()
932 && cached_order
933 .tags()
934 .is_some_and(|tags| tags.contains(&*TAG_RECONCILIATION))
935 {
936 log::debug!(
937 "Skipping closed reconciliation order {client_order_id}: \
938 synthetic position adjustment from previous session",
939 );
940 orders_skipped_duplicate += 1;
941 continue;
942 }
943
944 if let Some(order) = order_client {
945 let instrument = self.get_instrument(&report.instrument_id);
946 log::info!(
947 color = LogColor::Blue as u8;
948 "Reconciling {} {} {} [{}] -> [{}]",
949 client_order_id,
950 report.venue_order_id,
951 report.instrument_id,
952 order.status(),
953 report.order_status,
954 );
955
956 let order_fills: Vec<&FillReport> = fill_reports
957 .get(&report.venue_order_id)
958 .map(|f| f.iter().collect())
959 .unwrap_or_default();
960 let engine_ref = exec_engine.borrow();
961 let commission_client = engine_ref.get_client(&mass_status.client_id);
962
963 let order_events = self.reconcile_order_with_fills(
964 true,
965 order,
966 report,
967 &order_fills,
968 instrument.as_ref(),
969 &mut fill_queue,
970 commission_client,
971 );
972
973 drop(engine_ref);
974
975 orders_reconciled += usize::from(!order_events.is_empty());
976 fills_applied += order_events
977 .iter()
978 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
979 .count();
980 events.extend(order_events);
981
982 if let Err(e) = self
984 .cache
985 .borrow_mut()
986 .index_venue_order_id(client_order_id, &report.venue_order_id)
987 {
988 log::warn!("Failed to index venue order ID: {e}");
989 }
990 } else if let Some(order) = order.as_ref() {
991 let instrument = self.get_instrument(&report.instrument_id);
993
994 log::info!(
995 color = LogColor::Blue as u8;
996 "Reconciling {} (matched by venue_order_id {}) {} [{}] -> [{}]",
997 order.client_order_id(),
998 report.venue_order_id,
999 report.instrument_id,
1000 order.status(),
1001 report.order_status,
1002 );
1003
1004 let order_fills: Vec<&FillReport> = fill_reports
1005 .get(&report.venue_order_id)
1006 .map(|f| f.iter().collect())
1007 .unwrap_or_default();
1008 let engine_ref = exec_engine.borrow();
1009 let commission_client = engine_ref.get_client(&mass_status.client_id);
1010
1011 let order_events = self.reconcile_order_with_fills(
1012 true,
1013 order,
1014 report,
1015 &order_fills,
1016 instrument.as_ref(),
1017 &mut fill_queue,
1018 commission_client,
1019 );
1020
1021 drop(engine_ref);
1022
1023 orders_reconciled += usize::from(!order_events.is_empty());
1024 fills_applied += order_events
1025 .iter()
1026 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
1027 .count();
1028 events.extend(order_events);
1029
1030 if let Err(e) = self
1031 .cache
1032 .borrow_mut()
1033 .index_venue_order_id(&order.client_order_id(), &report.venue_order_id)
1034 {
1035 log::warn!("Failed to index venue order ID: {e}");
1036 }
1037 } else if let Some(instrument) = self.get_instrument(&report.instrument_id) {
1038 let venue_order_ids = order_venue_ids
1039 .as_ref()
1040 .and_then(|histories| histories.get(client_order_id));
1041 let order_fills: Vec<&FillReport> = venue_order_ids
1042 .map_or(std::slice::from_ref(&report.venue_order_id), Vec::as_slice)
1043 .iter()
1044 .filter_map(|id| fill_reports.get(id))
1045 .flatten()
1046 .collect();
1047 let engine_ref = exec_engine.borrow();
1048 let commission_client = engine_ref.get_client(&mass_status.client_id);
1049
1050 let (mut external_events, metadata) = self.handle_external_order(
1051 report,
1052 mass_status.account_id,
1053 &instrument,
1054 &order_fills,
1055 false, Some(&mut fill_queue),
1057 commission_client,
1058 );
1059
1060 drop(engine_ref);
1061
1062 self.restore_external_order_venue_ids(
1063 report,
1064 venue_order_ids.map(Vec::as_slice),
1065 &mut external_events,
1066 );
1067
1068 if !external_events.is_empty() {
1069 external_orders_created += 1;
1070 fills_applied += external_events
1071 .iter()
1072 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
1073 .count();
1074
1075 if report.order_status.is_open() {
1076 open_orders_initialized += 1;
1077 }
1078
1079 events.extend(external_events);
1080
1081 if let Some(m) = metadata {
1082 external_orders.push(m);
1083 }
1084 }
1085 } else {
1086 orders_skipped_no_instrument += 1;
1087 }
1088 } else if let Some(order) = order.as_ref() {
1089 let instrument = self.get_instrument(&report.instrument_id);
1091 log::info!(
1092 color = LogColor::Blue as u8;
1093 "Reconciling {} (matched by venue_order_id {}) {} [{}] -> [{}]",
1094 order.client_order_id(),
1095 report.venue_order_id,
1096 report.instrument_id,
1097 order.status(),
1098 report.order_status,
1099 );
1100
1101 let order_fills: Vec<&FillReport> = fill_reports
1102 .get(&report.venue_order_id)
1103 .map(|f| f.iter().collect())
1104 .unwrap_or_default();
1105 let engine_ref = exec_engine.borrow();
1106 let commission_client = engine_ref.get_client(&mass_status.client_id);
1107
1108 let order_events = self.reconcile_order_with_fills(
1109 true,
1110 order,
1111 report,
1112 &order_fills,
1113 instrument.as_ref(),
1114 &mut fill_queue,
1115 commission_client,
1116 );
1117
1118 drop(engine_ref);
1119
1120 orders_reconciled += usize::from(!order_events.is_empty());
1121 fills_applied += order_events
1122 .iter()
1123 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
1124 .count();
1125 events.extend(order_events);
1126
1127 if let Err(e) = self
1128 .cache
1129 .borrow_mut()
1130 .index_venue_order_id(&order.client_order_id(), &report.venue_order_id)
1131 {
1132 log::warn!("Failed to index venue order ID: {e}");
1133 }
1134 } else if let Some(instrument) = self.get_instrument(&report.instrument_id) {
1135 let is_synthetic = report.venue_order_id.as_str().starts_with("S-");
1137
1138 let order_fills: Vec<&FillReport> = fill_reports
1139 .get(&report.venue_order_id)
1140 .map(|f| f.iter().collect())
1141 .unwrap_or_default();
1142 let engine_ref = exec_engine.borrow();
1143 let commission_client = engine_ref.get_client(&mass_status.client_id);
1144
1145 let (external_events, metadata) = self.handle_external_order(
1146 report,
1147 mass_status.account_id,
1148 &instrument,
1149 &order_fills,
1150 is_synthetic,
1151 Some(&mut fill_queue),
1152 commission_client,
1153 );
1154
1155 drop(engine_ref);
1156
1157 if !external_events.is_empty() {
1158 external_orders_created += 1;
1159 fills_applied += external_events
1160 .iter()
1161 .filter(|e| matches!(e, OrderEventAny::Filled(_)))
1162 .count();
1163
1164 if report.order_status.is_open() {
1165 open_orders_initialized += 1;
1166 }
1167
1168 events.extend(external_events);
1169
1170 if let Some(m) = metadata {
1171 external_orders.push(m);
1172 }
1173 }
1174 } else {
1175 orders_skipped_no_instrument += 1;
1176 }
1177
1178 if events.len() > events_start
1179 && let Some(mut order) = order
1180 {
1181 for event in &events[events_start..] {
1182 if event.client_order_id() == order.client_order_id()
1183 && let Err(e) = order.apply(event.clone())
1184 {
1185 log::warn!("Cannot project queued reconciliation event: {e}");
1186 }
1187 }
1188 orders_projected.insert(order.client_order_id(), order);
1189 }
1190 }
1191
1192 let mut dropped_fill_groups: Vec<(VenueOrderId, InstrumentId, usize)> = Vec::new();
1194
1195 for (venue_order_id, fills) in fill_reports {
1196 if order_reports.contains_key(venue_order_id) {
1197 continue;
1198 }
1199
1200 let Some(first_fill) = fills.first() else {
1201 continue;
1202 };
1203
1204 if !self.should_reconcile_instrument(&first_fill.instrument_id) {
1205 log::debug!(
1206 "Skipping orphan fills for {}: not in reconciliation_instrument_ids",
1207 first_fill.instrument_id
1208 );
1209 continue;
1210 }
1211
1212 if let Some(client_order_id) = &first_fill.client_order_id
1214 && self
1215 .config
1216 .filtered_client_order_ids
1217 .contains(client_order_id)
1218 {
1219 log::debug!(
1220 "Skipping orphan fills for {client_order_id}: in filtered_client_order_ids"
1221 );
1222 continue;
1223 }
1224
1225 let order = first_fill
1226 .client_order_id
1227 .as_ref()
1228 .and_then(|id| self.get_order(*id))
1229 .or_else(|| self.get_order_by_venue_order_id(*venue_order_id));
1230
1231 if let Some(ref order) = order
1233 && self
1234 .config
1235 .filtered_client_order_ids
1236 .contains(&order.client_order_id())
1237 {
1238 log::debug!(
1239 "Skipping orphan fills for {}: in filtered_client_order_ids",
1240 order.client_order_id()
1241 );
1242 continue;
1243 }
1244
1245 if let Some(order) = order {
1246 let instrument_id = order.instrument_id();
1247 if let Some(instrument) = self.get_instrument(&instrument_id) {
1248 let mut sorted_fills: Vec<&FillReport> = fills.iter().collect();
1249 sorted_fills.sort_by_key(|f| f.ts_event);
1250
1251 for fill in sorted_fills {
1252 if let Some((event, fill_key)) = self.create_order_fill(
1253 &order,
1254 fill,
1255 &instrument,
1256 &fill_queue.pending_fill_keys,
1257 ) {
1258 fills_applied += 1;
1259 fill_queue.push(&mut events, event, fill_key);
1260 }
1261 }
1262 } else {
1263 orders_skipped_no_instrument += 1;
1264 }
1265 } else if fills.iter().any(FillReport::has_venue_position_id) {
1266 if !self.config.generate_missing_orders {
1267 log::debug!(
1268 "Skipping orphan fills for venue order {venue_order_id}: \
1269 `generate_missing_orders` is disabled"
1270 );
1271 orders_skipped_filtered += 1;
1272 continue;
1273 }
1274
1275 let Some(instrument) = self.get_instrument(&first_fill.instrument_id) else {
1276 orders_skipped_no_instrument += 1;
1277 continue;
1278 };
1279
1280 let mut sorted_fills: Vec<&FillReport> = fills.iter().collect();
1281 sorted_fills.sort_by_key(|fill| fill.ts_event);
1282
1283 let report = match create_orphan_fill_order_report(&sorted_fills, &instrument) {
1284 Ok(report) => report,
1285 Err(e) => {
1286 log::error!(
1287 "Cannot materialize orphan fills for venue order {venue_order_id}: {e}"
1288 );
1289
1290 continue;
1291 }
1292 };
1293
1294 let engine_ref = exec_engine.borrow();
1295 let commission_client = engine_ref.get_client(&mass_status.client_id);
1296
1297 let (external_events, metadata) = self.handle_external_order(
1298 &report,
1299 mass_status.account_id,
1300 &instrument,
1301 &sorted_fills,
1302 false,
1303 Some(&mut fill_queue),
1304 commission_client,
1305 );
1306
1307 drop(engine_ref);
1308
1309 if !external_events.is_empty() {
1310 external_orders_created += 1;
1311 fills_applied += external_events
1312 .iter()
1313 .filter(|event| matches!(event, OrderEventAny::Filled(_)))
1314 .count();
1315
1316 events.extend(external_events);
1317
1318 if let Some(metadata) = metadata {
1319 external_orders.push(metadata);
1320 }
1321 }
1322 } else {
1323 log::debug!(
1324 "Dropping {} fill(s) for venue order {venue_order_id} ({}): \
1325 no order report or cached order",
1326 fills.len(),
1327 first_fill.instrument_id,
1328 );
1329 dropped_fill_groups.push((*venue_order_id, first_fill.instrument_id, fills.len()));
1330 }
1331 }
1332
1333 sort_reconciliation_events(&mut events);
1334
1335 let mut unapplied_fill_position_ids = IndexSet::new();
1336
1337 for event in &events {
1338 if let OrderEventAny::Filled(fill) = event
1339 && should_project_fill(
1340 fill,
1341 &retained_fill_state,
1342 &reported_fill_keys,
1343 &order_only_ids,
1344 &order_only_fill_keys,
1345 )
1346 {
1347 exec_engine.borrow_mut().project_reconciliation_fill(fill);
1348 } else {
1349 exec_engine.borrow_mut().process(event);
1350 }
1351
1352 if let OrderEventAny::Filled(fill) = event
1353 && let Some(fill_key) = fill_queue.event_fill_keys.get(&fill.event_id).copied()
1354 {
1355 if self.is_fill_applied(fill, fill_key) {
1356 self.fills_processed.mark(fill_key);
1357 } else if let Some(venue_position_id) = fill.position_id {
1358 log::error!(
1359 "Skipping reconciliation for venue position {venue_position_id}: historical fill {} was not applied",
1360 fill.trade_id,
1361 );
1362
1363 unapplied_fill_position_ids.insert(venue_position_id);
1364 }
1365 }
1366 }
1367
1368 let mut positions_created = 0usize;
1369
1370 if !self.config.filter_position_reports {
1371 let instruments_with_unattributed_fills: IndexSet<InstrumentId> = mass_status
1374 .fill_reports()
1375 .values()
1376 .flatten()
1377 .filter(|f| !f.last_qty.is_zero() && f.venue_position_id.is_none())
1378 .map(|f| f.instrument_id)
1379 .chain(
1380 mass_status
1381 .order_reports()
1382 .values()
1383 .filter(|r| !r.filled_qty.is_zero() && r.venue_position_id.is_none())
1384 .map(|r| r.instrument_id),
1385 )
1386 .collect();
1387
1388 for (instrument_id, reports) in mass_status.position_reports() {
1389 if !self.should_reconcile_instrument(&instrument_id) {
1390 log::debug!(
1391 "Skipping position reports for {instrument_id}: not in reconciliation_instrument_ids"
1392 );
1393 continue;
1394 }
1395
1396 for report in reports {
1397 if report.venue_position_id.is_some_and(|venue_position_id| {
1398 unapplied_fill_position_ids.contains(&venue_position_id)
1399 }) {
1400 continue;
1401 }
1402
1403 if let Some(position_events) = self.reconcile_position_report(
1404 &report,
1405 mass_status.account_id,
1406 &instruments_with_unattributed_fills,
1407 ) {
1408 for event in position_events {
1409 exec_engine.borrow_mut().process(&event);
1410 events.push(event);
1411 }
1412
1413 positions_created += 1;
1414 }
1415 }
1416 }
1417 }
1418
1419 if orders_skipped_no_instrument > 0 {
1420 log::warn!("{orders_skipped_no_instrument} orders skipped (instrument not in cache)");
1421 }
1422
1423 if !dropped_fill_groups.is_empty() {
1424 let dropped_fill_count: usize =
1425 dropped_fill_groups.iter().map(|(_, _, count)| count).sum();
1426
1427 let samples = dropped_fill_groups
1428 .iter()
1429 .take(LOG_SAMPLES_MAX)
1430 .map(|(venue_order_id, instrument_id, _)| {
1431 format!("{venue_order_id} ({instrument_id})")
1432 })
1433 .collect::<Vec<_>>()
1434 .join(", ");
1435
1436 log::warn!(
1437 "Dropped {dropped_fill_count} fill(s) in {} fill group(s) without an order report \
1438 or cached order ({samples})",
1439 dropped_fill_groups.len(),
1440 );
1441 }
1442
1443 if orders_skipped_duplicate > 0 {
1444 log::debug!("{orders_skipped_duplicate} orders skipped (already in sync)");
1445 }
1446
1447 if orders_skipped_filtered > 0 {
1448 log::debug!("{orders_skipped_filtered} orders skipped (filtered by config)");
1449 }
1450
1451 log::info!(
1452 color = LogColor::Blue as u8;
1453 "Reconciliation complete for {venue}: reconciled={orders_reconciled}, external={external_orders_created}, open={open_orders_initialized}, fills={fills_applied}, positions={positions_created}, skipped={orders_skipped_duplicate}, filtered={orders_skipped_filtered}",
1454 );
1455
1456 ReconciliationResult {
1457 events,
1458 external_orders,
1459 unresolved_positions: self.unresolved_mass_status_positions(mass_status),
1460 }
1461 }
1462
1463 fn preserve_external_order_history(
1464 mass_status: &ExecutionMassStatus,
1465 order_venue_ids: Option<&IndexMap<ClientOrderId, Vec<VenueOrderId>>>,
1466 adjusted: &mut AdjustedReports,
1467 ) {
1468 let Some(order_venue_ids) = order_venue_ids else {
1469 return;
1470 };
1471
1472 let reports = mass_status.order_reports();
1473 let fills = mass_status.fill_reports();
1474
1475 for id in order_venue_ids.values().flatten() {
1476 if let Some(report) = reports.get(id) {
1477 adjusted.orders.entry(*id).or_insert_with(|| report.clone());
1478 }
1479
1480 let retained = adjusted.fills.entry(*id).or_default();
1481 for fill in fills.get(id).into_iter().flatten() {
1482 if !fill.last_qty.is_zero()
1483 && !retained
1484 .iter()
1485 .any(|existing| existing.trade_id == fill.trade_id)
1486 {
1487 adjusted.order_only_fill_keys.insert((
1489 fill.account_id,
1490 fill.instrument_id,
1491 fill.trade_id,
1492 ));
1493 retained.push(fill.clone());
1494 }
1495 }
1496 }
1497 }
1498
1499 fn restore_external_order_venue_ids(
1500 &self,
1501 report: &OrderStatusReport,
1502 venue_order_ids: Option<&[VenueOrderId]>,
1503 events: &mut Vec<OrderEventAny>,
1504 ) {
1505 let Some((first, remaining)) = venue_order_ids.and_then(|ids| ids.split_first()) else {
1506 return;
1507 };
1508
1509 let Some(OrderEventAny::Accepted(mut accepted)) = events.first().cloned() else {
1510 return;
1511 };
1512
1513 let Some(order) = report.client_order_id.and_then(|id| self.get_order(id)) else {
1514 return;
1515 };
1516
1517 accepted.venue_order_id = *first;
1518 let mut lifecycle = vec![OrderEventAny::Accepted(accepted)];
1519
1520 for venue_order_id in remaining {
1522 lifecycle.push(OrderEventAny::Updated(OrderUpdated::new(
1523 order.trader_id(),
1524 order.strategy_id(),
1525 order.instrument_id(),
1526 order.client_order_id(),
1527 order.quantity(),
1528 UUID4::new(),
1529 report.ts_accepted,
1530 self.clock.borrow().timestamp_ns(),
1531 true,
1532 Some(*venue_order_id),
1533 Some(accepted.account_id),
1534 order.price(),
1535 order.trigger_price(),
1536 None,
1537 order.is_quote_quantity(),
1538 )));
1539 }
1540
1541 if let Err(e) =
1542 self.cache
1543 .borrow_mut()
1544 .add_venue_order_id(&order.client_order_id(), first, true)
1545 {
1546 log::error!("Cannot initialize venue order ID history: {e}");
1547 return;
1548 }
1549
1550 events.splice(..1, lifecycle);
1551 }
1552
1553 fn unresolved_mass_status_positions(&self, mass_status: &ExecutionMassStatus) -> Vec<String> {
1554 if self.config.filter_position_reports {
1555 return Vec::new();
1556 }
1557
1558 let mut unresolved = Vec::new();
1559
1560 for (instrument_id, reports) in mass_status.position_reports() {
1561 if !self.should_reconcile_instrument(&instrument_id) {
1562 continue;
1563 }
1564
1565 let reports = distinct_position_reports(reports);
1566
1567 if self
1568 .netting_position_reports_match((instrument_id, mass_status.account_id), &reports)
1569 {
1570 continue;
1571 }
1572
1573 for report in &reports {
1574 if let Some(reason) =
1575 self.unresolved_position_report(report, mass_status.account_id, &reports)
1576 {
1577 unresolved.push(reason);
1578 }
1579 }
1580 }
1581
1582 unresolved
1583 }
1584
1585 fn netting_position_reports_match(
1586 &self,
1587 key: InstrumentAccountKey,
1588 reports: &[PositionStatusReport],
1589 ) -> bool {
1590 let comparison = self.position_quantity_comparison(key, reports);
1591 let cache = self.cache.borrow();
1592
1593 !comparison.cached_positions.is_empty()
1594 && comparison
1595 .cached_positions
1596 .iter()
1597 .all(|position| cache.oms_type(&position.id) == Some(OmsType::Netting))
1598 && comparison.quantities_match(self.position_reconciliation_tolerance(key.1))
1599 && self.position_report_prices_match(&comparison.cached_positions, reports)
1600 }
1601
1602 fn unresolved_position_report(
1603 &self,
1604 report: &PositionStatusReport,
1605 account_id: AccountId,
1606 reports: &[PositionStatusReport],
1607 ) -> Option<String> {
1608 let cache = self.cache.borrow();
1609 let instrument_id = report.instrument_id;
1610
1611 let open_positions =
1612 cache.positions_open(None, Some(&instrument_id), None, Some(&account_id), None);
1613 let is_netting = !open_positions.is_empty()
1614 && open_positions
1615 .iter()
1616 .all(|position| cache.oms_type(&position.id) == Some(OmsType::Netting));
1617
1618 let venue_position_id = report.venue_position_id.filter(|_| !is_netting);
1619 let venue_qty = report.signed_decimal_qty;
1620
1621 let cached_positions: Vec<Position> = if let Some(position_id) = venue_position_id {
1622 cache
1623 .position(&position_id)
1624 .filter(|p| p.account_id == account_id && p.instrument_id == instrument_id)
1625 .into_iter()
1626 .map(|p| (*p).clone())
1627 .collect()
1628 } else {
1629 open_positions.into_iter().map(|p| (*p).clone()).collect()
1630 };
1631
1632 let cached_qty: Decimal = cached_positions
1633 .iter()
1634 .map(Position::signed_decimal_qty)
1635 .sum();
1636
1637 let quantities_match = if is_netting {
1638 self.position_quantity_comparison((instrument_id, account_id), reports)
1639 .quantities_match(self.position_reconciliation_tolerance(account_id))
1640 } else if venue_position_id.is_some() {
1641 cached_qty == venue_qty
1642 } else {
1643 (cached_qty - venue_qty).abs() <= self.position_reconciliation_tolerance(account_id)
1644 };
1645
1646 let price_reports = if is_netting {
1647 reports
1648 } else {
1649 std::slice::from_ref(report)
1650 };
1651
1652 let prices_match = self.position_report_prices_match(&cached_positions, price_reports);
1653
1654 if quantities_match && prices_match {
1655 return None;
1656 }
1657
1658 let reason = if cache.instrument(&instrument_id).is_none() {
1659 "instrument missing from cache"
1660 } else if cache.account(&account_id).is_none() {
1661 "account missing from cache"
1662 } else if !self.config.generate_missing_orders {
1663 "generate_missing_orders is disabled"
1664 } else if report.avg_px_open.is_none() && cached_qty == Decimal::ZERO {
1665 "missing avg_px_open for position recovery"
1666 } else if quantities_match {
1667 "position recovery did not restore the reported average entry price"
1668 } else {
1669 "position recovery did not restore the reported quantity"
1670 };
1671
1672 Some(format!(
1673 "account={account_id}, instrument={instrument_id}, venue_position_id={:?}, venue_quantity={venue_qty}: {reason}",
1674 report.venue_position_id,
1675 ))
1676 }
1677
1678 fn position_report_prices_match(
1679 &self,
1680 cached_positions: &[Position],
1681 reports: &[PositionStatusReport],
1682 ) -> bool {
1683 for negative in [false, true] {
1684 let reports: Vec<_> = reports
1685 .iter()
1686 .filter(|report| {
1687 !report.signed_decimal_qty.is_zero()
1688 && report.signed_decimal_qty.is_sign_negative() == negative
1689 })
1690 .collect();
1691
1692 if !reports.iter().any(|report| report.avg_px_open.is_some())
1693 || reports.iter().any(|report| {
1694 report.avg_px_open_reconciliation == AvgPxReconciliation::OpeningOnly
1695 })
1696 {
1697 continue;
1698 }
1699
1700 let Some((venue_value, venue_qty)) = reports.iter().try_fold(
1701 (Decimal::ZERO, Decimal::ZERO),
1702 |(value, quantity), report| {
1703 let price = report.avg_px_open.filter(|price| *price > Decimal::ZERO)?;
1704 let qty = report.signed_decimal_qty.abs();
1705 Some((value + price * qty, quantity + qty))
1706 },
1707 ) else {
1708 return false;
1709 };
1710
1711 let positions: Vec<&Position> = cached_positions
1712 .iter()
1713 .filter(|position| {
1714 !position.quantity.is_zero()
1715 && position.signed_decimal_qty().is_sign_negative() == negative
1716 })
1717 .collect();
1718
1719 if positions.iter().any(|position| position.avg_px_open <= 0.0) {
1720 return false;
1721 }
1722
1723 if positions.is_empty()
1724 && venue_qty <= self.position_reconciliation_tolerance(reports[0].account_id)
1725 {
1726 continue;
1727 }
1728
1729 let Some(cached_avg_px) = position_avg_px(positions) else {
1730 return false;
1731 };
1732
1733 let venue_precision = reports
1735 .iter()
1736 .filter_map(|report| report.avg_px_open_precision)
1737 .min();
1738
1739 if !position_prices_match(
1740 cached_avg_px,
1741 venue_value / venue_qty,
1742 None,
1743 venue_precision,
1744 ) {
1745 return false;
1746 }
1747 }
1748
1749 true
1750 }
1751
1752 fn retained_fill_state(&self) -> RetainedFillState {
1753 let cache = self.cache.borrow();
1754 let positions = cache.positions(None, None, None, None, None);
1755 let mut fill_keys = IndexSet::new();
1756 let mut missing_order_ids = IndexSet::new();
1757 let mut missing_venue_order_ids = IndexSet::new();
1758 let mut netting_lifecycle_starts = IndexMap::new();
1759
1760 for position in positions {
1761 for fill in &position.events {
1762 fill_keys.insert((position.account_id, position.instrument_id, fill.trade_id));
1763 if cache.order(&fill.client_order_id).is_none() {
1764 missing_order_ids.insert((
1765 position.account_id,
1766 position.instrument_id,
1767 fill.client_order_id,
1768 ));
1769 missing_venue_order_ids.insert((
1770 position.account_id,
1771 position.instrument_id,
1772 fill.venue_order_id,
1773 ));
1774 }
1775 }
1776
1777 if cache.oms_type(&position.id) == Some(OmsType::Netting) {
1778 netting_lifecycle_starts.insert(
1779 (
1780 position.account_id,
1781 position.instrument_id,
1782 position.strategy_id,
1783 ),
1784 position.ts_opened,
1785 );
1786 }
1787 }
1788
1789 RetainedFillState {
1790 fill_keys,
1791 missing_order_ids,
1792 missing_venue_order_ids,
1793 netting_lifecycle_starts,
1794 }
1795 }
1796
1797 fn order_only_ids(
1798 &self,
1799 mass_status: &ExecutionMassStatus,
1800 order_reports: &IndexMap<VenueOrderId, OrderStatusReport>,
1801 fill_reports: &IndexMap<VenueOrderId, Vec<FillReport>>,
1802 ) -> IndexSet<VenueOrderId> {
1803 if mass_status.lookback_start().is_none() {
1804 return IndexSet::new();
1805 }
1806
1807 let reported_instruments: IndexSet<InstrumentId> = if self.config.filter_position_reports {
1808 IndexSet::new()
1809 } else {
1810 mass_status
1811 .position_reports()
1812 .into_iter()
1813 .filter(|(instrument_id, reports)| {
1814 !reports.is_empty() && self.should_reconcile_instrument(instrument_id)
1815 })
1816 .map(|(instrument_id, _)| instrument_id)
1817 .collect()
1818 };
1819
1820 let mut venue_order_ids = IndexSet::new();
1821
1822 for (venue_order_id, report) in order_reports {
1823 if report.filled_qty.is_zero() || reported_instruments.contains(&report.instrument_id) {
1824 continue;
1825 }
1826
1827 venue_order_ids.insert(*venue_order_id);
1828 }
1829
1830 for (venue_order_id, fills) in fill_reports {
1831 let Some(fill) = fills.first() else {
1832 continue;
1833 };
1834
1835 if reported_instruments.contains(&fill.instrument_id) {
1836 continue;
1837 }
1838
1839 venue_order_ids.insert(*venue_order_id);
1840 }
1841
1842 venue_order_ids
1843 }
1844
1845 fn validate_mass_status_order_sources(&self, mass_status: &ExecutionMassStatus) {
1849 let cache = self.cache.borrow();
1850 let mut checked_client_order_ids = IndexSet::new();
1851 let mut missing_origins: Vec<ClientOrderId> = Vec::new();
1852 let mut mismatched_origins: Vec<(ClientOrderId, ClientId)> = Vec::new();
1853
1854 let mut validate_report_source =
1855 |direct_client_order_id: Option<ClientOrderId>, venue_order_id: VenueOrderId| {
1856 let direct_client_order_id = direct_client_order_id
1857 .filter(|client_order_id| cache.order_exists(client_order_id));
1858 let indexed_client_order_id = cache
1859 .client_order_id(&venue_order_id)
1860 .copied()
1861 .filter(|client_order_id| cache.order_exists(client_order_id));
1862
1863 for client_order_id in [direct_client_order_id, indexed_client_order_id]
1864 .into_iter()
1865 .flatten()
1866 .filter(|client_order_id| checked_client_order_ids.insert(*client_order_id))
1867 {
1868 match cache.client_id(&client_order_id) {
1869 Some(cached_client_id) if *cached_client_id == mass_status.client_id => {}
1870 Some(cached_client_id) => {
1871 mismatched_origins.push((client_order_id, *cached_client_id));
1872 }
1873 None => missing_origins.push(client_order_id),
1874 }
1875 }
1876 };
1877
1878 for report in mass_status.order_reports().values() {
1879 validate_report_source(report.client_order_id, report.venue_order_id);
1880 }
1881
1882 for fills in mass_status.fill_reports().values() {
1883 for fill in fills {
1884 validate_report_source(fill.client_order_id, fill.venue_order_id);
1885 }
1886 }
1887
1888 if !missing_origins.is_empty() {
1889 let samples = missing_origins
1890 .iter()
1891 .take(LOG_SAMPLES_MAX)
1892 .map(ToString::to_string)
1893 .collect::<Vec<_>>()
1894 .join(", ");
1895
1896 log::warn!(
1897 "Found {} cached order(s) without an execution client origin ({}): \
1898 continuing reconciliation against mass status client {} for compatibility \
1899 with existing cache data",
1900 missing_origins.len(),
1901 samples,
1902 mass_status.client_id,
1903 );
1904 }
1905
1906 if !mismatched_origins.is_empty() {
1907 let samples = mismatched_origins
1908 .iter()
1909 .take(LOG_SAMPLES_MAX)
1910 .map(|(client_order_id, cached)| format!("{client_order_id} -> {cached}"))
1911 .collect::<Vec<_>>()
1912 .join(", ");
1913
1914 log::warn!(
1915 "Found {} cached order(s) with an execution client origin conflicting with \
1916 mass status client {} ({}): continuing reconciliation for compatibility; \
1917 this conflict will become a startup error in a future release, verify cached \
1918 order ownership and execution client configuration",
1919 mismatched_origins.len(),
1920 mass_status.client_id,
1921 samples,
1922 );
1923 }
1924 }
1925
1926 pub fn check_inflight_orders(&mut self) -> InflightCheckResult {
1933 let mut result = InflightCheckResult::default();
1934 let now = dst::time::Instant::now();
1935 let threshold = Duration::from_millis(self.config().inflight_threshold_ms);
1936
1937 let mut to_check = Vec::new();
1938
1939 for (client_order_id, check) in &self.order_inflight_checks {
1940 if now
1941 .checked_duration_since(check.submitted_at)
1942 .is_some_and(|elapsed| elapsed > threshold)
1943 {
1944 to_check.push(*client_order_id);
1945 }
1946 }
1947
1948 for client_order_id in to_check {
1949 self.confirm_submission_outcome(&client_order_id);
1950
1951 if self
1952 .config
1953 .filtered_client_order_ids
1954 .contains(&client_order_id)
1955 {
1956 self.clear_recon_tracking(&client_order_id, true);
1957 continue;
1958 }
1959
1960 if self.order_query_pending.contains(&client_order_id) {
1961 continue;
1962 }
1963
1964 let Some(check) = self.order_inflight_checks.get_mut(&client_order_id) else {
1965 continue;
1966 };
1967
1968 if let Some(last_query_at) = check.last_query_at
1969 && now
1970 .checked_duration_since(last_query_at)
1971 .is_none_or(|elapsed| elapsed < threshold)
1972 {
1973 continue;
1974 }
1975
1976 check.retry_count += 1;
1977 check.last_query_at = Some(now);
1978 self.order_query_recency.mark(client_order_id);
1979 self.order_recon_retries
1980 .insert(client_order_id, check.retry_count);
1981
1982 if check.retry_count >= self.config.inflight_max_retries {
1983 let retry_count = check.retry_count;
1984
1985 if self.exhaust_submission(
1986 client_order_id,
1987 SubmissionRecoverySource::Inflight,
1988 retry_count,
1989 ) {
1990 continue;
1991 }
1992
1993 let ts_now = self.clock.borrow().timestamp_ns();
1994
1995 if let Some(order) = self.get_order(client_order_id) {
1996 match order.status() {
1997 OrderStatus::Submitted => {
1998 if let Some(event) = create_reconciliation_rejected(
2000 &order,
2001 Some("INFLIGHT_TIMEOUT"),
2002 ts_now,
2003 ) {
2004 result.events.push(event);
2005 }
2006 }
2007 OrderStatus::PendingUpdate | OrderStatus::PendingCancel => {
2008 result
2010 .events
2011 .push(create_reconciliation_canceled(&order, ts_now));
2012 }
2013 _ => {
2014 }
2016 }
2017 }
2018
2019 self.clear_recon_tracking(&client_order_id, true);
2020 } else if let Some(order) = self.get_order(client_order_id) {
2021 let ts_now = self.clock.borrow().timestamp_ns();
2023 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
2024 let query = TradingCommand::QueryOrder(QueryOrder::new(
2025 order.trader_id(),
2026 client_id,
2027 order.strategy_id(),
2028 order.instrument_id(),
2029 order.client_order_id(),
2030 order.venue_order_id(),
2031 UUID4::new(),
2032 ts_now,
2033 None,
2034 None, ));
2036 result.queries.push(query);
2037 }
2038 }
2039
2040 result
2041 }
2042
2043 pub(crate) fn register_submission_retention(&mut self, client: &dyn ExecutionClient) {
2044 if client.retain_unresolved_submissions() {
2045 self.submission_retention
2046 .insert(client.client_id(), client.account_id());
2047 }
2048 }
2049
2050 fn submission_retention_enabled(
2051 &self,
2052 client_order_id: ClientOrderId,
2053 client_id: Option<ClientId>,
2054 ) -> bool {
2055 if self.config.submission_recovery_policy == SubmissionRecoveryPolicy::RetainUnresolved {
2056 return true;
2057 }
2058
2059 if self.submission_retention.is_empty() {
2060 return false;
2061 }
2062
2063 let cache = self.cache.borrow();
2064
2065 if let Some(client_id) = client_id.or_else(|| cache.client_id(&client_order_id).copied()) {
2066 return self.submission_retention.contains_key(&client_id);
2067 }
2068
2069 cache
2070 .order_ref(&client_order_id)
2071 .and_then(|order| order.account_id())
2072 .is_some_and(|account_id| {
2073 self.submission_retention
2074 .values()
2075 .any(|id| *id == account_id)
2076 })
2077 }
2078
2079 fn submission_is_retained(&self, client_order_id: ClientOrderId) -> bool {
2080 self.submissions
2081 .get(&client_order_id)
2082 .is_some_and(|submission| submission.exhausted)
2083 && self.submission_recovery_pending(client_order_id)
2084 }
2085
2086 fn filtered_open_orders_for_reconciliation(&self) -> Vec<OrderAny> {
2087 let cache = self.cache.borrow();
2088 let mut orders = cache.orders_open(None, None, None, None, None);
2089 orders.extend(cache.orders_inflight(None, None, None, None, None));
2090 let mut seen_client_order_ids = IndexSet::new();
2091
2092 orders
2093 .into_iter()
2094 .filter(|order| {
2095 seen_client_order_ids.insert(order.client_order_id())
2096 && !self.submission_is_retained(order.client_order_id())
2097 && !self
2098 .config
2099 .filtered_client_order_ids
2100 .contains(&order.client_order_id())
2101 && self.should_reconcile_instrument(&order.instrument_id())
2102 })
2103 .map(|order| order.clone())
2104 .collect()
2105 }
2106
2107 fn open_position_keys_for_reconciliation(&self) -> IndexSet<InstrumentAccountKey> {
2108 let cache = self.cache.borrow();
2109 let positions = cache.positions_open(None, None, None, None, None);
2110 let mut position_keys = IndexSet::new();
2111
2112 for position in positions {
2113 if !self.should_reconcile_instrument(&position.instrument_id) {
2114 continue;
2115 }
2116
2117 position_keys.insert((position.instrument_id, position.account_id));
2118 }
2119
2120 position_keys
2121 }
2122
2123 pub async fn check_open_orders(
2127 &mut self,
2128 clients: &[&dyn ExecutionClient],
2129 ) -> Vec<OrderEventAny> {
2130 log::debug!("Checking order consistency between cached-state and venues");
2131
2132 let check = self.prepare_open_order_report_check(UUID4::new(), clients);
2133 let mut all_reports = Vec::new();
2134 let mut queried_clients = IndexSet::new();
2135 let mut failed_clients = IndexSet::new();
2136
2137 for client in clients {
2138 let client_id = client.client_id();
2139 queried_clients.insert(client_id);
2140
2141 match client.generate_order_status_reports(&check.command).await {
2142 Ok(reports) => {
2143 all_reports.extend(
2144 reports
2145 .into_iter()
2146 .map(|report| SourcedOrderStatusReport { client_id, report }),
2147 );
2148 }
2149 Err(e) => {
2150 failed_clients.insert(client_id);
2151 log::warn!("Failed to query order reports from {client_id}: {e}");
2152 }
2153 }
2154 }
2155
2156 let result = self.reconcile_open_order_reports(
2157 &check,
2158 all_reports,
2159 &queried_clients,
2160 &failed_clients,
2161 clients,
2162 );
2163
2164 let mut events = result.events;
2165
2166 if !result.targeted_queries.is_empty() {
2167 let query_delay =
2168 Duration::from_millis(u64::from(self.config.single_order_query_delay_ms));
2169 let query_results =
2170 request_targeted_order_reports(result.targeted_queries, clients, query_delay).await;
2171 events.extend(self.reconcile_targeted_order_reports(query_results, clients));
2172 }
2173
2174 events
2175 }
2176
2177 pub(crate) fn prepare_open_order_report_check(
2179 &mut self,
2180 command_id: UUID4,
2181 clients: &[&dyn ExecutionClient],
2182 ) -> OpenOrderReportCheck {
2183 for client in clients {
2184 self.register_submission_retention(*client);
2185 }
2186
2187 let filtered_orders = self.filtered_open_orders_for_reconciliation();
2188 let active_order_ids: IndexSet<ClientOrderId> =
2189 filtered_orders.iter().map(Order::client_order_id).collect();
2190 self.order_coverage_warnings
2191 .retain(|client_order_id| active_order_ids.contains(client_order_id));
2192 self.order_lookback_warnings
2193 .retain(|client_order_id| active_order_ids.contains(client_order_id));
2194 self.order_coverage_unresolved
2195 .retain(|client_order_id| active_order_ids.contains(client_order_id));
2196
2197 let mut client_coverage = IndexMap::new();
2198
2199 for order in &filtered_orders {
2200 let client_order_id = order.client_order_id();
2201 let coverage = self.resolve_order_report_client_coverage(order, clients);
2202
2203 match &coverage {
2204 ReportClientCoverage::Resolved(_) => {
2205 if self
2206 .order_coverage_unresolved
2207 .shift_remove(&client_order_id)
2208 {
2209 self.order_coverage_warnings.shift_remove(&client_order_id);
2210 }
2211 }
2212 ReportClientCoverage::Unavailable(_) | ReportClientCoverage::Unresolved => {
2213 self.order_coverage_unresolved.insert(client_order_id);
2214 }
2215 }
2216
2217 client_coverage.insert(client_order_id, coverage);
2218 }
2219
2220 log::debug!(
2221 "Found {} order{} open in cache",
2222 filtered_orders.len(),
2223 if filtered_orders.len() == 1 { "" } else { "s" }
2224 );
2225
2226 let ts_now = self.clock.borrow().timestamp_ns();
2227 let start = self
2228 .config
2229 .open_check_lookback_mins
2230 .map(DurationNanos::from_mins)
2231 .map(|lookback| ts_now.saturating_sub(lookback));
2232
2233 let mut command = GenerateOrderStatusReports::new(
2234 command_id,
2235 ts_now,
2236 self.config.open_check_open_only,
2237 None,
2238 start,
2239 None,
2240 None,
2241 None,
2242 );
2243 command.log_receipt_level = LogLevel::Debug;
2244
2245 OpenOrderReportCheck {
2246 command,
2247 filtered_orders,
2248 client_coverage,
2249 }
2250 }
2251
2252 fn resolve_order_report_client_coverage(
2253 &self,
2254 order: &OrderAny,
2255 clients: &[&dyn ExecutionClient],
2256 ) -> ReportClientCoverage {
2257 if let Some(client_id) = self.cache.borrow().client_id(&order.client_order_id()) {
2258 return ReportClientCoverage::Resolved(IndexSet::from([*client_id]));
2259 }
2260
2261 let client_ids =
2262 responsible_client_ids(order.account_id(), order.instrument_id().venue, clients);
2263
2264 if client_ids.is_empty() {
2265 ReportClientCoverage::Unresolved
2266 } else {
2267 ReportClientCoverage::Resolved(client_ids)
2268 }
2269 }
2270
2271 pub fn check_open_order_queries(&mut self) -> Vec<TradingCommand> {
2273 let now = dst::time::Instant::now();
2274 let query_delay = Duration::from_millis(u64::from(self.config.single_order_query_delay_ms));
2275 let query_limit = self.config.max_single_order_queries_per_cycle as usize;
2276
2277 if query_limit == 0 {
2278 return Vec::new();
2279 }
2280
2281 let mut filtered_orders = self.filtered_open_orders_for_reconciliation();
2282 filtered_orders.sort_by_key(|order| {
2283 let client_order_id = order.client_order_id();
2284 (
2285 self.order_query_recency.last_marked(&client_order_id),
2286 client_order_id,
2287 )
2288 });
2289
2290 let mut queries = Vec::new();
2291
2292 for order in filtered_orders {
2293 if queries.len() >= query_limit {
2294 break;
2295 }
2296
2297 let client_order_id = order.client_order_id();
2298 let threshold = Duration::from(self.config.open_check_threshold_ns);
2299
2300 if let Some(elapsed) = self.order_activity.elapsed_at(&client_order_id, now)
2301 && elapsed < threshold
2302 {
2303 let elapsed_ms = elapsed.as_millis();
2304 let threshold_ms = threshold.as_millis();
2305 log::debug!(
2306 "Deferring open order query for {client_order_id}: recent local activity \
2307 ({elapsed_ms}ms < threshold={threshold_ms}ms)",
2308 );
2309 continue;
2310 }
2311
2312 if self
2313 .order_query_recency
2314 .within_at(&client_order_id, now, query_delay)
2315 {
2316 continue;
2317 }
2318
2319 self.order_query_recency.mark(client_order_id);
2320 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
2321 let ts_now = self.clock.borrow().timestamp_ns();
2322
2323 let cmd = TradingCommand::QueryOrder(QueryOrder::new(
2324 order.trader_id(),
2325 client_id,
2326 order.strategy_id(),
2327 order.instrument_id(),
2328 client_order_id,
2329 order.venue_order_id(),
2330 UUID4::new(),
2331 ts_now,
2332 None,
2333 None,
2334 ));
2335 queries.push(cmd);
2336 }
2337
2338 queries
2339 }
2340
2341 pub(crate) fn reconcile_open_order_reports(
2343 &mut self,
2344 check: &OpenOrderReportCheck,
2345 mut all_reports: Vec<SourcedOrderStatusReport>,
2346 queried_clients: &IndexSet<ClientId>,
2347 failed_clients: &IndexSet<ClientId>,
2348 clients: &[&dyn ExecutionClient],
2349 ) -> OpenOrderReconciliationResult {
2350 all_reports.retain(|sourced| !self.should_skip_order_report(&sourced.report));
2351 let mut venue_reported_ids = IndexSet::new();
2352
2353 for sourced in &all_reports {
2354 let report = &sourced.report;
2355
2356 let client_order_id = report.client_order_id.or_else(|| {
2357 self.cache
2358 .borrow()
2359 .client_order_id(&report.venue_order_id)
2360 .copied()
2361 });
2362
2363 let Some(client_order_id) = client_order_id else {
2364 continue;
2365 };
2366
2367 venue_reported_ids.insert(client_order_id);
2368 self.order_coverage_warnings.shift_remove(&client_order_id);
2369 self.order_lookback_warnings.shift_remove(&client_order_id);
2370
2371 self.order_recon_retries.shift_remove(&client_order_id);
2375 }
2376
2377 let mut events = Vec::new();
2378 let mut targeted_candidates = Vec::new();
2379
2380 for sourced in all_reports {
2381 let report = sourced.report;
2382
2383 let order = match report.client_order_id {
2384 Some(client_order_id) => self.get_order(client_order_id),
2385 None => self.get_order_by_venue_order_id(report.venue_order_id),
2386 };
2387
2388 let Some(order) = order else {
2389 continue;
2390 };
2391
2392 let client_order_id = order.client_order_id();
2393
2394 let threshold = Duration::from(self.config.open_check_threshold_ns);
2396 let activity_elapsed = self.order_activity.elapsed(&client_order_id);
2397 self.confirm_submitted_report(&report);
2398
2399 if let Some(elapsed) = activity_elapsed
2400 && elapsed < threshold
2401 {
2402 let elapsed_ms = elapsed.as_millis();
2403 let threshold_ms = threshold.as_millis();
2404 log::debug!(
2405 "Deferring reconciliation for {client_order_id}: recent local activity ({elapsed_ms}ms < threshold={threshold_ms}ms)",
2406 );
2407 continue;
2408 }
2409
2410 let instrument = self.get_instrument(&report.instrument_id);
2411
2412 if terminal_report_has_missing_fills(&report, order.filled_qty()) {
2413 targeted_candidates.push((
2414 order,
2415 IndexSet::from([sourced.client_id]),
2416 Some(report),
2417 ));
2418 continue;
2419 }
2420
2421 let commission_client = clients
2422 .iter()
2423 .find(|client| client.client_id() == sourced.client_id)
2424 .copied();
2425
2426 match self.reconcile_order_report(
2427 &order,
2428 &report,
2429 instrument.as_ref(),
2430 commission_client,
2431 ) {
2432 Ok(order_events) => events.extend(order_events),
2433 Err(e) => log::error!(
2434 "Deferring reconciliation for {client_order_id}: venue commission calculation failed: {e}"
2435 ),
2436 }
2437 }
2438
2439 if self.config.open_check_open_only {
2444 let cached_ids: IndexSet<ClientOrderId> = check
2445 .filtered_orders
2446 .iter()
2447 .map(Order::client_order_id)
2448 .collect();
2449 let missing_at_venue: IndexSet<ClientOrderId> = cached_ids
2450 .difference(&venue_reported_ids)
2451 .copied()
2452 .collect();
2453
2454 if !missing_at_venue.is_empty() {
2455 log::debug!(
2456 "{} cached open order{} not present in venue current response",
2457 missing_at_venue.len(),
2458 if missing_at_venue.len() == 1 {
2459 " is"
2460 } else {
2461 "s are"
2462 },
2463 );
2464
2465 for client_order_id in missing_at_venue {
2466 log::debug!("Cached open order missing from venue response: {client_order_id}");
2467 }
2468 }
2469 } else {
2470 let candidates: Vec<&OrderAny> = if let Some(cutoff) = check.command.start {
2471 let mut candidates = Vec::new();
2472
2473 for order in &check.filtered_orders {
2474 let client_order_id = order.client_order_id();
2475 if order.ts_last() >= cutoff {
2476 self.order_lookback_warnings.shift_remove(&client_order_id);
2477 candidates.push(order);
2478 } else if !venue_reported_ids.contains(&client_order_id)
2479 && self.order_lookback_warnings.insert(client_order_id)
2480 {
2481 log::warn!(
2482 "Skipping missing-order reconciliation for {client_order_id}: its last update predates the configured open-check lookback window; absence from the bulk response cannot be treated as evidence and no targeted query will be issued from it"
2483 );
2484 }
2485 }
2486
2487 candidates
2488 } else {
2489 check.filtered_orders.iter().collect()
2490 };
2491
2492 for order in candidates {
2493 let client_order_id = order.client_order_id();
2494 if venue_reported_ids.contains(&client_order_id) {
2495 continue;
2496 }
2497
2498 let responsible_clients = match check.client_coverage.get(&client_order_id) {
2499 Some(ReportClientCoverage::Resolved(responsible_clients))
2500 if !responsible_clients.is_empty() =>
2501 {
2502 responsible_clients
2503 }
2504 _ => {
2505 if self.order_coverage_warnings.insert(client_order_id) {
2506 log::warn!(
2507 "Skipping order reconciliation for {client_order_id}: responsible execution client coverage is unresolved"
2508 );
2509 }
2510
2511 continue;
2512 }
2513 };
2514
2515 let missing_clients = responsible_clients
2516 .difference(queried_clients)
2517 .copied()
2518 .collect::<IndexSet<_>>();
2519
2520 if !missing_clients.is_empty() {
2521 if self.order_coverage_warnings.insert(client_order_id) {
2522 log::warn!(
2523 "Skipping order reconciliation for {client_order_id}: responsible execution clients were not queried: {missing_clients:?}"
2524 );
2525 }
2526
2527 continue;
2528 }
2529
2530 let failed_responsible_clients = responsible_clients
2531 .intersection(failed_clients)
2532 .copied()
2533 .collect::<IndexSet<_>>();
2534
2535 if !failed_responsible_clients.is_empty() {
2536 log::warn!(
2537 "Skipping order reconciliation for {client_order_id}: failed to query responsible execution clients: {failed_responsible_clients:?}"
2538 );
2539 continue;
2540 }
2541
2542 self.order_coverage_warnings.shift_remove(&client_order_id);
2543 if let Some(order) = self.prepare_missing_order_query(client_order_id) {
2544 targeted_candidates.push((order, responsible_clients.clone(), None));
2545 }
2546 }
2547 }
2548
2549 targeted_candidates.sort_by_key(|(order, _, _)| {
2550 let client_order_id = order.client_order_id();
2551 (
2552 self.order_query_recency.last_marked(&client_order_id),
2553 client_order_id,
2554 )
2555 });
2556
2557 let query_limit = self.config.max_single_order_queries_per_cycle as usize;
2558 let mut planned_queries = 0usize;
2559 let mut cap_deferred_orders = 0usize;
2560 let mut targeted_queries = Vec::new();
2561
2562 for (order, responsible_clients, report) in targeted_candidates {
2563 let client_order_id = order.client_order_id();
2564
2565 let required_queries = responsible_clients.len();
2566 let exceeds_query_limit = planned_queries + required_queries > query_limit;
2567 let can_run_oversized_group = planned_queries == 0 && query_limit > 0;
2568 if required_queries == 0 || (exceeds_query_limit && !can_run_oversized_group) {
2569 cap_deferred_orders += 1;
2570 continue;
2571 }
2572
2573 if required_queries > query_limit {
2574 log::warn!(
2575 "Targeted order query for {client_order_id} requires {required_queries} responsible clients, exceeding the per-cycle limit {query_limit} to avoid indefinite deferral"
2576 );
2577 }
2578
2579 planned_queries += required_queries;
2580 self.order_query_recency.mark(client_order_id);
2581 self.order_query_pending.insert(client_order_id);
2582 let command_id = UUID4::new();
2583 let ts_now = self.clock.borrow().timestamp_ns();
2584
2585 let command = GenerateOrderStatusReport::new(
2586 command_id,
2587 ts_now,
2588 Some(order.instrument_id()),
2589 Some(client_order_id),
2590 order.venue_order_id(),
2591 None,
2592 None,
2593 );
2594 targeted_queries.push(TargetedOrderQuery {
2595 client_order_id,
2596 responsible_clients,
2597 report,
2598 filled_qty: order.filled_qty(),
2599 command,
2600 });
2601 }
2602
2603 if cap_deferred_orders > 0 {
2604 log::warn!(
2605 "Reached max single-order queries ({query_limit}) this cycle, deferring {cap_deferred_orders} order(s)"
2606 );
2607 }
2608
2609 OpenOrderReconciliationResult {
2610 events,
2611 targeted_queries,
2612 }
2613 }
2614
2615 pub(crate) fn reconcile_targeted_order_reports(
2617 &mut self,
2618 results: Vec<TargetedOrderReportResult>,
2619 clients: &[&dyn ExecutionClient],
2620 ) -> Vec<OrderEventAny> {
2621 let mut events = Vec::new();
2622 let mut fill_queue = ReconciliationFillQueue::default();
2623
2624 for result in results {
2625 let client_order_id = result.client_order_id;
2626 self.remove_targeted_order_queries(&[client_order_id]);
2627
2628 if let Some(report) = result.report {
2629 self.order_recon_retries.shift_remove(&client_order_id);
2630 self.order_coverage_warnings.shift_remove(&client_order_id);
2631
2632 let Some(order) = self.get_order(client_order_id) else {
2633 continue;
2634 };
2635
2636 let instrument = self.get_instrument(&report.instrument_id);
2637
2638 let commission_client = result.client_id.and_then(|client_id| {
2639 clients
2640 .iter()
2641 .find(|client| client.client_id() == client_id)
2642 .copied()
2643 });
2644
2645 log::info!(
2646 color = LogColor::Blue as u8;
2647 "Found {client_order_id} via targeted order status query: {}",
2648 report.order_status,
2649 );
2650
2651 let fills = result.fills.iter().collect::<Vec<_>>();
2652 events.extend(self.reconcile_order_with_fills(
2653 false,
2654 &order,
2655 &report,
2656 &fills,
2657 instrument.as_ref(),
2658 &mut fill_queue,
2659 commission_client,
2660 ));
2661 continue;
2662 }
2663
2664 if result.coverage_complete {
2665 events.extend(self.resolve_missing_order(client_order_id));
2666 } else {
2667 log::warn!(
2668 "Deferring missing-order resolution for {client_order_id}: targeted order status coverage was incomplete"
2669 );
2670 }
2671 }
2672
2673 events
2674 }
2675
2676 pub async fn check_positions_consistency(
2681 &mut self,
2682 clients: &[&dyn ExecutionClient],
2683 ) -> Vec<OrderEventAny> {
2684 let check = self.prepare_position_report_check(UUID4::new(), clients);
2685 let mut reports = Vec::new();
2686 let mut queried_clients = IndexSet::new();
2687 let mut failed_clients = IndexSet::new();
2688
2689 for client in clients {
2690 let client_id = client.client_id();
2691 queried_clients.insert(client_id);
2692 self.set_position_reconciliation_tolerance(
2693 client.account_id(),
2694 client.position_reconciliation_tolerance(),
2695 );
2696
2697 match client
2698 .generate_position_status_reports(&check.command)
2699 .await
2700 {
2701 Ok(client_reports) => {
2702 reports.extend(client_reports);
2703 }
2704 Err(e) => {
2705 failed_clients.insert(client_id);
2706 log::warn!("Failed to query position reports from {client_id}: {e}");
2707 }
2708 }
2709 }
2710
2711 let active_keys = self
2712 .open_position_keys_for_reconciliation()
2713 .into_iter()
2714 .chain(reports.iter().filter_map(|report| {
2715 (self.should_reconcile_instrument(&report.instrument_id)
2716 && report.signed_decimal_qty != Decimal::ZERO)
2717 .then_some((report.instrument_id, report.account_id))
2718 }))
2719 .collect();
2720
2721 let events =
2722 self.reconcile_position_reports(&check, reports, &queried_clients, &failed_clients);
2723
2724 self.retain_position_reconciliation(&active_keys);
2726
2727 events
2728 }
2729
2730 #[must_use]
2735 pub fn prepare_position_report_check(
2736 &self,
2737 command_id: UUID4,
2738 clients: &[&dyn ExecutionClient],
2739 ) -> PositionReportCheck {
2740 let position_keys = self.open_position_keys_for_reconciliation();
2741
2742 let client_coverage = position_keys
2743 .iter()
2744 .map(|key| (*key, resolve_position_report_client_coverage(*key, clients)))
2745 .collect();
2746
2747 let mut activity_revisions = self.position_activity_revisions.clone();
2748
2749 for key in &position_keys {
2750 activity_revisions
2751 .entry(*key)
2752 .or_insert_with(|| self.position_activity_revision(key));
2753 }
2754
2755 log::debug!(
2756 "Found {} unique instrument/account combination{} with open positions",
2757 position_keys.len(),
2758 if position_keys.len() == 1 { "" } else { "s" }
2759 );
2760
2761 let ts_now = self.clock.borrow().timestamp_ns();
2762
2763 let mut command = GeneratePositionStatusReports::new(
2764 command_id, ts_now, None, None, None, None, None, );
2770 command.log_receipt_level = LogLevel::Debug;
2771
2772 PositionReportCheck {
2773 command,
2774 client_coverage,
2775 activity_revisions,
2776 }
2777 }
2778
2779 pub fn plan_position_fill_reports(
2784 &mut self,
2785 check: &mut PositionReportCheck,
2786 reports: &[PositionStatusReport],
2787 queried_clients: &IndexSet<ClientId>,
2788 failed_clients: &IndexSet<ClientId>,
2789 clients: &[&dyn ExecutionClient],
2790 ) -> PositionFillReportPlan {
2791 let venue_positions = self.group_position_reports(reports.iter().cloned());
2792
2793 let keys = check
2794 .client_coverage
2795 .keys()
2796 .copied()
2797 .chain(venue_positions.iter().filter_map(|(key, reports)| {
2798 reports
2799 .iter()
2800 .any(|report| report.signed_decimal_qty != Decimal::ZERO)
2801 .then_some(*key)
2802 }))
2803 .collect::<IndexSet<_>>();
2804
2805 let query_end = self.timestamp_ns();
2806 let lookback = DurationNanos::from_mins(self.config.position_check_lookback_mins);
2807 let query_start = query_end.saturating_sub(lookback);
2808 let mut discrepancy_keys = IndexSet::new();
2809 let mut queries = Vec::new();
2810
2811 for &key in &keys {
2812 let coverage = check
2813 .client_coverage
2814 .entry(key)
2815 .or_insert_with(|| resolve_position_report_client_coverage(key, clients));
2816 let prepared_revision = *check.activity_revisions.entry(key).or_default();
2817 let venue_reports = venue_positions
2818 .get(&key)
2819 .map(Vec::as_slice)
2820 .unwrap_or_default();
2821 let comparison = self.position_quantity_comparison(key, venue_reports);
2822 let tolerance = self.position_reconciliation_tolerance(key.1);
2823
2824 if comparison.quantities_match(tolerance) {
2825 self.clear_position_reconciliation(&key);
2826 continue;
2827 }
2828
2829 discrepancy_keys.insert(key);
2830
2831 if self.position_activity_revision(&key) > prepared_revision
2832 || self.position_activity_is_recent(&key)
2833 {
2834 continue;
2835 }
2836
2837 let report_shape = comparison.report_shape();
2838 let retries = self.position_reconciliation_retries(&key, report_shape);
2839 if retries >= self.config().position_check_retries {
2840 continue;
2841 }
2842
2843 let responsible_clients = match coverage {
2844 ReportClientCoverage::Resolved(responsible_clients) => responsible_clients,
2845
2846 ReportClientCoverage::Unavailable(responsible_clients)
2848 if !venue_reports.is_empty() =>
2849 {
2850 responsible_clients
2851 }
2852 ReportClientCoverage::Unavailable(_) => {
2853 log::debug!(
2854 "Skipping fill report query for {}/{}: responsible execution client coverage is unavailable",
2855 key.0,
2856 key.1,
2857 );
2858 continue;
2859 }
2860 ReportClientCoverage::Unresolved => {
2861 log::warn!(
2862 "Skipping fill report query for {}/{}: responsible execution client coverage is unresolved",
2863 key.0,
2864 key.1,
2865 );
2866 continue;
2867 }
2868 };
2869
2870 if responsible_clients.is_empty()
2871 || !responsible_clients.is_subset(queried_clients)
2872 || !responsible_clients.is_disjoint(failed_clients)
2873 {
2874 log::warn!(
2875 "Skipping fill report query for {}/{}: responsible position report coverage is incomplete",
2876 key.0,
2877 key.1,
2878 );
2879 continue;
2880 }
2881
2882 for client_id in responsible_clients.iter() {
2883 let mut command = GenerateFillReports::new(
2884 UUID4::new(),
2885 query_end,
2886 Some(key.0),
2887 None,
2888 Some(query_start),
2889 Some(query_end),
2890 None,
2891 Some(check.command.command_id),
2892 );
2893 command.log_receipt_level = LogLevel::Debug;
2894 queries.push(PositionFillReportQuery {
2895 key,
2896 client_id: *client_id,
2897 command,
2898 });
2899 }
2900 }
2901
2902 self.retain_position_reconciliation(&keys);
2903
2904 PositionFillReportPlan {
2905 queries,
2906 discrepancy_keys,
2907 }
2908 }
2909
2910 #[must_use]
2912 pub fn position_report_check_is_current(
2913 &self,
2914 check: &PositionReportCheck,
2915 key: &InstrumentAccountKey,
2916 ) -> bool {
2917 check
2918 .activity_revisions
2919 .get(key)
2920 .is_some_and(|revision| self.position_activity_revision(key) == *revision)
2921 }
2922
2923 pub fn prepare_position_fill_report(
2930 &self,
2931 report: &mut FillReport,
2932 venue_reports: &[PositionStatusReport],
2933 ) -> anyhow::Result<PositionFillReportPreparation> {
2934 let cache = self.cache();
2935 let venue_client_order_id = cache.client_order_id(&report.venue_order_id).copied();
2936 if let (Some(report_client_order_id), Some(venue_client_order_id)) =
2937 (report.client_order_id, venue_client_order_id)
2938 {
2939 anyhow::ensure!(
2940 report_client_order_id == venue_client_order_id,
2941 "fill {} client order ID {report_client_order_id} conflicts with venue order mapping {venue_client_order_id}",
2942 report.trade_id,
2943 );
2944 }
2945
2946 let client_order_id = report.client_order_id.or(venue_client_order_id);
2947 let order = client_order_id.and_then(|id| cache.order(&id));
2948 if let Some(order) = &order {
2949 anyhow::ensure!(
2950 order.instrument_id() == report.instrument_id
2951 && order.order_side() == report.order_side
2952 && order
2953 .account_id()
2954 .is_none_or(|account_id| account_id == report.account_id)
2955 && order
2956 .venue_order_id()
2957 .is_none_or(|venue_order_id| venue_order_id == report.venue_order_id),
2958 "fill {} conflicts with cached order {}",
2959 report.trade_id,
2960 order.client_order_id(),
2961 );
2962 }
2963
2964 let hedge_context = report.venue_position_id.is_some()
2965 || venue_reports
2966 .iter()
2967 .any(|venue_report| venue_report.venue_position_id.is_some());
2968 let flip = client_order_id
2969 .map(|id| cache.hedging_flip_position_ids(id))
2970 .transpose()?
2971 .flatten();
2972
2973 let mapped_position_id = flip.map(|ids| ids[1]).or_else(|| {
2974 client_order_id
2975 .and_then(|client_order_id| cache.position_id(&client_order_id))
2976 .copied()
2977 });
2978
2979 if hedge_context
2980 && let (Some(venue_position_id), Some(mapped_position_id)) =
2981 (report.venue_position_id, mapped_position_id)
2982 {
2983 anyhow::ensure!(
2984 venue_position_id == mapped_position_id
2985 || flip.is_some_and(|ids| ids.contains(&venue_position_id)),
2986 "fill {} position ID {venue_position_id} conflicts with cached order position {mapped_position_id}",
2987 report.trade_id,
2988 );
2989 }
2990
2991 if let Some(order) = order
2992 && has_active_inferred_fill(&order)?
2993 {
2994 return Ok(PositionFillReportPreparation::InferredOverlap);
2995 }
2996
2997 if fill_precedes_snapshot_reconciled_position(&cache, report) {
2998 return Ok(PositionFillReportPreparation::SnapshotOverlap);
2999 }
3000
3001 if report.venue_position_id.is_none() && flip.is_some() {
3002 report.venue_position_id = mapped_position_id;
3003 return Ok(PositionFillReportPreparation::Ready);
3004 }
3005
3006 if !hedge_context {
3007 return Ok(PositionFillReportPreparation::Ready);
3008 }
3009
3010 if report.venue_position_id.is_some() {
3011 return Ok(PositionFillReportPreparation::Ready);
3012 }
3013
3014 let Some(position_id) = mapped_position_id else {
3015 return Ok(PositionFillReportPreparation::Unattributed);
3016 };
3017
3018 let position = cache.position(&position_id).ok_or_else(|| {
3019 anyhow::anyhow!(
3020 "fill {} maps to position {position_id}, which is not cached",
3021 report.trade_id,
3022 )
3023 })?;
3024
3025 anyhow::ensure!(
3026 position.account_id == report.account_id
3027 && position.instrument_id == report.instrument_id,
3028 "fill {} maps to position {position_id} with a different account or instrument",
3029 report.trade_id,
3030 );
3031 anyhow::ensure!(
3032 position.is_open(),
3033 "fill {} maps to non-open position {position_id}",
3034 report.trade_id,
3035 );
3036 anyhow::ensure!(
3037 !position.is_opposite_side(report.order_side) || report.last_qty <= position.quantity,
3038 "fill {} without a venue position ID would cross position {position_id}",
3039 report.trade_id,
3040 );
3041
3042 report.venue_position_id = Some(position_id);
3043 Ok(PositionFillReportPreparation::Ready)
3044 }
3045
3046 pub fn record_unapplied_fill_report(&mut self, report: &FillReport) {
3048 let fill_key = (report.account_id, report.instrument_id, report.trade_id);
3049 if !self.fills_unapplied.contains_key(&fill_key) {
3050 self.fills_unapplied.mark(fill_key);
3051 }
3052
3053 let ttl = Duration::from_mins(self.config.position_check_lookback_mins)
3054 .max(Duration::from_mins(1));
3055 self.fills_unapplied.prune_older_than(ttl);
3056 }
3057
3058 #[must_use]
3062 pub fn is_unapplied_fill_report_expired(&self, report: &FillReport) -> bool {
3063 let fill_key = (report.account_id, report.instrument_id, report.trade_id);
3064 self.fills_unapplied.contains_key(&fill_key)
3065 && !self.fills_unapplied.within(
3066 &fill_key,
3067 Duration::from(self.config.position_check_threshold_ns),
3068 )
3069 }
3070
3071 #[must_use]
3073 pub fn position_contains_fill_report(&self, report: &FillReport) -> bool {
3074 let cache = self.cache();
3075 let client_order_id = report
3076 .client_order_id
3077 .or_else(|| cache.client_order_id(&report.venue_order_id).copied());
3078
3079 let flip = match client_order_id
3080 .map(|id| cache.hedging_flip_position_ids(id))
3081 .transpose()
3082 {
3083 Ok(flip) => flip.flatten(),
3084 Err(_) => return false,
3085 };
3086
3087 if let (Some(ids), Some(id)) = (flip, report.venue_position_id)
3088 && !ids.contains(&id)
3089 {
3090 return false;
3091 }
3092
3093 let positions = cache.positions(
3094 None,
3095 Some(&report.instrument_id),
3096 None,
3097 Some(&report.account_id),
3098 None,
3099 );
3100 let mut matched = false;
3101 let mut quantity = Quantity::zero(report.last_qty.precision);
3102 let mut commission = Money::zero(report.commission.currency);
3103
3104 for position in positions {
3105 if flip.map_or_else(
3106 || report.venue_position_id.is_some_and(|id| position.id != id),
3107 |ids| !ids.contains(&position.id),
3108 ) {
3109 continue;
3110 }
3111
3112 let Some((position_quantity, position_commission)) =
3113 position_fill_report_totals(&position, report, client_order_id, flip.is_some())
3114 else {
3115 return false;
3116 };
3117
3118 let Some(next_quantity) = quantity.checked_add(position_quantity) else {
3119 return false;
3120 };
3121
3122 let Some(next_commission) = commission.checked_add(position_commission) else {
3123 return false;
3124 };
3125
3126 matched |= position_quantity.non_zero();
3127 quantity = next_quantity;
3128 commission = next_commission;
3129 }
3130
3131 matched && quantity == report.last_qty && commission == report.commission
3132 }
3133
3134 #[must_use]
3140 pub fn reconcile_position_reports(
3141 &mut self,
3142 check: &PositionReportCheck,
3143 reports: Vec<PositionStatusReport>,
3144 queried_clients: &IndexSet<ClientId>,
3145 failed_clients: &IndexSet<ClientId>,
3146 ) -> Vec<OrderEventAny> {
3147 log::debug!("Checking position consistency between cached-state and venues");
3148
3149 let venue_positions = self.group_position_reports(reports);
3150 let mut events = Vec::new();
3151
3152 for key in check.client_coverage.keys() {
3153 let prepared_revision = check
3154 .activity_revisions
3155 .get(key)
3156 .copied()
3157 .unwrap_or_default();
3158
3159 if self.position_activity_revision(key) > prepared_revision {
3160 log::debug!(
3161 "Deferring position reconciliation for {}/{}: local activity recorded during report request",
3162 key.0,
3163 key.1,
3164 );
3165 continue;
3166 }
3167
3168 let venue_reports = venue_positions
3169 .get(key)
3170 .map(Vec::as_slice)
3171 .unwrap_or_default();
3172
3173 if venue_reports.is_empty() {
3174 match check.client_coverage.get(key) {
3175 Some(ReportClientCoverage::Resolved(responsible_clients))
3176 if !responsible_clients.is_empty()
3177 && responsible_clients.is_subset(queried_clients)
3178 && responsible_clients.is_disjoint(failed_clients) => {}
3179 Some(ReportClientCoverage::Resolved(responsible_clients))
3180 if responsible_clients.is_empty() =>
3181 {
3182 log::warn!(
3183 "Skipping position reconciliation for {}/{}: responsible execution client coverage is unresolved",
3184 key.0,
3185 key.1,
3186 );
3187 continue;
3188 }
3189 Some(ReportClientCoverage::Resolved(responsible_clients))
3190 if !responsible_clients.is_subset(queried_clients) =>
3191 {
3192 log::warn!(
3193 "Skipping position reconciliation for {}/{}: responsible execution clients were not all queried",
3194 key.0,
3195 key.1,
3196 );
3197 continue;
3198 }
3199 Some(ReportClientCoverage::Resolved(responsible_clients)) => {
3200 let failed_responsible_clients = responsible_clients
3201 .intersection(failed_clients)
3202 .copied()
3203 .collect::<IndexSet<_>>();
3204 log::warn!(
3205 "Skipping position reconciliation for {}/{}: failed to query responsible execution clients: {failed_responsible_clients:?}",
3206 key.0,
3207 key.1,
3208 );
3209 continue;
3210 }
3211 Some(ReportClientCoverage::Unavailable(responsible_clients)) => {
3212 log::debug!(
3213 "Skipping position reconciliation for {}/{}: complete bulk position coverage is unavailable from responsible execution clients: {responsible_clients:?}",
3214 key.0,
3215 key.1,
3216 );
3217 continue;
3218 }
3219 Some(ReportClientCoverage::Unresolved) | None => {
3220 log::warn!(
3221 "Skipping position reconciliation for {}/{}: responsible execution client coverage is unresolved",
3222 key.0,
3223 key.1,
3224 );
3225 continue;
3226 }
3227 }
3228 }
3229
3230 if let Some(discrepancy_events) = self.check_position_discrepancy(*key, venue_reports) {
3231 events.extend(discrepancy_events);
3232 }
3233 }
3234
3235 let current_position_keys = self.open_position_keys_for_reconciliation();
3236
3237 for (key, venue_reports) in &venue_positions {
3238 if check.client_coverage.contains_key(key)
3239 || venue_reports
3240 .iter()
3241 .all(|report| report.signed_decimal_qty == Decimal::ZERO)
3242 {
3243 continue;
3244 }
3245
3246 if current_position_keys.contains(key) {
3247 log::debug!(
3248 "Deferring position reconciliation for {}/{}: position opened after client coverage was recorded",
3249 key.0,
3250 key.1,
3251 );
3252 continue;
3253 }
3254
3255 if let Some(discrepancy_events) = self.check_position_discrepancy(*key, venue_reports) {
3256 events.extend(discrepancy_events);
3257 }
3258 }
3259
3260 events
3261 }
3262
3263 fn group_position_reports(
3264 &self,
3265 reports: impl IntoIterator<Item = PositionStatusReport>,
3266 ) -> IndexMap<InstrumentAccountKey, Vec<PositionStatusReport>> {
3267 let mut venue_positions: IndexMap<InstrumentAccountKey, Vec<PositionStatusReport>> =
3268 IndexMap::new();
3269
3270 for report in reports {
3271 if !self.should_reconcile_instrument(&report.instrument_id) {
3272 continue;
3273 }
3274
3275 venue_positions
3276 .entry((report.instrument_id, report.account_id))
3277 .or_default()
3278 .push(report);
3279 }
3280
3281 venue_positions
3282 .into_iter()
3283 .map(|(key, reports)| (key, distinct_position_reports(reports)))
3284 .collect()
3285 }
3286
3287 #[must_use]
3289 pub fn get_external_order_claim(&self, instrument_id: &InstrumentId) -> Option<StrategyId> {
3290 self.cache.borrow().external_order_claim(instrument_id)
3291 }
3292
3293 pub fn claim_external_orders(
3299 &mut self,
3300 instrument_id: InstrumentId,
3301 strategy_id: StrategyId,
3302 ) -> anyhow::Result<()> {
3303 self.cache
3304 .borrow_mut()
3305 .register_external_order_claims(strategy_id, &[instrument_id])
3306 }
3307
3308 pub fn observe_order_event(&mut self, event: &OrderEventAny) {
3318 match event {
3319 OrderEventAny::Filled(fill) => {
3320 self.record_position_activity(fill.instrument_id, fill.account_id);
3321 }
3322 OrderEventAny::Accepted(_)
3323 | OrderEventAny::Rejected(_)
3324 | OrderEventAny::Canceled(_)
3325 | OrderEventAny::Expired(_)
3326 | OrderEventAny::Denied(_)
3327 | OrderEventAny::Updated(_)
3328 | OrderEventAny::ModifyRejected(_)
3329 | OrderEventAny::CancelRejected(_) => {
3330 self.clear_recon_tracking(&event.client_order_id(), true);
3331 }
3332 _ => {}
3333 }
3334
3335 self.record_local_activity(event.client_order_id());
3336 }
3337
3338 pub fn observe_execution_report(&mut self, report: &ExecutionReport) {
3351 match report {
3352 ExecutionReport::Order(order_report) => {
3353 self.observe_order_report(order_report);
3354 }
3355 ExecutionReport::Fill(fill_report) => {
3356 let client_order_id = fill_report.client_order_id.or_else(|| {
3357 self.cache
3358 .borrow()
3359 .client_order_id(&fill_report.venue_order_id)
3360 .copied()
3361 });
3362
3363 if let Some(coid) = client_order_id {
3364 self.record_local_activity(coid);
3365 }
3366
3367 self.record_position_activity(fill_report.instrument_id, fill_report.account_id);
3368 }
3369 ExecutionReport::OrderWithFills(order_report, fills) => {
3370 self.observe_order_report(order_report);
3371
3372 for fill_report in fills {
3373 self.record_position_activity(
3374 fill_report.instrument_id,
3375 fill_report.account_id,
3376 );
3377 }
3378 }
3379 ExecutionReport::Position(position_report) => {
3380 self.record_position_activity(
3381 position_report.instrument_id,
3382 position_report.account_id,
3383 );
3384 }
3385 ExecutionReport::MassStatus(_) => {
3386 }
3388 }
3389 }
3390
3391 fn confirm_submitted_report(&mut self, report: &OrderStatusReport) -> bool {
3392 if report.order_status == OrderStatus::Submitted
3393 && let Some(client_order_id) = report.client_order_id
3394 && self.submission_retention_enabled(client_order_id, None)
3395 && let Some(order) = self.get_order(client_order_id)
3396 && self
3397 .submissions
3398 .get(&client_order_id)
3399 .is_none_or(|submission| {
3400 submission.trader_id == order.trader_id()
3401 && submission.strategy_id == order.strategy_id()
3402 && submission.instrument_id == order.instrument_id()
3403 })
3404 && report.instrument_id == order.instrument_id()
3405 && order.account_id() == Some(report.account_id)
3406 && order
3407 .venue_order_id()
3408 .is_none_or(|id| id == report.venue_order_id)
3409 && self
3410 .cache
3411 .borrow()
3412 .client_order_id(&report.venue_order_id)
3413 .is_none_or(|id| *id == client_order_id)
3414 {
3415 if Self::submission_is_unacknowledged(&order) {
3416 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
3417 self.track_submission(order.init_event(), client_id);
3418 }
3419
3420 if let Some(submission) = self.submissions.get_mut(&client_order_id) {
3421 submission.venue_confirmed = true;
3422 }
3423
3424 self.clear_recon_tracking(&client_order_id, false);
3425 self.record_local_activity(client_order_id);
3426 return true;
3427 }
3428
3429 false
3430 }
3431
3432 fn observe_order_report(&mut self, report: &OrderStatusReport) {
3433 let Some(client_order_id) = report.client_order_id else {
3434 return;
3435 };
3436
3437 if self.confirm_submitted_report(report) {
3438 return;
3439 }
3440
3441 let accepted_during_pending_command = report.order_status == OrderStatus::Accepted
3442 && self.get_order(client_order_id).is_some_and(|order| {
3443 matches!(
3444 order.status(),
3445 OrderStatus::PendingUpdate | OrderStatus::PendingCancel
3446 )
3447 });
3448
3449 if !matches!(
3450 report.order_status,
3451 OrderStatus::PendingUpdate | OrderStatus::PendingCancel
3452 ) && !accepted_during_pending_command
3453 {
3454 self.clear_recon_tracking(&client_order_id, report.order_status.is_closed());
3455 }
3456
3457 self.record_local_activity(client_order_id);
3461 }
3462
3463 pub fn purge_closed_orders(&mut self) {
3465 let Some(buffer_mins) = self.config.purge_closed_orders_buffer_mins else {
3466 return;
3467 };
3468
3469 let ts_now = self.timestamp_ns();
3470 let buffer_secs = mins_to_secs(u64::from(buffer_mins));
3471
3472 self.cache
3473 .borrow_mut()
3474 .purge_closed_orders(ts_now, buffer_secs);
3475 }
3476
3477 pub fn purge_closed_positions(&mut self) {
3479 let Some(buffer_mins) = self.config.purge_closed_positions_buffer_mins else {
3480 return;
3481 };
3482
3483 let ts_now = self.clock.borrow().timestamp_ns();
3484 let buffer_secs = mins_to_secs(u64::from(buffer_mins));
3485
3486 self.cache
3487 .borrow_mut()
3488 .purge_closed_positions(ts_now, buffer_secs);
3489 }
3490
3491 pub fn purge_account_events(&mut self) {
3493 let Some(lookback_mins) = self.config.purge_account_events_lookback_mins else {
3494 return;
3495 };
3496
3497 let ts_now = self.clock.borrow().timestamp_ns();
3498 let lookback_secs = mins_to_secs(u64::from(lookback_mins));
3499
3500 self.cache
3501 .borrow_mut()
3502 .purge_account_events(ts_now, lookback_secs);
3503 }
3504
3505 fn get_order(&self, client_order_id: ClientOrderId) -> Option<OrderAny> {
3506 self.cache().order(&client_order_id).map(|o| o.clone())
3507 }
3508
3509 fn get_order_by_venue_order_id(&self, venue_order_id: VenueOrderId) -> Option<OrderAny> {
3510 let cache = self.cache();
3511 cache
3512 .client_order_id(&venue_order_id)
3513 .and_then(|client_order_id| cache.order(client_order_id).map(|o| o.clone()))
3514 }
3515
3516 fn get_instrument(&self, instrument_id: &InstrumentId) -> Option<InstrumentAny> {
3517 self.cache().instrument(instrument_id).cloned()
3518 }
3519
3520 fn should_skip_order_report(&self, report: &OrderStatusReport) -> bool {
3521 let client_order_id = report.client_order_id.or_else(|| {
3522 self.cache
3523 .borrow()
3524 .client_order_id(&report.venue_order_id)
3525 .copied()
3526 });
3527
3528 if let Some(client_order_id) = client_order_id
3529 && self
3530 .config
3531 .filtered_client_order_ids
3532 .contains(&client_order_id)
3533 {
3534 log::debug!(
3535 "Skipping order report {client_order_id}: in filtered_client_order_ids list"
3536 );
3537 return true;
3538 }
3539
3540 if !self.should_reconcile_instrument(&report.instrument_id) {
3541 log::debug!(
3542 "Skipping order report for {}: not in reconciliation_instrument_ids",
3543 report.instrument_id
3544 );
3545 return true;
3546 }
3547
3548 false
3549 }
3550
3551 fn should_reconcile_instrument(&self, instrument_id: &InstrumentId) -> bool {
3553 self.config.reconciliation_instrument_ids.is_empty()
3554 || self
3555 .config
3556 .reconciliation_instrument_ids
3557 .contains(instrument_id)
3558 }
3559
3560 fn prepare_missing_order_query(&mut self, client_order_id: ClientOrderId) -> Option<OrderAny> {
3561 let order = self.get_order(client_order_id)?;
3562
3563 if self.submission_is_retained(client_order_id) {
3564 return None;
3565 }
3566
3567 if order.status().is_closed() {
3571 log::debug!(
3572 "Skipping missing-order resolution for {client_order_id}: already {}",
3573 order.status()
3574 );
3575 self.clear_recon_tracking(&client_order_id, true);
3576 return None;
3577 }
3578
3579 if self.order_activity.within(
3583 &client_order_id,
3584 Duration::from(self.config.open_check_threshold_ns),
3585 ) {
3586 return None;
3587 }
3588
3589 if order.is_inflight() && Self::submission_is_unacknowledged(&order) {
3590 let client_id = self.cache.borrow().client_id(&client_order_id).copied();
3591 self.track_submission(order.init_event(), client_id);
3592 }
3593
3594 let retries = self.order_recon_retries.entry(client_order_id).or_insert(0);
3595 *retries = retries.saturating_add(1);
3596
3597 if *retries < self.config.open_check_missing_retries {
3598 log::debug!(
3599 "Order {} not found at venue, retry {}/{}",
3600 client_order_id,
3601 retries,
3602 self.config.open_check_missing_retries
3603 );
3604 return None;
3605 }
3606
3607 Some(order)
3608 }
3609
3610 fn resolve_missing_order(&mut self, client_order_id: ClientOrderId) -> Vec<OrderEventAny> {
3611 let mut events = Vec::new();
3612
3613 let Some(order) = self.get_order(client_order_id) else {
3614 return events;
3615 };
3616
3617 if order.status().is_closed() {
3618 log::debug!(
3619 "Skipping missing-order resolution for {client_order_id}: already {}",
3620 order.status()
3621 );
3622 self.clear_recon_tracking(&client_order_id, true);
3623 return events;
3624 }
3625
3626 if self.order_activity.within(
3627 &client_order_id,
3628 Duration::from(self.config.open_check_threshold_ns),
3629 ) {
3630 log::debug!(
3631 "Deferring missing-order resolution for {client_order_id}: recent local activity"
3632 );
3633 return events;
3634 }
3635
3636 let retries = self.recon_check_retry_count(&client_order_id);
3637 let ts_now = self.clock.borrow().timestamp_ns();
3638
3639 let exhausted_retries = retries.max(self.config.open_check_missing_retries);
3642
3643 if self.submission_recovery_pending(client_order_id)
3644 && !self.order_inflight_checks.contains_key(&client_order_id)
3645 {
3646 self.exhaust_submission(
3647 client_order_id,
3648 SubmissionRecoverySource::MissingOrder,
3649 exhausted_retries,
3650 );
3651 return events;
3652 }
3653
3654 match order.status() {
3655 OrderStatus::Accepted | OrderStatus::Submitted => {
3656 if order.status() == OrderStatus::Submitted
3657 && self.exhaust_submission(
3658 client_order_id,
3659 SubmissionRecoverySource::MissingOrder,
3660 exhausted_retries,
3661 )
3662 {
3663 return events;
3664 }
3665 log::warn!(
3666 "Order {client_order_id} not found at venue after {retries} retries and a targeted query, marking as REJECTED"
3667 );
3668
3669 if let Some(rejected) =
3670 create_reconciliation_rejected(&order, Some("NOT_FOUND_AT_VENUE"), ts_now)
3671 {
3672 events.push(rejected);
3673 }
3674 }
3675 OrderStatus::PartiallyFilled => {
3676 log::warn!(
3677 "Order {client_order_id} not found at venue after {retries} retries and a targeted query, marking as CANCELED"
3678 );
3679 events.push(create_reconciliation_canceled(&order, ts_now));
3680 }
3681 OrderStatus::PendingUpdate | OrderStatus::PendingCancel => {
3682 log::debug!(
3683 "Deferring resolution for {client_order_id}: still inflight as {}",
3684 order.status()
3685 );
3686
3687 if self.submission_recovery_pending(client_order_id) {
3688 return events;
3689 }
3690
3691 self.order_recon_retries.shift_remove(&client_order_id);
3700 if let Some(check) = self.order_inflight_checks.get_mut(&client_order_id) {
3701 check.retry_count = 0;
3702 check.last_query_at = Some(dst::time::Instant::now());
3703 }
3704
3705 self.order_query_recency.mark(client_order_id);
3706 return events;
3707 }
3708 status => {
3709 log::warn!(
3710 "Skipping missing-order resolution for {client_order_id}: unexpected status {status}"
3711 );
3712 }
3713 }
3714
3715 self.clear_recon_tracking(&client_order_id, true);
3716 events
3717 }
3718
3719 fn position_quantity_comparison(
3721 &self,
3722 key: InstrumentAccountKey,
3723 venue_reports: &[PositionStatusReport],
3724 ) -> PositionQuantityComparison {
3725 let (instrument_id, account_id) = key;
3726
3727 let cached_positions = {
3728 let cache = self.cache.borrow();
3729 cache
3730 .positions_open(None, Some(&instrument_id), None, Some(&account_id), None)
3731 .into_iter()
3732 .map(|position| (*position).clone())
3733 .collect::<Vec<_>>()
3734 };
3735
3736 let (cached_signed_qty, cached_long_qty, cached_short_qty) =
3737 position_qty_aggregates(cached_positions.iter().map(Position::signed_decimal_qty));
3738 let (venue_signed_qty, venue_long_qty, venue_short_qty) =
3739 position_qty_aggregates(venue_reports.iter().map(|report| report.signed_decimal_qty));
3740 let nonflat_count = venue_reports
3741 .iter()
3742 .filter(|report| report.signed_decimal_qty != Decimal::ZERO)
3743 .count();
3744 let venue_report = venue_reports
3745 .iter()
3746 .find(|report| report.signed_decimal_qty != Decimal::ZERO)
3747 .or_else(|| venue_reports.last())
3748 .cloned();
3749 let venue_has_side_reports = venue_reports.iter().any(PositionStatusReport::is_long)
3750 && venue_reports.iter().any(PositionStatusReport::is_short);
3751
3752 PositionQuantityComparison {
3753 cached_positions,
3754 cached_signed_qty,
3755 cached_long_qty,
3756 cached_short_qty,
3757 venue_signed_qty,
3758 venue_long_qty,
3759 venue_short_qty,
3760 nonflat_count,
3761 venue_report,
3762 venue_has_side_reports,
3763 }
3764 }
3765
3766 fn check_position_discrepancy(
3767 &mut self,
3768 key: InstrumentAccountKey,
3769 venue_reports: &[PositionStatusReport],
3770 ) -> Option<Vec<OrderEventAny>> {
3771 let (instrument_id, account_id) = key;
3772 let comparison = self.position_quantity_comparison(key, venue_reports);
3773 let tolerance = self.position_reconciliation_tolerance(account_id);
3774 let quantities_match = comparison.quantities_match(tolerance);
3775 let report_shape = comparison.report_shape();
3776 let PositionQuantityComparison {
3777 cached_positions,
3778 cached_signed_qty,
3779 cached_long_qty,
3780 cached_short_qty,
3781 venue_signed_qty,
3782 venue_long_qty,
3783 venue_short_qty,
3784 venue_report,
3785 ..
3786 } = comparison;
3787
3788 if quantities_match {
3789 self.clear_position_reconciliation(&key);
3790 return None;
3791 }
3792
3793 if !self.config.generate_missing_orders {
3794 log::debug!(
3795 "Discrepancy for {instrument_id} position when `generate_missing_orders` disabled, skipping"
3796 );
3797 return None;
3798 }
3799
3800 let ts_now = self.clock.borrow().timestamp_ns();
3801
3802 if self.position_activity_is_recent(&key) {
3804 log::debug!(
3805 "Skipping position reconciliation for {instrument_id}: recent activity within threshold"
3806 );
3807 return None;
3808 }
3809
3810 let retries = self.position_reconciliation_retries(&key, report_shape);
3811
3812 if retries >= self.config.position_check_retries {
3813 return None;
3814 }
3815
3816 if report_shape == PositionReportShape::MultiLeg {
3817 let new_retries = retries + 1;
3818 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3819 log::warn!(
3820 "Deferring position reconciliation for {instrument_id}/{account_id}: venue reports have ambiguous side aggregates (cached net={cached_signed_qty}, long={cached_long_qty}, short={cached_short_qty}; venue net={venue_signed_qty}, long={venue_long_qty}, short={venue_short_qty})"
3821 );
3822
3823 if new_retries >= self.config.position_check_retries {
3824 log::error!(
3825 "Position discrepancy for {instrument_id}/{account_id} unresolved after {} attempts; no further reconciliation attempts will be made for the current report shape",
3826 self.config.position_check_retries,
3827 );
3828 }
3829
3830 return None;
3831 }
3832
3833 log::warn!(
3834 "Position discrepancy detected for {instrument_id}: cached_signed_qty={cached_signed_qty}, venue_signed_qty={venue_signed_qty}"
3835 );
3836
3837 let Some(instrument) = self.get_instrument(&instrument_id) else {
3838 log::debug!("Cannot reconcile position for {instrument_id}: instrument not in cache");
3839 let new_retries = retries + 1;
3840 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3841 if new_retries >= self.config.position_check_retries {
3842 log::error!(
3843 "Position discrepancy for {instrument_id} unresolved after {} attempts \
3844 (cached_qty={cached_signed_qty}, venue_qty={venue_signed_qty}); \
3845 no further reconciliation attempts will be made for the current report shape",
3846 self.config.position_check_retries,
3847 );
3848 }
3849
3850 return None;
3851 };
3852
3853 let cached_avg_px = position_avg_px(&cached_positions);
3854 let venue_avg_px = venue_report.as_ref().and_then(|r| r.avg_px_open);
3855 let venue_position_id = venue_report
3856 .as_ref()
3857 .and_then(|report| report.venue_position_id);
3858 let venue_ts_last = venue_report.as_ref().map_or(ts_now, |r| r.ts_last);
3859
3860 let crosses_zero = (cached_signed_qty > Decimal::ZERO && venue_signed_qty < Decimal::ZERO)
3861 || (cached_signed_qty < Decimal::ZERO && venue_signed_qty > Decimal::ZERO);
3862
3863 let result = if crosses_zero {
3864 let position_ids = match venue_position_id {
3865 Some(open_position_id) => match cached_positions.as_slice() {
3866 [position] => Some((Some(position.id), Some(open_position_id))),
3867 _ => {
3868 log::warn!(
3869 "Deferring hedge cross-zero reconciliation for {instrument_id}/{account_id}: cached and venue position identities are ambiguous"
3870 );
3871 None
3872 }
3873 },
3874 None => Some((None, None)),
3875 };
3876
3877 position_ids.and_then(|(close_position_id, open_position_id)| {
3878 self.reconcile_cross_zero_position(
3879 &instrument,
3880 account_id,
3881 cached_signed_qty,
3882 cached_avg_px,
3883 venue_signed_qty,
3884 venue_avg_px,
3885 close_position_id,
3886 open_position_id,
3887 ts_now,
3888 venue_ts_last,
3889 )
3890 })
3891 } else {
3892 let qty_diff = venue_signed_qty - cached_signed_qty;
3893
3894 let order_side = if qty_diff > Decimal::ZERO {
3895 OrderSide::Buy
3896 } else {
3897 OrderSide::Sell
3898 };
3899
3900 let reconciliation_px = calculate_reconciliation_price(
3901 cached_signed_qty,
3902 cached_avg_px,
3903 venue_signed_qty,
3904 venue_avg_px,
3905 venue_report
3906 .as_ref()
3907 .map(|report| report.avg_px_open_reconciliation)
3908 .unwrap_or_default(),
3909 venue_report
3910 .as_ref()
3911 .and_then(|report| report.avg_px_open_precision),
3912 );
3913
3914 reconciliation_px
3915 .or(venue_avg_px)
3916 .or(cached_avg_px)
3917 .and_then(|fill_px| {
3918 let quantity =
3919 Quantity::from_decimal_dp(qty_diff.abs(), instrument.size_precision())
3920 .ok()?;
3921 let order_report = create_position_reconciliation_report(
3922 &instrument,
3923 account_id,
3924 order_side,
3925 quantity,
3926 fill_px,
3927 venue_position_id,
3928 None,
3929 ts_now,
3930 venue_ts_last,
3931 );
3932
3933 log::info!(
3934 color = LogColor::Blue as u8;
3935 "Generating synthetic fill for position reconciliation {instrument_id}: side={order_side:?}, qty={}, px={fill_px}", qty_diff.abs(),
3936 );
3937
3938 let (events, _) = self.handle_external_order(
3939 &order_report,
3940 account_id,
3941 &instrument,
3942 &[],
3943 true,
3944 None,
3945 None,
3946 );
3947 Some(events)
3948 })
3949 };
3950
3951 if result.as_ref().is_none_or(Vec::is_empty) {
3953 let new_retries = retries + 1;
3954 self.set_position_reconciliation_retries(key, report_shape, new_retries);
3955 if new_retries >= self.config.position_check_retries {
3956 log::error!(
3957 "Position discrepancy for {} unresolved after {} attempts \
3958 (cached_qty={}, venue_qty={}); \
3959 no further reconciliation attempts will be made for the current report shape",
3960 instrument_id,
3961 self.config.position_check_retries,
3962 cached_signed_qty,
3963 venue_signed_qty,
3964 );
3965 }
3966 } else {
3967 self.clear_position_reconciliation(&key);
3968 }
3969
3970 result
3971 }
3972
3973 #[expect(clippy::too_many_arguments)]
3976 fn reconcile_cross_zero_position(
3977 &self,
3978 instrument: &InstrumentAny,
3979 account_id: AccountId,
3980 cached_signed_qty: Decimal,
3981 cached_avg_px: Option<Decimal>,
3982 venue_signed_qty: Decimal,
3983 venue_avg_px: Option<Decimal>,
3984 close_position_id: Option<PositionId>,
3985 open_position_id: Option<PositionId>,
3986 ts_now: UnixNanos,
3987 venue_ts_last: UnixNanos,
3988 ) -> Option<Vec<OrderEventAny>> {
3989 let instrument_id = instrument.id();
3990
3991 log::info!(
3992 color = LogColor::Blue as u8;
3993 "Position crosses zero for {instrument_id}: cached={cached_signed_qty}, venue={venue_signed_qty}. Splitting into two fills",
3994 );
3995
3996 let close_qty = cached_signed_qty.abs();
3997
3998 let close_side = if cached_signed_qty < Decimal::ZERO {
3999 OrderSide::Buy } else {
4001 OrderSide::Sell };
4003
4004 let open_qty = venue_signed_qty.abs();
4005
4006 let open_side = if venue_signed_qty > Decimal::ZERO {
4007 OrderSide::Buy } else {
4009 OrderSide::Sell };
4011
4012 let Some(close_px) = cached_avg_px else {
4013 log::warn!("Cannot close position for {instrument_id}: no cached average price");
4014 return None;
4015 };
4016
4017 let open_report = match venue_avg_px {
4018 Some(open_px) => {
4019 let quantity =
4020 Quantity::from_decimal_dp(open_qty, instrument.size_precision()).ok()?;
4021 let report = create_position_reconciliation_report(
4022 instrument,
4023 account_id,
4024 open_side,
4025 quantity,
4026 open_px,
4027 open_position_id,
4028 Some("OPEN"),
4029 ts_now,
4030 venue_ts_last,
4031 );
4032 Some((report, open_px))
4033 }
4034 None => None,
4035 };
4036
4037 let quantity = Quantity::from_decimal_dp(close_qty, instrument.size_precision()).ok()?;
4038 let close_report = create_position_reconciliation_report(
4039 instrument,
4040 account_id,
4041 close_side,
4042 quantity,
4043 close_px,
4044 close_position_id,
4045 Some("CLOSE"),
4046 ts_now,
4047 venue_ts_last,
4048 );
4049
4050 log::info!(
4051 color = LogColor::Blue as u8;
4052 "Generating close fill for cross-zero {instrument_id}: side={close_side:?}, qty={close_qty}, px={close_px}",
4053 );
4054
4055 let (close_events, _) = self.handle_external_order(
4056 &close_report,
4057 account_id,
4058 instrument,
4059 &[],
4060 true,
4061 None,
4062 None,
4063 );
4064 let mut all_events = close_events;
4065
4066 if let Some((open_report, open_px)) = open_report {
4067 log::info!(
4068 color = LogColor::Blue as u8;
4069 "Generating open fill for cross-zero {instrument_id}: side={open_side:?}, qty={open_qty}, px={open_px}",
4070 );
4071
4072 let (open_events, _) = self.handle_external_order(
4073 &open_report,
4074 account_id,
4075 instrument,
4076 &[],
4077 true,
4078 None,
4079 None,
4080 );
4081 all_events.extend(open_events);
4082 } else {
4083 log::warn!("Cannot open new position for {instrument_id}: no venue average price");
4084 }
4085
4086 Some(all_events)
4087 }
4088
4089 fn reconcile_position_report(
4090 &self,
4091 report: &PositionStatusReport,
4092 account_id: AccountId,
4093 instruments_with_unattributed_fills: &IndexSet<InstrumentId>,
4094 ) -> Option<Vec<OrderEventAny>> {
4095 match report.venue_position_id {
4096 Some(venue_position_id) => self.reconcile_position_report_hedging(
4097 report,
4098 venue_position_id,
4099 account_id,
4100 instruments_with_unattributed_fills,
4101 ),
4102 None => self.reconcile_position_report_netting(report, account_id),
4103 }
4104 }
4105
4106 fn reconcile_position_report_hedging(
4107 &self,
4108 report: &PositionStatusReport,
4109 venue_position_id: PositionId,
4110 account_id: AccountId,
4111 instruments_with_unattributed_fills: &IndexSet<InstrumentId>,
4112 ) -> Option<Vec<OrderEventAny>> {
4113 if instruments_with_unattributed_fills.contains(&report.instrument_id) {
4116 log::debug!(
4117 "Skipping hedge position {venue_position_id} reconciliation: unattributed fills in batch"
4118 );
4119 return None;
4120 }
4121
4122 log::debug!(
4123 "Reconciling HEDGE position for {}, venue_position_id={}",
4124 report.instrument_id,
4125 venue_position_id
4126 );
4127
4128 let position = {
4129 let cache = self.cache.borrow();
4130 cache.position_owned(&venue_position_id)
4131 };
4132
4133 match position {
4134 Some(position) => {
4135 let cached_signed_qty = position.signed_decimal_qty();
4136 let venue_signed_qty = report.signed_decimal_qty;
4137
4138 if cached_signed_qty == venue_signed_qty {
4139 log::debug!(
4140 "Hedge position {venue_position_id} matches venue: qty={cached_signed_qty}"
4141 );
4142 return None;
4143 }
4144
4145 if !self.config.generate_missing_orders {
4146 log::error!(
4147 "Cannot reconcile {} {}: position net qty {} != reported net qty {} \
4148 and `generate_missing_orders` is disabled",
4149 report.instrument_id,
4150 venue_position_id,
4151 cached_signed_qty,
4152 venue_signed_qty
4153 );
4154 return None;
4155 }
4156
4157 self.reconcile_hedge_position_discrepancy(
4158 report,
4159 venue_position_id,
4160 account_id,
4161 &position,
4162 cached_signed_qty,
4163 )
4164 }
4165 None => {
4166 if report.signed_decimal_qty == Decimal::ZERO {
4167 return None;
4168 }
4169
4170 if !self.config.generate_missing_orders {
4171 log::error!(
4172 "Cannot reconcile position: {venue_position_id} not found and `generate_missing_orders` is disabled"
4173 );
4174 return None;
4175 }
4176
4177 self.reconcile_missing_hedge_position(report, venue_position_id, account_id)
4178 }
4179 }
4180 }
4181
4182 fn reconcile_hedge_position_discrepancy(
4183 &self,
4184 report: &PositionStatusReport,
4185 venue_position_id: PositionId,
4186 account_id: AccountId,
4187 position: &Position,
4188 cached_signed_qty: Decimal,
4189 ) -> Option<Vec<OrderEventAny>> {
4190 let instrument = self.get_instrument(&report.instrument_id)?;
4191 let venue_signed_qty = report.signed_decimal_qty;
4192
4193 let diff = (cached_signed_qty - venue_signed_qty).abs();
4194 let diff_qty = Quantity::from_decimal_dp(diff, instrument.size_precision()).ok()?;
4195
4196 if diff_qty.is_zero() {
4197 log::debug!(
4198 "Difference quantity rounds to zero for {}, skipping",
4199 instrument.id()
4200 );
4201 return None;
4202 }
4203
4204 log::warn!(
4205 "Hedge position discrepancy for {} {}: cached={}, venue={}, generating reconciliation order",
4206 report.instrument_id,
4207 venue_position_id,
4208 cached_signed_qty,
4209 venue_signed_qty
4210 );
4211
4212 let current_avg_px = if position.avg_px_open > 0.0 {
4213 Decimal::from_str(&position.avg_px_open.to_string()).ok()
4214 } else {
4215 None
4216 };
4217
4218 self.create_position_reconciliation_order(
4219 report,
4220 account_id,
4221 &instrument,
4222 cached_signed_qty,
4223 diff_qty,
4224 current_avg_px,
4225 )
4226 }
4227
4228 fn reconcile_missing_hedge_position(
4229 &self,
4230 report: &PositionStatusReport,
4231 venue_position_id: PositionId,
4232 account_id: AccountId,
4233 ) -> Option<Vec<OrderEventAny>> {
4234 let instrument = self.get_instrument(&report.instrument_id)?;
4235 let venue_signed_qty = report.signed_decimal_qty;
4236
4237 let qty = venue_signed_qty.abs();
4238 let diff_qty = Quantity::from_decimal_dp(qty, instrument.size_precision()).ok()?;
4239
4240 if diff_qty.is_zero() {
4241 return None;
4242 }
4243
4244 log::warn!(
4245 "Missing hedge position for {} {}: venue reports {}, generating reconciliation order",
4246 report.instrument_id,
4247 venue_position_id,
4248 venue_signed_qty
4249 );
4250
4251 self.create_position_reconciliation_order(
4252 report,
4253 account_id,
4254 &instrument,
4255 Decimal::ZERO,
4256 diff_qty,
4257 None,
4258 )
4259 }
4260
4261 fn reconcile_position_report_netting(
4262 &self,
4263 report: &PositionStatusReport,
4264 account_id: AccountId,
4265 ) -> Option<Vec<OrderEventAny>> {
4266 let instrument_id = report.instrument_id;
4267
4268 log::debug!("Reconciling NET position for {instrument_id}");
4269
4270 let instrument = self.get_instrument(&instrument_id)?;
4271
4272 let (cached_signed_qty, cached_avg_px) = {
4273 let cache = self.cache.borrow();
4274 let positions =
4275 cache.positions_open(None, Some(&instrument_id), None, Some(&account_id), None);
4276 let cached_signed_qty = positions
4277 .iter()
4278 .map(|position| position.signed_decimal_qty())
4279 .sum::<Decimal>();
4280 let cached_avg_px = position_avg_px(positions.iter().map(|position| &**position));
4281
4282 (cached_signed_qty, cached_avg_px)
4283 };
4284
4285 let venue_signed_qty = report.signed_decimal_qty;
4286
4287 log::debug!("venue_signed_qty={venue_signed_qty}, cached_signed_qty={cached_signed_qty}");
4288
4289 let diff = (cached_signed_qty - venue_signed_qty).abs();
4290 let tolerance = self.position_reconciliation_tolerance(account_id);
4291 if diff <= tolerance {
4292 log::debug!("Position quantities match for {instrument_id}, no reconciliation needed");
4293 return None;
4294 }
4295
4296 if !self.config.generate_missing_orders {
4297 log::debug!(
4298 "Discrepancy for {instrument_id} position when `generate_missing_orders` disabled, skipping"
4299 );
4300 return None;
4301 }
4302
4303 let diff_qty = Quantity::from_decimal_dp(diff, instrument.size_precision()).ok()?;
4304
4305 if diff_qty.is_zero() {
4306 log::debug!(
4307 "Difference quantity rounds to zero for {instrument_id}, skipping order generation"
4308 );
4309 return None;
4310 }
4311
4312 let crosses_zero = (cached_signed_qty > Decimal::ZERO && venue_signed_qty < Decimal::ZERO)
4313 || (cached_signed_qty < Decimal::ZERO && venue_signed_qty > Decimal::ZERO);
4314
4315 if crosses_zero {
4316 let ts_now = self.clock.borrow().timestamp_ns();
4317 return self.reconcile_cross_zero_position(
4318 &instrument,
4319 account_id,
4320 cached_signed_qty,
4321 cached_avg_px,
4322 venue_signed_qty,
4323 report.avg_px_open,
4324 None,
4325 None,
4326 ts_now,
4327 report.ts_last,
4328 );
4329 }
4330
4331 if cached_signed_qty == Decimal::ZERO {
4332 return self.create_position_from_report(report, account_id, &instrument);
4333 }
4334
4335 self.create_position_reconciliation_order(
4336 report,
4337 account_id,
4338 &instrument,
4339 cached_signed_qty,
4340 diff_qty,
4341 cached_avg_px,
4342 )
4343 }
4344
4345 fn create_position_from_report(
4350 &self,
4351 report: &PositionStatusReport,
4352 account_id: AccountId,
4353 instrument: &InstrumentAny,
4354 ) -> Option<Vec<OrderEventAny>> {
4355 let instrument_id = report.instrument_id;
4356 let venue_signed_qty = report.signed_decimal_qty;
4357
4358 if venue_signed_qty == Decimal::ZERO {
4359 return None;
4360 }
4361
4362 let order_side = if venue_signed_qty > Decimal::ZERO {
4363 OrderSide::Buy
4364 } else {
4365 OrderSide::Sell
4366 };
4367
4368 let qty_abs = venue_signed_qty.abs();
4369 let venue_avg_px = report.avg_px_open?;
4370
4371 let ts_now = self.clock.borrow().timestamp_ns();
4372 let order_qty = Quantity::from_decimal_dp(qty_abs, instrument.size_precision()).ok()?;
4373 let order_report = create_position_reconciliation_report(
4374 instrument,
4375 account_id,
4376 order_side,
4377 order_qty,
4378 venue_avg_px,
4379 report.venue_position_id,
4380 None,
4381 ts_now,
4382 report.ts_last,
4383 );
4384
4385 log::info!(
4386 color = LogColor::Blue as u8;
4387 "Creating position from venue report for {instrument_id}: side={order_side:?}, qty={qty_abs}, avg_px={venue_avg_px}",
4388 );
4389
4390 let (events, _) = self.handle_external_order(
4391 &order_report,
4392 account_id,
4393 instrument,
4394 &[],
4395 true,
4396 None,
4397 None,
4398 );
4399 Some(events)
4400 }
4401
4402 fn create_position_reconciliation_order(
4403 &self,
4404 report: &PositionStatusReport,
4405 account_id: AccountId,
4406 instrument: &InstrumentAny,
4407 cached_signed_qty: Decimal,
4408 diff_qty: Quantity,
4409 current_avg_px: Option<Decimal>,
4410 ) -> Option<Vec<OrderEventAny>> {
4411 let venue_signed_qty = report.signed_decimal_qty;
4412 let instrument_id = report.instrument_id;
4413
4414 let order_side = if venue_signed_qty > cached_signed_qty {
4415 OrderSide::Buy
4416 } else {
4417 OrderSide::Sell
4418 };
4419
4420 let reconciliation_px = calculate_reconciliation_price(
4421 cached_signed_qty,
4422 current_avg_px,
4423 venue_signed_qty,
4424 report.avg_px_open,
4425 report.avg_px_open_reconciliation,
4426 report.avg_px_open_precision,
4427 );
4428
4429 let fill_px = reconciliation_px
4430 .or(report.avg_px_open)
4431 .or(current_avg_px)?;
4432
4433 let ts_now = self.clock.borrow().timestamp_ns();
4434 let order_report = create_position_reconciliation_report(
4435 instrument,
4436 account_id,
4437 order_side,
4438 diff_qty,
4439 fill_px,
4440 report.venue_position_id,
4441 None,
4442 ts_now,
4443 report.ts_last,
4444 );
4445
4446 log::info!(
4447 color = LogColor::Blue as u8;
4448 "Generating reconciliation order for {instrument_id}: side={order_side:?}, qty={diff_qty}, px={fill_px}",
4449 );
4450
4451 let (events, _) = self.handle_external_order(
4452 &order_report,
4453 account_id,
4454 instrument,
4455 &[],
4456 true,
4457 None,
4458 None,
4459 );
4460 Some(events)
4461 }
4462
4463 fn reconcile_order_report(
4464 &self,
4465 order: &OrderAny,
4466 report: &OrderStatusReport,
4467 instrument: Option<&InstrumentAny>,
4468 commission_client: Option<&dyn ExecutionClient>,
4469 ) -> anyhow::Result<Vec<OrderEventAny>> {
4470 let has_missing_fills = terminal_report_has_missing_fills(report, order.filled_qty());
4471 anyhow::ensure!(
4472 !has_missing_fills,
4473 "terminal report for {} has unaccounted fills; waiting for fill reports",
4474 order.client_order_id(),
4475 );
4476 let ts_now = self.clock.borrow().timestamp_ns();
4477
4478 let commission = if matches!(
4479 report.order_status,
4480 OrderStatus::PartiallyFilled | OrderStatus::Filled
4481 ) && report.filled_qty > order.filled_qty()
4482 && let Some(instrument) = instrument
4483 {
4484 let fill_qty = report.filled_qty - order.filled_qty();
4485 let price_and_liquidity =
4486 incremental_inferred_fill_price_and_liquidity(order, report, instrument);
4487
4488 resolve_inferred_fill_commission(
4489 fill_qty,
4490 price_and_liquidity,
4491 instrument,
4492 commission_client,
4493 )?
4494 } else {
4495 None
4496 };
4497
4498 if matches!(
4499 report.order_status,
4500 OrderStatus::Canceled | OrderStatus::Expired
4501 ) && order.status() == OrderStatus::Filled
4502 && report.filled_qty == order.filled_qty()
4503 {
4504 return Ok(Vec::new());
4505 }
4506
4507 Ok(
4508 reconcile_order_report_with_commission(order, report, instrument, ts_now, commission)
4509 .into_iter()
4510 .collect(),
4511 )
4512 }
4513
4514 #[expect(
4516 clippy::too_many_arguments,
4517 reason = "Snapshot and continuous reports share fill projection"
4518 )]
4519 fn reconcile_order_with_fills(
4520 &mut self,
4521 is_snapshot: bool,
4522 order: &OrderAny,
4523 report: &OrderStatusReport,
4524 fills: &[&FillReport],
4525 instrument: Option<&InstrumentAny>,
4526 fill_queue: &mut ReconciliationFillQueue,
4527 commission_client: Option<&dyn ExecutionClient>,
4528 ) -> Vec<OrderEventAny> {
4529 self.confirm_submitted_report(report);
4530 let mut events = Vec::new();
4531 let mut working = order.clone();
4532 let mut sorted_fills: Vec<&FillReport> = fills.to_vec();
4533 sorted_fills.sort_by_key(|f| f.ts_event);
4534
4535 let ts_now = self.clock.borrow().timestamp_ns();
4536
4537 if matches!(
4538 report.order_status,
4539 OrderStatus::Canceled | OrderStatus::Expired
4540 ) && report.ts_triggered.is_some()
4541 && working.status() != OrderStatus::Triggered
4542 && TRIGGERABLE_ORDER_TYPES.contains(&working.order_type())
4543 {
4544 let triggered = create_reconciliation_triggered(&working, report, ts_now);
4545 if working.apply(triggered.clone()).is_ok() {
4546 events.push(triggered);
4547 }
4548 }
4549
4550 let requires_snapshot_projection = !sorted_fills.is_empty()
4551 || is_snapshot
4552 && (report.order_status == OrderStatus::Voided
4553 || report.filled_qty < working.filled_qty());
4554
4555 if !requires_snapshot_projection {
4556 match self.reconcile_order_report(&working, report, instrument, commission_client) {
4557 Ok(order_events) => events.extend(order_events),
4558 Err(e) => log::error!(
4559 "Deferring order reconciliation for {}: {e}",
4560 order.client_order_id(),
4561 ),
4562 }
4563
4564 return events;
4565 }
4566
4567 for event in generate_reconciliation_order_pre_fill_events(&working, report, ts_now) {
4568 if let Err(e) = working.apply(event.clone()) {
4569 log::warn!(
4570 "Cannot project reconciliation event for {}: {e}",
4571 order.client_order_id()
4572 );
4573 return events;
4574 }
4575
4576 events.push(event);
4577 }
4578
4579 if let Some(inst) = instrument {
4580 for fill in sorted_fills {
4581 let Some((event, fill_key)) =
4582 self.create_order_fill(&working, fill, inst, &fill_queue.pending_fill_keys)
4583 else {
4584 continue;
4585 };
4586
4587 if let Err(e) = working.apply(OrderEventAny::Filled(event.clone())) {
4588 if self.is_fill_applied(&event, fill_key) {
4589 self.fills_processed.mark(fill_key);
4590 continue;
4591 }
4592
4593 log::warn!(
4594 "Cannot project reconciliation fill for {}: {e}",
4595 order.client_order_id()
4596 );
4597 return events;
4598 }
4599
4600 fill_queue.push(&mut events, event, fill_key);
4601 }
4602 }
4603
4604 if !is_snapshot {
4606 match self.reconcile_order_report(&working, report, instrument, commission_client) {
4607 Ok(order_events) => events.extend(order_events),
4608 Err(e) => log::warn!("Deferring order reconciliation: {e}"),
4609 }
4610
4611 return events;
4612 }
4613
4614 if terminal_report_has_missing_fills(report, working.filled_qty()) {
4615 log::warn!(
4616 "Deferring terminal reconciliation for {}: fill reports are incomplete",
4617 order.client_order_id(),
4618 );
4619 return events;
4620 }
4621
4622 let commission = if report.filled_qty > working.filled_qty()
4623 && let Some(instrument) = instrument
4624 {
4625 let fill_qty = report.filled_qty - working.filled_qty();
4626
4627 let price_and_liquidity =
4628 incremental_inferred_fill_price_and_liquidity(&working, report, instrument);
4629
4630 match resolve_inferred_fill_commission(
4631 fill_qty,
4632 price_and_liquidity,
4633 instrument,
4634 commission_client,
4635 ) {
4636 Ok(commission) => commission,
4637 Err(e) => {
4638 log::error!(
4639 "Deferring inferred fill for {}: venue commission calculation failed: {e}",
4640 order.client_order_id(),
4641 );
4642 return events;
4643 }
4644 }
4645 } else {
4646 None
4647 };
4648
4649 for event in generate_reconciliation_order_snapshot_events_with_commission(
4650 &working, report, instrument, ts_now, commission,
4651 ) {
4652 if let Err(e) = working.apply(event.clone()) {
4653 log::warn!(
4654 "Cannot project reconciliation snapshot event for {}: {e}",
4655 order.client_order_id()
4656 );
4657 break;
4658 }
4659
4660 events.push(event);
4661 }
4662
4663 events
4664 }
4665
4666 #[expect(clippy::too_many_arguments)]
4667 fn handle_external_order(
4668 &self,
4669 report: &OrderStatusReport,
4670 account_id: AccountId,
4671 instrument: &InstrumentAny,
4672 fills: &[&FillReport],
4673 is_synthetic: bool,
4674 fill_queue: Option<&mut ReconciliationFillQueue>,
4675 commission_client: Option<&dyn ExecutionClient>,
4676 ) -> (Vec<OrderEventAny>, Option<ExternalOrderMetadata>) {
4677 let claimed_strategy = self
4678 .cache
4679 .borrow()
4680 .external_order_claim(&report.instrument_id);
4681
4682 let (strategy_id, tags) = if let Some(claimed_strategy) = claimed_strategy {
4683 let order_id = report
4684 .client_order_id
4685 .map_or_else(|| report.venue_order_id.to_string(), |id| id.to_string());
4686 log::info!(
4687 color = LogColor::Blue as u8;
4688 "External order {} for {} claimed by strategy {}",
4689 order_id,
4690 report.instrument_id,
4691 claimed_strategy,
4692 );
4693
4694 (
4696 claimed_strategy,
4697 is_synthetic.then(|| vec![*TAG_RECONCILIATION]),
4698 )
4699 } else {
4700 let tag = if is_synthetic {
4702 *TAG_RECONCILIATION
4703 } else {
4704 *TAG_VENUE
4705 };
4706
4707 (StrategyId::from("EXTERNAL"), Some(vec![tag]))
4708 };
4709
4710 if self.config.filter_unclaimed_external && claimed_strategy.is_none() && !is_synthetic {
4712 return (Vec::new(), None);
4713 }
4714
4715 let client_order_id = report
4716 .client_order_id
4717 .unwrap_or_else(|| ClientOrderId::from(report.venue_order_id.as_str()));
4718
4719 if !report.quantity.is_positive() {
4720 log::error!(
4721 "Skipping external order {} ({}) for {}: non-positive quantity in report {:?}",
4722 client_order_id,
4723 report.venue_order_id,
4724 report.instrument_id,
4725 report,
4726 );
4727 return (Vec::new(), None);
4728 }
4729
4730 let ts_now = self.clock.borrow().timestamp_ns();
4731
4732 let Some(order_side) = report.order_side else {
4733 log::error!(
4734 "Skipping external order {} ({}) for {}: order side is not specified",
4735 client_order_id,
4736 report.venue_order_id,
4737 report.instrument_id,
4738 );
4739 return (Vec::new(), None);
4740 };
4741
4742 let initialized = match OrderInitialized::new_checked(
4743 self.config.trader_id,
4744 strategy_id,
4745 report.instrument_id,
4746 client_order_id,
4747 order_side,
4748 report.order_type,
4749 report.quantity,
4750 report.time_in_force,
4751 report.post_only,
4752 report.reduce_only,
4753 false, true, UUID4::new(),
4756 ts_now,
4757 ts_now,
4758 report.price,
4759 report.activation_price,
4760 report.trigger_price,
4761 report.trigger_type,
4762 report.limit_offset,
4763 report.trailing_offset,
4764 report.trailing_offset_type,
4765 report.expire_time,
4766 report.display_qty,
4767 None, None, report.contingency_type,
4770 report.order_list_id,
4771 report.linked_order_ids.clone(),
4772 report.parent_order_id,
4773 None, None, None, tags,
4777 ) {
4778 Ok(initialized) => initialized,
4779 Err(e) => {
4780 log::error!("Failed to create order from report: {e}");
4781 return (Vec::new(), None);
4782 }
4783 };
4784
4785 let initialized = OrderEventAny::Initialized(initialized);
4786
4787 let order = match OrderAny::from_events(vec![initialized.clone()]) {
4788 Ok(order) => order,
4789 Err(e) => {
4790 log::error!("Failed to create order from report: {e}");
4791 return (Vec::new(), None);
4792 }
4793 };
4794
4795 let replace_inferred_fill = !fills.is_empty()
4796 && matches!(
4797 report.order_status,
4798 OrderStatus::Accepted
4799 | OrderStatus::Canceled
4800 | OrderStatus::Expired
4801 | OrderStatus::Filled
4802 | OrderStatus::PartiallyFilled
4803 | OrderStatus::Triggered
4804 );
4805 let mut prepared_fills = Vec::new();
4806 let mut prepared_fill_keys = fill_queue
4807 .as_deref()
4808 .map(|queue| queue.pending_fill_keys.clone())
4809 .unwrap_or_default();
4810 let mut real_fill_total = Decimal::ZERO;
4811
4812 if replace_inferred_fill {
4813 let mut sorted_fills: Vec<&FillReport> = fills.to_vec();
4814 sorted_fills.sort_by_key(|fill| fill.ts_event);
4815
4816 if fill_queue.is_none() {
4817 log::error!(
4818 "Cannot reconcile external order {client_order_id}: fill queue is unavailable"
4819 );
4820 return (Vec::new(), None);
4821 }
4822
4823 for fill in sorted_fills {
4824 if let Some((fill_event, fill_key)) =
4825 self.create_order_fill(&order, fill, instrument, &prepared_fill_keys)
4826 {
4827 real_fill_total += fill.last_qty.as_decimal();
4828 prepared_fill_keys.insert(fill_key);
4829 prepared_fills.push((fill_event, fill_key));
4830 }
4831 }
4832 }
4833
4834 let report_filled = report.filled_qty.as_decimal();
4835
4836 let inferred_qty = if report_filled.is_zero()
4837 || matches!(
4838 report.order_status,
4839 OrderStatus::Accepted | OrderStatus::Triggered
4840 ) {
4841 None
4842 } else if replace_inferred_fill {
4843 if real_fill_total < report_filled {
4844 match Quantity::from_decimal_dp(
4845 report_filled - real_fill_total,
4846 instrument.size_precision(),
4847 ) {
4848 Ok(quantity) => Some(quantity),
4849 Err(e) => {
4850 log::error!(
4851 "Cannot reconcile external order {client_order_id}: residual fill quantity is invalid: {e}"
4852 );
4853 return (Vec::new(), None);
4854 }
4855 }
4856 } else {
4857 None
4858 }
4859 } else if matches!(
4860 report.order_status,
4861 OrderStatus::PartiallyFilled
4862 | OrderStatus::Filled
4863 | OrderStatus::Canceled
4864 | OrderStatus::Expired
4865 | OrderStatus::Voided
4866 ) {
4867 Some(report.filled_qty)
4868 } else {
4869 None
4870 };
4871
4872 let defer_terminal = !is_synthetic
4873 && claimed_strategy.is_some()
4874 && matches!(
4875 report.order_status,
4876 OrderStatus::Canceled | OrderStatus::Expired
4877 )
4878 && inferred_qty.is_some();
4879
4880 if defer_terminal {
4881 log::warn!(
4882 "Deferring terminal reconciliation for claimed order {client_order_id}: fill reports are incomplete"
4883 );
4884
4885 if prepared_fills.is_empty() {
4886 return (Vec::new(), None);
4887 }
4888 }
4889
4890 let inferred_commission = if is_synthetic || defer_terminal {
4891 None
4892 } else if let Some(inferred_qty) = inferred_qty {
4893 let price_and_liquidity = inferred_fill_price_and_liquidity(&order, report, instrument);
4894
4895 match resolve_inferred_fill_commission(
4896 inferred_qty,
4897 price_and_liquidity,
4898 instrument,
4899 commission_client,
4900 ) {
4901 Ok(commission) => commission,
4902 Err(e) => {
4903 log::error!(
4904 "Deferring external order {client_order_id}: venue commission calculation failed: {e}"
4905 );
4906 return (Vec::new(), None);
4907 }
4908 }
4909 } else {
4910 None
4911 };
4912
4913 {
4914 let mut cache = self.cache.borrow_mut();
4915
4916 let source_client_id = if is_synthetic {
4917 None
4918 } else {
4919 commission_client.map(ExecutionClient::client_id)
4920 };
4921
4922 if let Err(e) = cache.add_order(order.clone(), None, source_client_id, false) {
4923 match cache.order(&client_order_id) {
4927 Some(existing) if is_synthetic && existing.is_closed() => {
4928 log::debug!(
4929 "Skipping synthetic reconciliation order {client_order_id} for {}: \
4930 replay deduped (cached status={:?})",
4931 report.instrument_id,
4932 existing.status(),
4933 );
4934 }
4935 Some(existing) if is_synthetic => {
4936 log::warn!(
4937 "Synthetic reconciliation order {client_order_id} for {} exists in \
4938 cache in non-terminal state {:?}; fill not regenerated",
4939 report.instrument_id,
4940 existing.status(),
4941 );
4942 }
4943 _ => {
4944 log::error!("Failed to add external order to cache: {e}");
4945 }
4946 }
4947
4948 return (Vec::new(), None);
4949 }
4950
4951 if let Err(e) = cache.index_venue_order_id(&client_order_id, &report.venue_order_id) {
4952 log::warn!("Failed to index venue order ID: {e}");
4953 }
4954 }
4955
4956 Self::publish_order_event(&initialized);
4957
4958 log::info!(
4959 color = LogColor::Blue as u8;
4960 "Created external order {} ({}) for {} [{}]",
4961 client_order_id,
4962 report.venue_order_id,
4963 report.instrument_id,
4964 report.order_status,
4965 );
4966
4967 let ts_now = self.clock.borrow().timestamp_ns();
4968 let mut order_events = generate_external_order_status_events_with_commission(
4969 &order,
4970 report,
4971 &account_id,
4972 instrument,
4973 ts_now,
4974 inferred_commission,
4975 );
4976
4977 if replace_inferred_fill {
4978 let terminal_event = if order_events.last().is_some_and(|event| {
4979 matches!(
4980 event,
4981 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_),
4982 )
4983 }) {
4984 order_events.pop()
4985 } else {
4986 None
4987 };
4988
4989 if order_events
4990 .last()
4991 .is_some_and(|event| matches!(event, OrderEventAny::Filled(_)))
4992 {
4993 order_events.pop();
4994 }
4995
4996 let fill_queue =
4997 fill_queue.expect("fill queue availability was checked before cache mutation");
4998
4999 for (fill_event, fill_key) in prepared_fills {
5000 fill_queue.push(&mut order_events, fill_event, fill_key);
5001 }
5002
5003 if !defer_terminal
5004 && let Some(inferred_qty) = inferred_qty
5005 && let Some(inferred_fill) = create_inferred_fill_for_qty(
5006 &order,
5007 report,
5008 &account_id,
5009 instrument,
5010 inferred_qty,
5011 ts_now,
5012 inferred_commission,
5013 )
5014 {
5015 order_events.push(inferred_fill);
5016 }
5017
5018 if !defer_terminal && let Some(event) = terminal_event {
5019 order_events.push(event);
5020 }
5021 }
5022
5023 let metadata = ExternalOrderMetadata {
5024 client_order_id,
5025 venue_order_id: report.venue_order_id,
5026 instrument_id: report.instrument_id,
5027 strategy_id,
5028 ts_init: ts_now,
5029 };
5030
5031 (order_events, Some(metadata))
5032 }
5033
5034 fn publish_order_event(event: &OrderEventAny) {
5035 let topic = switchboard::get_event_order_topic(event.strategy_id());
5036 msgbus::publish_order_event(topic, event);
5037 }
5038
5039 fn adjust_mass_status_fills(&self, mass_status: &ExecutionMassStatus) -> AdjustedReports {
5044 let mut final_orders: IndexMap<VenueOrderId, OrderStatusReport> =
5045 mass_status.order_reports();
5046 let mut final_fills: IndexMap<VenueOrderId, Vec<FillReport>> = mass_status.fill_reports();
5047 let mut order_only_ids = IndexSet::new();
5048 let mut order_only_fill_keys = IndexSet::new();
5049
5050 final_fills.retain(|_, fills| {
5051 fills.retain(|fill| {
5052 if fill.last_qty.is_zero() {
5053 log::warn!("Skipping zero-quantity fill report: {fill}");
5054 return false;
5055 }
5056
5057 true
5058 });
5059
5060 !fills.is_empty()
5061 });
5062
5063 if self.config.filter_position_reports {
5064 return AdjustedReports {
5065 orders: final_orders,
5066 fills: final_fills,
5067 order_only_ids,
5068 order_only_fill_keys,
5069 };
5070 }
5071
5072 let mut instruments_to_adjust = Vec::new();
5073
5074 for (instrument_id, position_reports) in mass_status.position_reports() {
5075 if !self.should_reconcile_instrument(&instrument_id) {
5076 log::debug!(
5077 "Skipping fill adjustment for {instrument_id}: not in reconciliation_instrument_ids"
5078 );
5079 continue;
5080 }
5081
5082 let is_hedge_mode = position_reports
5085 .iter()
5086 .any(|r| r.venue_position_id.is_some());
5087
5088 if is_hedge_mode {
5089 log::debug!(
5090 "Skipping fill adjustment for {instrument_id}: hedge mode (has venue_position_id)"
5091 );
5092 continue;
5093 }
5094
5095 let has_retained_position = {
5096 let cache = self.cache.borrow();
5097 !cache
5098 .positions_open(
5099 None,
5100 Some(&instrument_id),
5101 None,
5102 Some(&mass_status.account_id),
5103 None,
5104 )
5105 .is_empty()
5106 };
5107
5108 if has_retained_position {
5109 log::debug!(
5110 "Skipping fill adjustment for {instrument_id}: retained open position in cache"
5111 );
5112 continue;
5113 }
5114
5115 if let Some(instrument) = self.get_instrument(&instrument_id) {
5116 instruments_to_adjust.push(instrument);
5117 } else {
5118 log::debug!(
5119 "Skipping fill adjustment for {instrument_id}: instrument not found in cache"
5120 );
5121 }
5122 }
5123
5124 if instruments_to_adjust.is_empty() {
5125 return AdjustedReports {
5126 orders: final_orders,
5127 fills: final_fills,
5128 order_only_ids,
5129 order_only_fill_keys,
5130 };
5131 }
5132
5133 log_info!(
5134 "Adjusting fills for {} instrument(s) with position reports",
5135 instruments_to_adjust.len(),
5136 color = LogColor::Blue
5137 );
5138
5139 for instrument in &instruments_to_adjust {
5140 let instrument_id = instrument.id();
5141
5142 let result = if self.config.generate_missing_orders {
5143 process_mass_status_for_reconciliation(mass_status, instrument, None)
5144 } else {
5145 process_mass_status_for_reconciliation_without_synthetic_reports(
5146 mass_status,
5147 instrument,
5148 None,
5149 )
5150 };
5151
5152 match result {
5153 Ok(result) => {
5154 order_only_ids.extend(result.order_only_ids);
5155 order_only_fill_keys.extend(result.order_only_fill_keys);
5156 final_orders.retain(|_, order| order.instrument_id != instrument_id);
5157 final_fills.retain(|_, fills| {
5158 fills
5159 .first()
5160 .is_none_or(|f| f.instrument_id != instrument_id)
5161 });
5162
5163 final_orders.extend(result.orders);
5164 final_fills.extend(result.fills);
5165 }
5166 Err(e) => {
5167 log::warn!("Failed to adjust fills for {instrument_id}: {e}");
5168 }
5169 }
5170 }
5171
5172 log_info!(
5173 "After adjustment: {} order(s), {} fill group(s)",
5174 final_orders.len(),
5175 final_fills.len(),
5176 color = LogColor::Blue
5177 );
5178
5179 AdjustedReports {
5180 orders: final_orders,
5181 fills: final_fills,
5182 order_only_ids,
5183 order_only_fill_keys,
5184 }
5185 }
5186
5187 fn is_fill_applied(&self, fill: &OrderFilled, fill_key: FillKey) -> bool {
5188 if fill.last_qty.is_zero() {
5189 return false;
5190 }
5191
5192 let cache = self.cache();
5193 cache
5194 .order(&fill.client_order_id)
5195 .or_else(|| {
5196 cache
5197 .client_order_id(&fill.venue_order_id)
5198 .and_then(|client_order_id| cache.order(client_order_id))
5199 })
5200 .is_some_and(|order| {
5201 order.account_id() == Some(fill_key.0)
5202 && order.instrument_id() == fill_key.1
5203 && order.trade_ids().contains(&&fill_key.2)
5204 })
5205 }
5206
5207 fn create_order_fill(
5208 &self,
5209 order: &OrderAny,
5210 fill: &FillReport,
5211 instrument: &InstrumentAny,
5212 pending_fill_keys: &IndexSet<FillKey>,
5213 ) -> Option<(OrderFilled, FillKey)> {
5214 if fill.last_qty.is_zero() {
5215 log::warn!("Skipping zero-quantity fill report: {fill}");
5216 return None;
5217 }
5218
5219 let fill_key = (fill.account_id, fill.instrument_id, fill.trade_id);
5220 if self.fills_processed.contains_key(&fill_key) || pending_fill_keys.contains(&fill_key) {
5221 return None;
5222 }
5223
5224 let order_side = order.order_side();
5225 if fill.order_side != order_side {
5226 log::warn!(
5227 "Fill side mismatch for {}: cached={:?}, venue={:?}",
5228 order.client_order_id(),
5229 order_side,
5230 fill.order_side,
5231 );
5232 }
5233
5234 let ts_now = self.clock.borrow().timestamp_ns();
5235
5236 let event = OrderFilled::new(
5237 order.trader_id(),
5238 order.strategy_id(),
5239 order.instrument_id(),
5240 order.client_order_id(),
5241 fill.venue_order_id,
5242 fill.account_id,
5243 fill.trade_id,
5244 fill.order_side,
5245 order.order_type(),
5246 fill.last_qty,
5247 fill.last_px,
5248 instrument.quote_currency(),
5249 fill.liquidity_side,
5250 fill.report_id,
5251 fill.ts_event,
5252 ts_now,
5253 true, fill.venue_position_id,
5255 Some(fill.commission),
5256 None,
5257 );
5258
5259 Some((event, fill_key))
5260 }
5261}
5262
5263fn position_fill_report_totals(
5264 position: &Position,
5265 report: &FillReport,
5266 client_order_id: Option<ClientOrderId>,
5267 apply_voids: bool,
5268) -> Option<(Quantity, Money)> {
5269 let mut fills = position
5270 .replay_events
5271 .iter()
5272 .filter_map(|event| match event {
5273 PositionReplayEvent::Filled(fill)
5274 if fill.position_id == Some(position.id)
5275 && fill.account_id == report.account_id
5276 && fill.instrument_id == report.instrument_id
5277 && fill.venue_order_id == report.venue_order_id
5278 && fill.trade_id == report.trade_id
5279 && fill.order_side == report.order_side
5280 && fill.last_px == report.last_px
5281 && fill.liquidity_side == report.liquidity_side
5282 && client_order_id.is_none_or(|id| fill.client_order_id == id) =>
5283 {
5284 Some(fill)
5285 }
5286 _ => None,
5287 });
5288
5289 let (mut position_quantity, mut position_commission) = fills.try_fold(
5290 (
5291 Quantity::zero(report.last_qty.precision),
5292 Money::zero(report.commission.currency),
5293 ),
5294 |(quantity, commission), fill| {
5295 let fill_commission = fill.commission?;
5296 if fill_commission.currency != report.commission.currency {
5297 return None;
5298 }
5299
5300 Some((
5301 quantity.checked_add(fill.last_qty)?,
5302 commission.checked_add(fill_commission)?,
5303 ))
5304 },
5305 )?;
5306
5307 if apply_voids
5308 && let Some(correction) = position.fill_voids.iter().rev().find(|correction| {
5309 correction.event.trade_id == report.trade_id
5310 && correction.event.venue_order_id == report.venue_order_id
5311 && client_order_id.is_none_or(|id| correction.event.client_order_id == id)
5312 })
5313 {
5314 let corrected_quantity = position_quantity.checked_sub(correction.voided_qty)?;
5315 let corrected_commission = position_commission.checked_sub(
5316 correction
5317 .commission_voided
5318 .unwrap_or(Money::zero(report.commission.currency)),
5319 )?;
5320 position_quantity = corrected_quantity;
5321 position_commission = corrected_commission;
5322 }
5323
5324 Some((position_quantity, position_commission))
5325}
5326
5327#[cfg(test)]
5328mod tests {
5329 use nautilus_common::{clock::VirtualClock, config::ConfigError};
5330 use nautilus_core::{DurationNanos, Params};
5331 use nautilus_execution::reconciliation::generate_reconciliation_order_events;
5332 use nautilus_model::{
5333 accounts::AccountAny,
5334 enums::{LiquiditySide, OmsType, OrderType, PositionSide, TimeInForce},
5335 events::order::spec::{OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec},
5336 identifiers::{Symbol, Venue},
5337 instruments::{
5338 CurrencyPair, Instrument,
5339 stubs::{btcusd_bybit, crypto_perpetual_ethusdt},
5340 },
5341 orders::{OrderTestBuilder, stubs::TestOrderEventStubs},
5342 types::{AccountBalance, Currency, MarginBalance, Money, Price, fixed::FIXED_PRECISION},
5343 };
5344 use rstest::rstest;
5345 use rust_decimal_macros::dec;
5346
5347 use super::{
5348 super::reconciliation::tests::{CommissionOutcome, CommissionStubClient},
5349 *,
5350 };
5351
5352 #[rstest]
5353 fn test_new_validates_open_check_lookback_mins_boundaries() {
5354 let create_manager = |mins| {
5355 ExecutionManager::new(
5356 Rc::new(RefCell::new(VirtualClock::new())),
5357 Rc::new(RefCell::new(Cache::default())),
5358 ExecutionManagerConfig {
5359 open_check_lookback_mins: Some(mins),
5360 ..Default::default()
5361 },
5362 )
5363 };
5364
5365 assert!(create_manager(307_445_734).is_ok());
5366 assert!(matches!(
5367 create_manager(307_445_735),
5368 Err(ConfigError::Range { field, .. })
5369 if field == "ExecutionManagerConfig.open_check_lookback_mins"
5370 ));
5371 }
5372
5373 #[rstest]
5374 fn test_new_validates_reconciliation_lookback_mins_boundaries() {
5375 let create_manager = |mins| {
5376 ExecutionManager::new(
5377 Rc::new(RefCell::new(VirtualClock::new())),
5378 Rc::new(RefCell::new(Cache::default())),
5379 ExecutionManagerConfig {
5380 lookback_mins: Some(mins),
5381 ..Default::default()
5382 },
5383 )
5384 };
5385
5386 assert!(create_manager(307_445_734_561_825_860).is_ok());
5387 assert!(matches!(
5388 create_manager(307_445_734_561_825_861),
5389 Err(ConfigError::Range { field, .. })
5390 if field == "ExecutionManagerConfig.lookback_mins"
5391 ));
5392 }
5393
5394 #[rstest]
5395 fn test_new_reports_every_invalid_lookback_field() {
5396 let error = ExecutionManager::new(
5397 Rc::new(RefCell::new(VirtualClock::new())),
5398 Rc::new(RefCell::new(Cache::default())),
5399 ExecutionManagerConfig {
5400 lookback_mins: Some(307_445_734_561_825_861),
5401 open_check_lookback_mins: Some(307_445_735),
5402 position_check_lookback_mins: 307_445_735,
5403 ..Default::default()
5404 },
5405 )
5406 .expect_err("all lookback fields are out of range");
5407
5408 let ConfigError::Multiple { errors } = error else {
5409 panic!("expected a `Multiple` error, was {error:?}");
5410 };
5411
5412 let fields = errors
5413 .iter()
5414 .map(|e| match e {
5415 ConfigError::Range { field, .. } => field.as_str(),
5416 other => panic!("expected a `Range` error, was {other:?}"),
5417 })
5418 .collect::<Vec<_>>();
5419
5420 assert_eq!(
5421 fields,
5422 [
5423 "ExecutionManagerConfig.lookback_mins",
5424 "ExecutionManagerConfig.open_check_lookback_mins",
5425 "ExecutionManagerConfig.position_check_lookback_mins",
5426 ]
5427 );
5428 }
5429
5430 struct PositionCoverageStubClient;
5431
5432 #[async_trait::async_trait(?Send)]
5433 impl ExecutionClient for PositionCoverageStubClient {
5434 fn is_connected(&self) -> bool {
5435 true
5436 }
5437
5438 fn client_id(&self) -> ClientId {
5439 ClientId::from("BYBIT")
5440 }
5441
5442 fn account_id(&self) -> AccountId {
5443 AccountId::from("TEST-001")
5444 }
5445
5446 fn venue(&self) -> Venue {
5447 Venue::from("BYBIT")
5448 }
5449
5450 fn oms_type(&self) -> OmsType {
5451 OmsType::Netting
5452 }
5453
5454 fn get_account(&self) -> Option<AccountAny> {
5455 None
5456 }
5457
5458 fn provides_bulk_position_coverage(&self, instrument_id: InstrumentId) -> bool {
5459 !instrument_id.symbol.as_str().ends_with("-SPOT")
5460 }
5461
5462 fn generate_account_state(
5463 &self,
5464 _balances: Vec<AccountBalance>,
5465 _margins: Vec<MarginBalance>,
5466 _reported: bool,
5467 _ts_event: UnixNanos,
5468 _info: Option<Params>,
5469 ) -> anyhow::Result<()> {
5470 Ok(())
5471 }
5472
5473 fn start(&mut self) -> anyhow::Result<()> {
5474 Ok(())
5475 }
5476
5477 fn stop(&mut self) -> anyhow::Result<()> {
5478 Ok(())
5479 }
5480 }
5481
5482 fn cached_commission_fixtures() -> (
5483 ExecutionManager,
5484 Rc<RefCell<Cache>>,
5485 OrderAny,
5486 OrderStatusReport,
5487 InstrumentAny,
5488 ) {
5489 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5490 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5491 let cache = Rc::new(RefCell::new(Cache::default()));
5492 cache
5493 .borrow_mut()
5494 .add_instrument(instrument.clone())
5495 .expect("instrument is cacheable");
5496 let client_order_id = ClientOrderId::from("O-COMMISSION-CACHED");
5497 let venue_order_id = VenueOrderId::from("V-COMMISSION-CACHED");
5498 insert_accepted_limit_order(
5499 &cache,
5500 client_order_id,
5501 venue_order_id,
5502 instrument.id(),
5503 ClientId::from("STUB"),
5504 );
5505 let order = cache
5506 .borrow()
5507 .order_owned(&client_order_id)
5508 .expect("accepted order is cached");
5509 let report = OrderStatusReport::new(
5510 AccountId::from("TEST-001"),
5511 instrument.id(),
5512 Some(client_order_id),
5513 venue_order_id,
5514 OrderSide::Buy.into(),
5515 OrderType::Limit,
5516 TimeInForce::Gtc,
5517 OrderStatus::Filled,
5518 Quantity::from("10.0"),
5519 Quantity::from("10.0"),
5520 UnixNanos::from(1),
5521 UnixNanos::from(1),
5522 UnixNanos::from(1),
5523 None,
5524 )
5525 .with_avg_px(dec!(100.0));
5526 let manager =
5527 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5528 .expect("valid config");
5529
5530 (manager, cache, order, report, instrument)
5531 }
5532
5533 fn external_report_with_partial_fill(
5534 instrument: &InstrumentAny,
5535 ) -> (OrderStatusReport, FillReport) {
5536 let account_id = AccountId::from("STUB-001");
5537 let venue_order_id = VenueOrderId::from("V-EXT-1");
5538 let report = OrderStatusReport::new(
5539 account_id,
5540 instrument.id(),
5541 None,
5542 venue_order_id,
5543 OrderSide::Buy.into(),
5544 OrderType::Limit,
5545 TimeInForce::Gtc,
5546 OrderStatus::Filled,
5547 Quantity::from("10.0"),
5548 Quantity::from("10.0"),
5549 UnixNanos::from(1),
5550 UnixNanos::from(1),
5551 UnixNanos::from(1),
5552 None,
5553 )
5554 .with_price(Price::from("100.00"))
5555 .with_avg_px(dec!(100.0));
5556
5557 let fill = FillReport::new(
5558 account_id,
5559 instrument.id(),
5560 venue_order_id,
5561 TradeId::from("T-EXT-1"),
5562 OrderSide::Buy,
5563 Quantity::from("4.0"),
5564 Price::from("100.00"),
5565 Money::new(0.1, Currency::USDT()),
5566 LiquiditySide::Taker,
5567 None,
5568 None,
5569 UnixNanos::from(1),
5570 UnixNanos::from(1),
5571 None,
5572 );
5573
5574 (report, fill)
5575 }
5576
5577 fn inferred_fills(events: &[OrderEventAny]) -> Vec<OrderFilled> {
5578 events
5579 .iter()
5580 .filter_map(|event| match event {
5581 OrderEventAny::Filled(filled) if filled.last_qty == Quantity::from("6.0") => {
5582 Some(filled.clone())
5583 }
5584 _ => None,
5585 })
5586 .collect()
5587 }
5588
5589 #[rstest]
5590 fn test_handle_external_order_applies_venue_commission_to_inferred_fill() {
5591 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5592 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5593 let cache = Rc::new(RefCell::new(Cache::default()));
5594 cache
5595 .borrow_mut()
5596 .add_instrument(instrument.clone())
5597 .expect("instrument is cacheable");
5598 let manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
5599 .expect("valid config");
5600 let (report, fill) = external_report_with_partial_fill(&instrument);
5601 let expected = Money::new(2.5, Currency::USDT());
5602 let client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5603 let mut fill_queue = ReconciliationFillQueue::default();
5604
5605 let (events, _) = manager.handle_external_order(
5606 &report,
5607 AccountId::from("STUB-001"),
5608 &instrument,
5609 &[&fill],
5610 false,
5611 Some(&mut fill_queue),
5612 Some(&client),
5613 );
5614
5615 let inferred = inferred_fills(&events);
5616 assert_eq!(inferred.len(), 1, "one inferred fill covers the 6.0 gap");
5617 assert_eq!(inferred[0].commission, Some(expected));
5618 }
5619
5620 #[rstest]
5621 fn test_handle_external_order_skips_inferred_fill_when_commission_fails() {
5622 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5623 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5624 let cache = Rc::new(RefCell::new(Cache::default()));
5625 cache
5626 .borrow_mut()
5627 .add_instrument(instrument.clone())
5628 .expect("instrument is cacheable");
5629 let manager =
5630 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5631 .expect("valid config");
5632 let (report, fill) = external_report_with_partial_fill(&instrument);
5633 let client = CommissionStubClient::new(CommissionOutcome::Failure);
5634 let mut fill_queue = ReconciliationFillQueue::default();
5635
5636 let (events, metadata) = manager.handle_external_order(
5637 &report,
5638 AccountId::from("STUB-001"),
5639 &instrument,
5640 &[&fill],
5641 false,
5642 Some(&mut fill_queue),
5643 Some(&client),
5644 );
5645
5646 assert!(events.is_empty());
5647 assert!(metadata.is_none());
5648 assert!(fill_queue.pending_fill_keys.is_empty());
5649 assert!(
5650 cache
5651 .borrow()
5652 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5653 .is_none(),
5654 "commission failure must precede external order cache mutation"
5655 );
5656
5657 let expected = Money::new(2.5, Currency::USDT());
5658 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5659 let (retry_events, retry_metadata) = manager.handle_external_order(
5660 &report,
5661 AccountId::from("STUB-001"),
5662 &instrument,
5663 &[&fill],
5664 false,
5665 Some(&mut fill_queue),
5666 Some(&retry_client),
5667 );
5668 let inferred = inferred_fills(&retry_events);
5669
5670 assert!(retry_metadata.is_some());
5671 assert_eq!(inferred.len(), 1);
5672 assert_eq!(inferred[0].commission, Some(expected));
5673 assert_eq!(fill_queue.pending_fill_keys.len(), 1);
5674 assert!(
5675 cache
5676 .borrow()
5677 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5678 .is_some()
5679 );
5680 }
5681
5682 #[rstest]
5683 fn test_handle_external_order_without_explicit_fills_resolves_commission_before_cache() {
5684 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5685 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5686 let cache = Rc::new(RefCell::new(Cache::default()));
5687 cache
5688 .borrow_mut()
5689 .add_instrument(instrument.clone())
5690 .expect("instrument is cacheable");
5691 let manager =
5692 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
5693 .expect("valid config");
5694 let (report, _) = external_report_with_partial_fill(&instrument);
5695 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
5696
5697 let (failed_events, failed_metadata) = manager.handle_external_order(
5698 &report,
5699 AccountId::from("STUB-001"),
5700 &instrument,
5701 &[],
5702 false,
5703 None,
5704 Some(&failing_client),
5705 );
5706
5707 assert!(failed_events.is_empty());
5708 assert!(failed_metadata.is_none());
5709 assert!(
5710 cache
5711 .borrow()
5712 .order(&ClientOrderId::from(report.venue_order_id.as_str()))
5713 .is_none()
5714 );
5715
5716 let expected = Money::new(4.0, Currency::USDT());
5717 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5718 let (retry_events, retry_metadata) = manager.handle_external_order(
5719 &report,
5720 AccountId::from("STUB-001"),
5721 &instrument,
5722 &[],
5723 false,
5724 None,
5725 Some(&retry_client),
5726 );
5727
5728 let fills: Vec<_> = retry_events
5729 .iter()
5730 .filter_map(|event| match event {
5731 OrderEventAny::Filled(fill) => Some(fill),
5732 _ => None,
5733 })
5734 .collect();
5735
5736 assert!(retry_metadata.is_some());
5737 assert_eq!(fills.len(), 1);
5738 assert_eq!(fills[0].last_qty, Quantity::from("10.0"));
5739 assert_eq!(fills[0].commission, Some(expected));
5740 }
5741
5742 #[rstest]
5743 fn test_handle_external_order_with_no_override_emits_fill_without_commission() {
5744 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5745 let clock = Rc::new(RefCell::new(VirtualClock::new()));
5746 let cache = Rc::new(RefCell::new(Cache::default()));
5747 cache
5748 .borrow_mut()
5749 .add_instrument(instrument.clone())
5750 .expect("instrument is cacheable");
5751 let manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
5752 .expect("valid config");
5753 let (report, fill) = external_report_with_partial_fill(&instrument);
5754 let client = CommissionStubClient::new(CommissionOutcome::NoOverride);
5755 let mut fill_queue = ReconciliationFillQueue::default();
5756
5757 let (events, _) = manager.handle_external_order(
5758 &report,
5759 AccountId::from("STUB-001"),
5760 &instrument,
5761 &[&fill],
5762 false,
5763 Some(&mut fill_queue),
5764 Some(&client),
5765 );
5766
5767 let inferred = inferred_fills(&events);
5768 assert_eq!(inferred.len(), 1);
5769 assert_eq!(inferred[0].commission, None);
5770 }
5771
5772 #[rstest]
5773 #[case::filled(
5774 OrderStatus::Filled,
5775 "10.0",
5776 "6.0",
5777 Price::from_decimal_dp(dec!(33.3333333333333333), FIXED_PRECISION).unwrap(),
5778 1
5779 )]
5780 #[case::canceled(OrderStatus::Canceled, "8.0", "4.0", "20.00", 2)]
5781 #[case::expired(OrderStatus::Expired, "8.0", "4.0", "20.00", 2)]
5782 fn test_cached_reconciliation_applies_explicit_fill_and_defers_failed_residual(
5783 #[case] status: OrderStatus,
5784 #[case] filled_qty: Quantity,
5785 #[case] residual_qty: Quantity,
5786 #[case] residual_px: Price,
5787 #[case] event_count: usize,
5788 ) {
5789 let (mut manager, _cache, order, mut report, instrument) = cached_commission_fixtures();
5790 report.order_status = status;
5791 report.filled_qty = filled_qty;
5792 report.avg_px = Some(dec!(60.0));
5793
5794 let explicit_fill = FillReport::new(
5795 report.account_id,
5796 report.instrument_id,
5797 report.venue_order_id,
5798 TradeId::from("T-COMMISSION-EXPLICIT"),
5799 OrderSide::Buy,
5800 Quantity::from("4.0"),
5801 Price::from("100.0"),
5802 Money::new(0.25, Currency::USDT()),
5803 LiquiditySide::Taker,
5804 report.client_order_id,
5805 None,
5806 UnixNanos::from(1),
5807 UnixNanos::from(1),
5808 None,
5809 );
5810 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
5811 let mut fill_queue = ReconciliationFillQueue::default();
5812
5813 let first_events = manager.reconcile_order_with_fills(
5814 true,
5815 &order,
5816 &report,
5817 &[&explicit_fill],
5818 Some(&instrument),
5819 &mut fill_queue,
5820 Some(&failing_client),
5821 );
5822 let mut working = order;
5823
5824 for event in &first_events {
5825 working
5826 .apply(event.clone())
5827 .expect("explicit fill projects cleanly");
5828 }
5829
5830 assert_eq!(first_events.len(), 1);
5831
5832 let OrderEventAny::Filled(explicit) = &first_events[0] else {
5833 panic!("expected the valid explicit fill");
5834 };
5835
5836 assert_eq!(explicit.last_qty, Quantity::from("4.0"));
5837 assert_eq!(
5838 explicit.commission,
5839 Some(Money::new(0.25, Currency::USDT()))
5840 );
5841 assert_eq!(working.status(), OrderStatus::PartiallyFilled);
5842
5843 let expected = Money::new(1.5, Currency::USDT());
5844 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
5845 let mut reported_residual = explicit_fill.clone();
5846 reported_residual.trade_id = TradeId::from("T-COMMISSION-RESIDUAL");
5847 reported_residual.last_qty = residual_qty;
5848 reported_residual.last_px = residual_px;
5849 reported_residual.commission = expected;
5850
5851 let residual_reports = if status == OrderStatus::Filled {
5852 Vec::new()
5853 } else {
5854 vec![&reported_residual]
5855 };
5856
5857 let retry_events = manager.reconcile_order_with_fills(
5858 true,
5859 &working,
5860 &report,
5861 &residual_reports,
5862 Some(&instrument),
5863 &mut fill_queue,
5864 Some(&retry_client),
5865 );
5866
5867 assert_eq!(retry_events.len(), event_count);
5868
5869 let OrderEventAny::Filled(residual) = &retry_events[0] else {
5870 panic!("expected the residual fill");
5871 };
5872
5873 assert_eq!(residual.last_qty, residual_qty);
5874 assert_eq!(residual.last_px, residual_px);
5875 assert_eq!(residual.commission, Some(expected));
5876 assert_eq!(
5877 retry_client.seen(),
5878 (status == OrderStatus::Filled).then_some((
5879 residual_qty,
5880 residual.last_px,
5881 residual.liquidity_side
5882 )),
5883 "commission must use the exact price and liquidity carried by the residual fill"
5884 );
5885
5886 for event in &retry_events {
5887 working
5888 .apply(event.clone())
5889 .expect("residual precedes terminal status");
5890 }
5891
5892 retry_client.clear_seen();
5893 let replay = manager.reconcile_order_with_fills(
5894 true,
5895 &working,
5896 &report,
5897 &[],
5898 Some(&instrument),
5899 &mut fill_queue,
5900 Some(&retry_client),
5901 );
5902
5903 assert_eq!(working.status(), status);
5904 assert_eq!(working.filled_qty(), filled_qty);
5905 assert_eq!(
5906 working.commissions().get(&Currency::USDT()),
5907 Some(&Money::from("1.75 USDT"))
5908 );
5909 assert!(replay.is_empty());
5910 assert_eq!(retry_client.seen(), None);
5911 }
5912
5913 #[rstest]
5914 fn test_cached_reconciliation_preserves_explicit_fill_side() {
5915 let (mut manager, _cache, order, mut report, instrument) = cached_commission_fixtures();
5916 report.order_status = OrderStatus::PartiallyFilled;
5917 report.filled_qty = Quantity::from("4.0");
5918 let trade_id = TradeId::from("T-CONFLICTING-SIDE");
5919
5920 let explicit_fill = FillReport::new(
5921 report.account_id,
5922 report.instrument_id,
5923 report.venue_order_id,
5924 trade_id,
5925 OrderSide::Sell,
5926 Quantity::from("4.0"),
5927 Price::from("100.0"),
5928 Money::new(0.25, Currency::USDT()),
5929 LiquiditySide::Taker,
5930 report.client_order_id,
5931 None,
5932 UnixNanos::from(1),
5933 UnixNanos::from(1),
5934 None,
5935 );
5936 let mut fill_queue = ReconciliationFillQueue::default();
5937
5938 let events = manager.reconcile_order_with_fills(
5939 true,
5940 &order,
5941 &report,
5942 &[&explicit_fill],
5943 Some(&instrument),
5944 &mut fill_queue,
5945 None,
5946 );
5947
5948 assert_eq!(events.len(), 1);
5949
5950 let OrderEventAny::Filled(fill) = &events[0] else {
5951 panic!("expected the explicit fill");
5952 };
5953
5954 assert_eq!(fill.trade_id, trade_id);
5955 assert_eq!(fill.order_side, OrderSide::Sell);
5956 assert_eq!(fill.last_qty, Quantity::from("4.0"));
5957 assert_eq!(fill.last_px, Price::from("100.0"));
5958 }
5959
5960 #[rstest]
5961 fn test_continuous_report_preserves_newer_fills() {
5962 let (mut manager, _cache, mut order, mut report, instrument) = cached_commission_fixtures();
5963 let fill = TestOrderEventStubs::filled(
5964 &order,
5965 &instrument,
5966 Some(TradeId::from("T-NEWER-STREAM")),
5967 None,
5968 Some(Price::from("100.00")),
5969 Some(Quantity::from("2.0")),
5970 Some(LiquiditySide::Maker),
5971 None,
5972 None,
5973 Some(AccountId::from("TEST-001")),
5974 );
5975 order.apply(fill).unwrap();
5976 report.order_status = OrderStatus::PartiallyFilled;
5977 report.filled_qty = Quantity::from("1.0");
5978 let mut fill_queue = ReconciliationFillQueue::default();
5979
5980 let events = manager.reconcile_order_with_fills(
5981 false,
5982 &order,
5983 &report,
5984 &[],
5985 Some(&instrument),
5986 &mut fill_queue,
5987 None,
5988 );
5989
5990 assert!(events.is_empty());
5991 assert!(fill_queue.pending_fill_keys.is_empty());
5992 }
5993
5994 #[rstest]
5995 #[case::with_fills(true)]
5996 #[case::without_fills(false)]
5997 fn test_cached_snapshot_without_instrument_defers_unaccounted_fills(#[case] has_fills: bool) {
5998 let (mut manager, _cache, order, mut report, _instrument) = cached_commission_fixtures();
5999 report.order_status = OrderStatus::Canceled;
6000
6001 let explicit_fill = FillReport::new(
6002 report.account_id,
6003 report.instrument_id,
6004 report.venue_order_id,
6005 TradeId::from("T-MISSING-INSTRUMENT"),
6006 OrderSide::Buy,
6007 Quantity::from("4.0"),
6008 Price::from("100.0"),
6009 Money::new(0.25, Currency::USDT()),
6010 LiquiditySide::Taker,
6011 report.client_order_id,
6012 None,
6013 UnixNanos::from(1),
6014 UnixNanos::from(1),
6015 None,
6016 );
6017 let mut fill_queue = ReconciliationFillQueue::default();
6018
6019 let fills = if has_fills {
6020 vec![&explicit_fill]
6021 } else {
6022 Vec::new()
6023 };
6024
6025 let events = manager.reconcile_order_with_fills(
6026 true,
6027 &order,
6028 &report,
6029 &fills,
6030 None,
6031 &mut fill_queue,
6032 None,
6033 );
6034
6035 assert!(events.is_empty());
6036 assert_eq!(order.status(), OrderStatus::Accepted);
6037 assert_eq!(order.filled_qty(), Quantity::from("0.0"));
6038 assert!(fill_queue.pending_fill_keys.is_empty());
6039 }
6040
6041 #[rstest]
6042 #[case::canceled(OrderStatus::Canceled)]
6043 #[case::expired(OrderStatus::Expired)]
6044 fn test_terminal_order_report_does_not_void_cached_fills(#[case] status: OrderStatus) {
6045 let (manager, _cache, mut order, mut report, instrument) = cached_commission_fixtures();
6046 let fill = create_inferred_fill_for_qty(
6047 &order,
6048 &report,
6049 &report.account_id,
6050 &instrument,
6051 Quantity::from("4.0"),
6052 UnixNanos::from(1),
6053 None,
6054 )
6055 .unwrap();
6056 order.apply(fill).unwrap();
6057 report.order_status = status;
6058 report.filled_qty = Quantity::from("2.0");
6059 let client = CommissionStubClient::new(CommissionOutcome::Failure);
6060
6061 let events = manager
6062 .reconcile_order_report(&order, &report, Some(&instrument), Some(&client))
6063 .unwrap();
6064
6065 for event in &events {
6066 order.apply(event.clone()).unwrap();
6067 }
6068
6069 assert_eq!(events.len(), 1);
6070 assert_eq!(order.status(), status);
6071 assert_eq!(order.filled_qty(), Quantity::from("4.0"));
6072 assert_eq!(client.seen(), None);
6073 }
6074
6075 #[rstest]
6076 fn test_filled_order_ignores_superseded_cancel_report() {
6077 let (manager, cache, order, mut report, instrument) = cached_commission_fixtures();
6078 let venue_order_id = VenueOrderId::from("V-REPLACEMENT");
6079 let updated = OrderUpdatedSpec::builder()
6080 .trader_id(order.trader_id())
6081 .strategy_id(order.strategy_id())
6082 .instrument_id(order.instrument_id())
6083 .client_order_id(order.client_order_id())
6084 .account_id(report.account_id)
6085 .venue_order_id(venue_order_id)
6086 .quantity(order.quantity())
6087 .build();
6088 let order = cache
6089 .borrow_mut()
6090 .update_order(&OrderEventAny::Updated(updated))
6091 .unwrap();
6092 let mut fill_report = report.clone();
6093 fill_report.venue_order_id = venue_order_id;
6094 let commission = Money::from("1.25 USDT");
6095 let fill = create_inferred_fill_for_qty(
6096 &order,
6097 &fill_report,
6098 &report.account_id,
6099 &instrument,
6100 Quantity::from("10.0"),
6101 UnixNanos::from(2),
6102 Some(commission),
6103 )
6104 .unwrap();
6105 let order = cache.borrow_mut().update_order(&fill).unwrap();
6106 report.order_status = OrderStatus::Canceled;
6107 report.filled_qty = Quantity::from("0.0");
6108 report.avg_px = None;
6109 let client = CommissionStubClient::new(CommissionOutcome::Failure);
6110
6111 let events = manager
6112 .reconcile_order_report(&order, &report, Some(&instrument), Some(&client))
6113 .unwrap();
6114
6115 assert!(events.is_empty());
6116 assert_eq!(order.status(), OrderStatus::Filled);
6117 assert_eq!(order.venue_order_id(), Some(venue_order_id));
6118 assert_eq!(order.filled_qty(), Quantity::from("10.0"));
6119 assert_eq!(order.avg_px(), Some(dec!(100.0)));
6120 assert_eq!(
6121 order.commissions().get(&Currency::USDT()),
6122 Some(&commission)
6123 );
6124 assert_eq!(client.seen(), None);
6125 }
6126
6127 #[rstest]
6128 fn test_continuous_reconciliation_uses_source_client_and_retries_commission() {
6129 let (mut manager, _cache, order, report, _instrument) = cached_commission_fixtures();
6130 let client_id = ClientId::from("STUB");
6131
6132 let check = OpenOrderReportCheck {
6133 command: GenerateOrderStatusReports::new(
6134 UUID4::new(),
6135 UnixNanos::from(1),
6136 true,
6137 None,
6138 None,
6139 None,
6140 None,
6141 None,
6142 ),
6143 filtered_orders: vec![order],
6144 client_coverage: IndexMap::from([(
6145 report.client_order_id.unwrap(),
6146 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6147 )]),
6148 };
6149
6150 let queried_clients = IndexSet::from([client_id]);
6151 let failed_clients = IndexSet::new();
6152 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
6153
6154 let failed = manager.reconcile_open_order_reports(
6155 &check,
6156 vec![SourcedOrderStatusReport {
6157 client_id,
6158 report: report.clone(),
6159 }],
6160 &queried_clients,
6161 &failed_clients,
6162 &[&failing_client],
6163 );
6164
6165 assert!(failed.events.is_empty());
6166
6167 let expected = Money::new(1.5, Currency::USDT());
6168 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
6169 let retry = manager.reconcile_open_order_reports(
6170 &check,
6171 vec![SourcedOrderStatusReport { client_id, report }],
6172 &queried_clients,
6173 &failed_clients,
6174 &[&retry_client],
6175 );
6176
6177 assert_eq!(retry.events.len(), 1);
6178
6179 let OrderEventAny::Filled(fill) = &retry.events[0] else {
6180 panic!("expected inferred fill on valid retry");
6181 };
6182
6183 assert_eq!(fill.last_qty, Quantity::from("10.0"));
6184 assert_eq!(fill.commission, Some(expected));
6185 }
6186
6187 #[rstest]
6188 fn test_open_check_lookback_exclusion_warns_once_without_reconciliation_actions() {
6189 let client_order_id = ClientOrderId::from("O-LOOKBACK-OLD");
6190 let venue_order_id = VenueOrderId::from("V-LOOKBACK-OLD");
6191 let client_id = ClientId::from("BINANCE");
6192 let instrument_id = crypto_perpetual_ethusdt().id();
6193 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6194 let cache = Rc::new(RefCell::new(Cache::default()));
6195 insert_accepted_limit_order(
6196 &cache,
6197 client_order_id,
6198 venue_order_id,
6199 instrument_id,
6200 client_id,
6201 );
6202 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6203 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6204
6205 let check = OpenOrderReportCheck {
6206 command: GenerateOrderStatusReports::new(
6207 UUID4::new(),
6208 UnixNanos::from(1),
6209 false,
6210 None,
6211 Some(cutoff),
6212 None,
6213 None,
6214 None,
6215 ),
6216 filtered_orders: vec![order],
6217 client_coverage: IndexMap::from([(
6218 client_order_id,
6219 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6220 )]),
6221 };
6222
6223 let queried_clients = IndexSet::from([client_id]);
6224
6225 let mut manager = ExecutionManager::new(
6226 clock,
6227 cache.clone(),
6228 ExecutionManagerConfig {
6229 open_check_open_only: false,
6230 ..Default::default()
6231 },
6232 )
6233 .expect("valid config");
6234
6235 for _ in 0..2 {
6236 let result = manager.reconcile_open_order_reports(
6237 &check,
6238 Vec::new(),
6239 &queried_clients,
6240 &IndexSet::new(),
6241 &[],
6242 );
6243
6244 assert!(result.events.is_empty());
6245 assert!(result.targeted_queries.is_empty());
6246 assert_eq!(
6247 cache.borrow().order(&client_order_id).unwrap().status(),
6248 OrderStatus::Accepted
6249 );
6250 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
6251 assert!(!manager.order_query_recency.contains_key(&client_order_id));
6252 assert!(!manager.order_query_pending.contains(&client_order_id));
6253 assert_eq!(
6254 manager.order_lookback_warnings,
6255 IndexSet::from([client_order_id])
6256 );
6257 assert_eq!(manager.order_lookback_warnings.len(), 1);
6258 }
6259 }
6260
6261 #[rstest]
6262 fn test_open_check_lookback_warning_clears_at_boundary_and_rearms() {
6263 let client_order_id = ClientOrderId::from("O-LOOKBACK-REARM");
6264 let client_id = ClientId::from("BINANCE");
6265 let cache = Rc::new(RefCell::new(Cache::default()));
6266 insert_accepted_limit_order(
6267 &cache,
6268 client_order_id,
6269 VenueOrderId::from("V-LOOKBACK-REARM"),
6270 crypto_perpetual_ethusdt().id(),
6271 client_id,
6272 );
6273 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6274 let old_cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6275
6276 let make_check = |start| OpenOrderReportCheck {
6277 command: GenerateOrderStatusReports::new(
6278 UUID4::new(),
6279 UnixNanos::from(1),
6280 false,
6281 None,
6282 Some(start),
6283 None,
6284 None,
6285 None,
6286 ),
6287 filtered_orders: vec![order.clone()],
6288 client_coverage: IndexMap::from([(
6289 client_order_id,
6290 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6291 )]),
6292 };
6293
6294 let queried_clients = IndexSet::new();
6295
6296 let mut manager = ExecutionManager::new(
6297 Rc::new(RefCell::new(VirtualClock::new())),
6298 cache,
6299 ExecutionManagerConfig {
6300 open_check_open_only: false,
6301 ..Default::default()
6302 },
6303 )
6304 .expect("valid config");
6305
6306 manager.reconcile_open_order_reports(
6307 &make_check(old_cutoff),
6308 Vec::new(),
6309 &queried_clients,
6310 &IndexSet::new(),
6311 &[],
6312 );
6313 assert!(manager.order_lookback_warnings.contains(&client_order_id));
6314
6315 let boundary = order.ts_last();
6316 let boundary_result = manager.reconcile_open_order_reports(
6317 &make_check(boundary),
6318 Vec::new(),
6319 &queried_clients,
6320 &IndexSet::new(),
6321 &[],
6322 );
6323 assert!(boundary_result.targeted_queries.is_empty());
6324 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
6325 assert!(manager.order_coverage_warnings.contains(&client_order_id));
6326
6327 manager.reconcile_open_order_reports(
6328 &make_check(old_cutoff),
6329 Vec::new(),
6330 &queried_clients,
6331 &IndexSet::new(),
6332 &[],
6333 );
6334 assert_eq!(
6335 manager.order_lookback_warnings,
6336 IndexSet::from([client_order_id])
6337 );
6338 }
6339
6340 #[rstest]
6341 fn test_venue_order_id_mapped_report_clears_old_order_lookback_warning() {
6342 let client_order_id = ClientOrderId::from("O-LOOKBACK-MAPPED");
6343 let venue_order_id = VenueOrderId::from("V-LOOKBACK-MAPPED");
6344 let client_id = ClientId::from("BINANCE");
6345 let instrument_id = crypto_perpetual_ethusdt().id();
6346 let cache = Rc::new(RefCell::new(Cache::default()));
6347 insert_accepted_limit_order(
6348 &cache,
6349 client_order_id,
6350 venue_order_id,
6351 instrument_id,
6352 client_id,
6353 );
6354 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6355 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6356
6357 let check = OpenOrderReportCheck {
6358 command: GenerateOrderStatusReports::new(
6359 UUID4::new(),
6360 UnixNanos::from(1),
6361 false,
6362 None,
6363 Some(cutoff),
6364 None,
6365 None,
6366 None,
6367 ),
6368 filtered_orders: vec![order],
6369 client_coverage: IndexMap::from([(
6370 client_order_id,
6371 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6372 )]),
6373 };
6374
6375 let mut manager = ExecutionManager::new(
6376 Rc::new(RefCell::new(VirtualClock::new())),
6377 cache,
6378 ExecutionManagerConfig {
6379 open_check_open_only: false,
6380 ..Default::default()
6381 },
6382 )
6383 .expect("valid config");
6384
6385 manager.order_lookback_warnings.insert(client_order_id);
6386
6387 let report = OrderStatusReport::new(
6390 AccountId::from("TEST-001"),
6391 instrument_id,
6392 None,
6393 venue_order_id,
6394 OrderSide::Buy.into(),
6395 OrderType::Limit,
6396 TimeInForce::Gtc,
6397 OrderStatus::Accepted,
6398 Quantity::from("10.0"),
6399 Quantity::from("0.0"),
6400 UnixNanos::from(0),
6401 UnixNanos::from(0),
6402 UnixNanos::from(0),
6403 None,
6404 );
6405
6406 let result = manager.reconcile_open_order_reports(
6407 &check,
6408 vec![SourcedOrderStatusReport { client_id, report }],
6409 &IndexSet::from([client_id]),
6410 &IndexSet::new(),
6411 &[],
6412 );
6413
6414 assert!(result.targeted_queries.is_empty());
6415 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
6416 }
6417
6418 #[rstest]
6419 fn test_positive_report_clears_old_order_lookback_warning_without_reinserting_it() {
6420 let client_order_id = ClientOrderId::from("O-LOOKBACK-REPORTED");
6421 let venue_order_id = VenueOrderId::from("V-LOOKBACK-REPORTED");
6422 let client_id = ClientId::from("BINANCE");
6423 let instrument_id = crypto_perpetual_ethusdt().id();
6424 let cache = Rc::new(RefCell::new(Cache::default()));
6425 insert_accepted_limit_order(
6426 &cache,
6427 client_order_id,
6428 venue_order_id,
6429 instrument_id,
6430 client_id,
6431 );
6432 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6433 let cutoff = order.ts_last().saturating_add(DurationNanos::new(1));
6434
6435 let check = OpenOrderReportCheck {
6436 command: GenerateOrderStatusReports::new(
6437 UUID4::new(),
6438 UnixNanos::from(1),
6439 false,
6440 None,
6441 Some(cutoff),
6442 None,
6443 None,
6444 None,
6445 ),
6446 filtered_orders: vec![order],
6447 client_coverage: IndexMap::from([(
6448 client_order_id,
6449 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
6450 )]),
6451 };
6452
6453 let mut manager = ExecutionManager::new(
6454 Rc::new(RefCell::new(VirtualClock::new())),
6455 cache,
6456 ExecutionManagerConfig {
6457 open_check_open_only: false,
6458 ..Default::default()
6459 },
6460 )
6461 .expect("valid config");
6462
6463 manager.order_lookback_warnings.insert(client_order_id);
6464
6465 let report = OrderStatusReport::new(
6466 AccountId::from("TEST-001"),
6467 instrument_id,
6468 Some(client_order_id),
6469 venue_order_id,
6470 OrderSide::Buy.into(),
6471 OrderType::Limit,
6472 TimeInForce::Gtc,
6473 OrderStatus::Accepted,
6474 Quantity::from("10.0"),
6475 Quantity::from("0.0"),
6476 UnixNanos::from(0),
6477 UnixNanos::from(0),
6478 UnixNanos::from(0),
6479 None,
6480 );
6481
6482 let result = manager.reconcile_open_order_reports(
6483 &check,
6484 vec![SourcedOrderStatusReport { client_id, report }],
6485 &IndexSet::from([client_id]),
6486 &IndexSet::new(),
6487 &[],
6488 );
6489
6490 assert!(result.targeted_queries.is_empty());
6491 assert!(!manager.order_lookback_warnings.contains(&client_order_id));
6492 }
6493
6494 #[rstest]
6495 fn test_targeted_reconciliation_uses_source_client_and_retries_commission() {
6496 let (mut manager, _cache, _order, report, _instrument) = cached_commission_fixtures();
6497 let client_order_id = report.client_order_id.unwrap();
6498 let client_id = ClientId::from("STUB");
6499 let failing_client = CommissionStubClient::new(CommissionOutcome::Failure);
6500
6501 let failed = manager.reconcile_targeted_order_reports(
6502 vec![TargetedOrderReportResult {
6503 client_order_id,
6504 client_id: Some(client_id),
6505 report: Some(report.clone()),
6506 fills: Vec::new(),
6507 coverage_complete: true,
6508 }],
6509 &[&failing_client],
6510 );
6511
6512 assert!(failed.is_empty());
6513
6514 let expected = Money::new(1.5, Currency::USDT());
6515 let retry_client = CommissionStubClient::new(CommissionOutcome::Value(expected));
6516 let retry = manager.reconcile_targeted_order_reports(
6517 vec![TargetedOrderReportResult {
6518 client_order_id,
6519 client_id: Some(client_id),
6520 report: Some(report),
6521 fills: Vec::new(),
6522 coverage_complete: true,
6523 }],
6524 &[&retry_client],
6525 );
6526
6527 assert_eq!(retry.len(), 1);
6528
6529 let OrderEventAny::Filled(fill) = &retry[0] else {
6530 panic!("expected inferred fill on valid targeted retry");
6531 };
6532
6533 assert_eq!(fill.last_qty, Quantity::from("10.0"));
6534 assert_eq!(fill.commission, Some(expected));
6535 }
6536
6537 #[rstest]
6538 fn test_clear_recon_tracking_removes_targeted_query() {
6539 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6540 let cache = Rc::new(RefCell::new(Cache::default()));
6541 let mut manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
6542 .expect("valid config");
6543 let client_order_id = ClientOrderId::from("O-TARGETED-CLEAR");
6544 manager.order_query_pending.insert(client_order_id);
6545 manager.order_lookback_warnings.insert(client_order_id);
6546
6547 manager.clear_recon_tracking(&client_order_id, true);
6548
6549 assert!(manager.order_query_pending.is_empty());
6550 assert!(manager.order_lookback_warnings.is_empty());
6551 }
6552
6553 #[rstest]
6554 fn test_submission_confirmation_skips_untracked_order_cache(
6555 #[values(false, true)] previously_registered: bool,
6556 ) {
6557 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6558 let cache = Rc::new(RefCell::new(Cache::default()));
6559 let mut manager = ExecutionManager::new(
6560 clock,
6561 cache.clone(),
6562 ExecutionManagerConfig {
6563 submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
6564 ..Default::default()
6565 },
6566 )
6567 .expect("valid config");
6568 let order = OrderTestBuilder::new(OrderType::Limit)
6569 .instrument_id(InstrumentId::from("ETHUSDT.BYBIT"))
6570 .quantity(Quantity::from("10.0"))
6571 .price(Price::from("100.0"))
6572 .build();
6573 let client_order_id = order.client_order_id();
6574 let account_id = AccountId::from("TEST-001");
6575 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6576 if previously_registered {
6577 manager.register_submission(order.init_event(), None);
6578 }
6579 cache
6580 .borrow_mut()
6581 .add_order(order, None, None, false)
6582 .unwrap();
6583 let order = cache.borrow_mut().update_order(&submitted).unwrap();
6584 let accepted =
6585 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-UNTRACKED"));
6586 cache.borrow_mut().update_order(&accepted).unwrap();
6587
6588 if previously_registered {
6589 manager.confirm_submission_outcome(&client_order_id);
6590 }
6591 assert!(!manager.submissions.contains_key(&client_order_id));
6592 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6593
6594 let cached = cache.borrow_mut();
6596 manager.confirm_submission_outcome(&client_order_id);
6597
6598 assert_eq!(
6599 cached.order(&client_order_id).unwrap().status(),
6600 OrderStatus::Accepted,
6601 );
6602 }
6603
6604 #[rstest]
6605 fn test_register_inflight_skips_filtered_order() {
6606 let client_order_id = ClientOrderId::from("O-FILTERED-REGISTER");
6607 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6608 let cache = Rc::new(RefCell::new(Cache::default()));
6609
6610 let mut manager = ExecutionManager::new(
6611 clock,
6612 cache,
6613 ExecutionManagerConfig {
6614 filtered_client_order_ids: IndexSet::from([client_order_id]),
6615 ..Default::default()
6616 },
6617 )
6618 .expect("valid config");
6619
6620 manager.register_inflight(client_order_id);
6621
6622 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6623 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
6624 }
6625
6626 #[rstest]
6627 #[cfg_attr(
6628 not(all(feature = "simulation", madsim)),
6629 tokio::test(start_paused = true)
6630 )]
6631 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6632 async fn test_inflight_check_retires_order_filtered_after_registration() {
6633 let client_order_id = ClientOrderId::from("O-FILTERED-LATE");
6634 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6635 let cache = Rc::new(RefCell::new(Cache::default()));
6636
6637 let mut manager = ExecutionManager::new(
6638 clock,
6639 cache,
6640 ExecutionManagerConfig {
6641 inflight_threshold_ms: 100,
6642 ..Default::default()
6643 },
6644 )
6645 .expect("valid config");
6646
6647 manager.register_inflight(client_order_id);
6648 manager
6649 .config
6650 .filtered_client_order_ids
6651 .insert(client_order_id);
6652 dst::time::sleep(Duration::from_millis(101)).await;
6653
6654 let first = manager.check_inflight_orders();
6655
6656 assert!(first.events.is_empty());
6657 assert!(first.queries.is_empty());
6658 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6659 assert!(!manager.order_recon_retries.contains_key(&client_order_id));
6660
6661 dst::time::sleep(Duration::from_millis(101)).await;
6662 let second = manager.check_inflight_orders();
6663 assert!(second.events.is_empty());
6664 assert!(second.queries.is_empty());
6665 assert!(!manager.order_inflight_checks.contains_key(&client_order_id));
6666 }
6667
6668 #[rstest]
6669 #[case(false, OrderStatus::PendingUpdate, true, true, true)]
6670 #[case(false, OrderStatus::Accepted, false, true, true)]
6671 #[case(false, OrderStatus::Canceled, false, true, false)]
6672 #[case(true, OrderStatus::PendingCancel, true, true, true)]
6673 #[case(true, OrderStatus::Accepted, false, true, true)]
6674 #[case(true, OrderStatus::Filled, false, true, false)]
6675 fn test_observe_order_status_report_tracking_matrix(
6676 #[case] with_fills: bool,
6677 #[case] status: OrderStatus,
6678 #[case] expect_inflight: bool,
6679 #[case] expect_activity: bool,
6680 #[case] expect_last_query: bool,
6681 ) {
6682 let client_order_id = ClientOrderId::from("O-STATUS-MATRIX");
6683 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6684 let cache = Rc::new(RefCell::new(Cache::default()));
6685 let mut manager = ExecutionManager::new(clock, cache, ExecutionManagerConfig::default())
6686 .expect("valid config");
6687 manager.register_inflight(client_order_id);
6688 manager.order_query_recency.mark(client_order_id);
6689 manager.order_coverage_warnings.insert(client_order_id);
6690 manager.order_coverage_unresolved.insert(client_order_id);
6691 manager.order_query_pending.insert(client_order_id);
6692
6693 let order_report = OrderStatusReport::new(
6694 AccountId::from("TEST-001"),
6695 crypto_perpetual_ethusdt().id(),
6696 Some(client_order_id),
6697 VenueOrderId::from("V-STATUS-MATRIX"),
6698 OrderSide::Buy.into(),
6699 OrderType::Limit,
6700 TimeInForce::Gtc,
6701 status,
6702 Quantity::from("10.0"),
6703 Quantity::from("0.0"),
6704 UnixNanos::from(1_000),
6705 UnixNanos::from(1_000),
6706 UnixNanos::from(1_000),
6707 None,
6708 );
6709
6710 let report = if with_fills {
6711 ExecutionReport::OrderWithFills(Box::new(order_report), Vec::new())
6712 } else {
6713 ExecutionReport::Order(Box::new(order_report))
6714 };
6715
6716 manager.observe_execution_report(&report);
6717
6718 assert_eq!(
6719 manager.order_inflight_checks.contains_key(&client_order_id),
6720 expect_inflight,
6721 );
6722 assert_eq!(
6723 manager.order_recon_retries.contains_key(&client_order_id),
6724 expect_inflight,
6725 );
6726 assert_eq!(
6727 manager.order_activity.contains_key(&client_order_id),
6728 expect_activity,
6729 );
6730 assert_eq!(
6731 manager.order_query_recency.contains_key(&client_order_id),
6732 expect_last_query,
6733 );
6734 assert_eq!(
6735 manager.order_coverage_warnings.contains(&client_order_id),
6736 expect_inflight,
6737 );
6738 assert_eq!(
6739 manager.order_coverage_unresolved.contains(&client_order_id),
6740 expect_inflight,
6741 );
6742 assert_eq!(
6743 manager.order_query_pending.contains(&client_order_id),
6744 expect_inflight,
6745 );
6746 }
6747
6748 #[rstest]
6749 #[case(OrderStatus::PendingUpdate)]
6750 #[case(OrderStatus::PendingCancel)]
6751 fn test_accepted_report_during_pending_command_preserves_inflight_tracking(
6752 #[case] pending_status: OrderStatus,
6753 ) {
6754 let client_order_id = ClientOrderId::from("O-PENDING-COMMAND");
6755 let venue_order_id = VenueOrderId::from("V-PENDING-COMMAND");
6756 let account_id = AccountId::from("TEST-001");
6757 let client_id = ClientId::from("TEST");
6758 let instrument_id = crypto_perpetual_ethusdt().id();
6759 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6760 let cache = Rc::new(RefCell::new(Cache::default()));
6761 insert_accepted_limit_order(
6762 &cache,
6763 client_order_id,
6764 venue_order_id,
6765 instrument_id,
6766 client_id,
6767 );
6768
6769 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6770
6771 let event = match pending_status {
6772 OrderStatus::PendingUpdate => OrderEventAny::PendingUpdate(
6773 OrderPendingUpdateSpec::builder()
6774 .trader_id(order.trader_id())
6775 .strategy_id(order.strategy_id())
6776 .instrument_id(instrument_id)
6777 .client_order_id(client_order_id)
6778 .account_id(account_id)
6779 .venue_order_id(venue_order_id)
6780 .build(),
6781 ),
6782 OrderStatus::PendingCancel => OrderEventAny::PendingCancel(
6783 OrderPendingCancelSpec::builder()
6784 .trader_id(order.trader_id())
6785 .strategy_id(order.strategy_id())
6786 .instrument_id(instrument_id)
6787 .client_order_id(client_order_id)
6788 .account_id(account_id)
6789 .venue_order_id(venue_order_id)
6790 .build(),
6791 ),
6792 _ => unreachable!(),
6793 };
6794
6795 cache.borrow_mut().update_order(&event).unwrap();
6796
6797 let mut manager =
6798 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
6799 .expect("valid config");
6800 manager.register_inflight(client_order_id);
6801 manager.order_query_recency.mark(client_order_id);
6802 manager.order_coverage_warnings.insert(client_order_id);
6803 manager.order_coverage_unresolved.insert(client_order_id);
6804 manager.order_query_pending.insert(client_order_id);
6805 let report = OrderStatusReport::new(
6806 account_id,
6807 instrument_id,
6808 Some(client_order_id),
6809 venue_order_id,
6810 OrderSide::Buy.into(),
6811 OrderType::Limit,
6812 TimeInForce::Gtc,
6813 OrderStatus::Accepted,
6814 Quantity::from("10.0"),
6815 Quantity::from("0.0"),
6816 UnixNanos::from(1_000),
6817 UnixNanos::from(1_000),
6818 UnixNanos::from(1_000),
6819 None,
6820 )
6821 .with_price(Price::from("100.0"));
6822
6823 manager.observe_execution_report(&ExecutionReport::Order(Box::new(report.clone())));
6824 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6825 let events =
6826 generate_reconciliation_order_events(&order, &report, None, UnixNanos::from(1_000));
6827
6828 assert!(events.is_empty());
6829 assert_eq!(order.status(), pending_status);
6830 assert!(manager.order_inflight_checks.contains_key(&client_order_id));
6831 assert!(manager.order_recon_retries.contains_key(&client_order_id));
6832 assert!(manager.order_query_recency.contains_key(&client_order_id));
6833 assert!(manager.order_activity.contains_key(&client_order_id));
6834 assert!(manager.order_coverage_warnings.contains(&client_order_id));
6835 assert!(manager.order_coverage_unresolved.contains(&client_order_id));
6836 assert!(manager.order_query_pending.contains(&client_order_id));
6837 }
6838
6839 #[rstest]
6840 fn test_superseded_cancel_report_preserves_missing_order_grace() {
6841 let client_order_id = ClientOrderId::from("O-CANCEL-REPLACE");
6842 let old_venue_order_id = VenueOrderId::from("V-CANCEL-REPLACE-OLD");
6843 let new_venue_order_id = VenueOrderId::from("V-CANCEL-REPLACE-NEW");
6844 let account_id = AccountId::from("TEST-001");
6845 let client_id = ClientId::from("TEST");
6846 let instrument_id = crypto_perpetual_ethusdt().id();
6847 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6848 let cache = Rc::new(RefCell::new(Cache::default()));
6849 insert_accepted_limit_order(
6850 &cache,
6851 client_order_id,
6852 old_venue_order_id,
6853 instrument_id,
6854 client_id,
6855 );
6856
6857 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6858 let pending_update = OrderPendingUpdateSpec::builder()
6859 .trader_id(order.trader_id())
6860 .strategy_id(order.strategy_id())
6861 .instrument_id(order.instrument_id())
6862 .client_order_id(client_order_id)
6863 .account_id(account_id)
6864 .venue_order_id(old_venue_order_id)
6865 .build();
6866 cache
6867 .borrow_mut()
6868 .update_order(&OrderEventAny::PendingUpdate(pending_update))
6869 .unwrap();
6870 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6871 let updated = OrderUpdatedSpec::builder()
6872 .trader_id(order.trader_id())
6873 .strategy_id(order.strategy_id())
6874 .instrument_id(order.instrument_id())
6875 .client_order_id(client_order_id)
6876 .quantity(order.quantity())
6877 .venue_order_id(new_venue_order_id)
6878 .account_id(account_id)
6879 .build();
6880 cache
6881 .borrow_mut()
6882 .update_order(&OrderEventAny::Updated(updated))
6883 .unwrap();
6884
6885 let mut manager = ExecutionManager::new(
6886 clock,
6887 cache.clone(),
6888 ExecutionManagerConfig {
6889 open_check_missing_retries: 1,
6890 ..Default::default()
6891 },
6892 )
6893 .expect("valid config");
6894
6895 manager.record_local_activity(client_order_id);
6896 assert!(
6897 manager
6898 .prepare_missing_order_query(client_order_id)
6899 .is_none()
6900 );
6901
6902 let report = OrderStatusReport::new(
6903 account_id,
6904 instrument_id,
6905 Some(client_order_id),
6906 old_venue_order_id,
6907 OrderSide::Buy.into(),
6908 OrderType::Limit,
6909 TimeInForce::Gtc,
6910 OrderStatus::Canceled,
6911 Quantity::from("10.0"),
6912 Quantity::from("0.0"),
6913 UnixNanos::from(1_000),
6914 UnixNanos::from(2_000),
6915 UnixNanos::from(3_000),
6916 None,
6917 );
6918
6919 manager.observe_execution_report(&ExecutionReport::Order(Box::new(report.clone())));
6920 let order = cache.borrow().order_owned(&client_order_id).unwrap();
6921 let events =
6922 generate_reconciliation_order_events(&order, &report, None, UnixNanos::from(1_000));
6923
6924 assert!(events.is_empty());
6925 assert_eq!(order.status(), OrderStatus::Accepted);
6926 assert_eq!(order.venue_order_id(), Some(new_venue_order_id));
6927 assert!(manager.order_activity.contains_key(&client_order_id));
6928 assert!(
6929 manager
6930 .prepare_missing_order_query(client_order_id)
6931 .is_none()
6932 );
6933 assert_eq!(manager.recon_check_retry_count(&client_order_id), 0);
6934 }
6935
6936 #[rstest]
6937 #[cfg_attr(
6938 not(all(feature = "simulation", madsim)),
6939 tokio::test(start_paused = true)
6940 )]
6941 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6942 async fn test_prune_order_local_activity_uses_open_check_threshold() {
6943 let old_id = ClientOrderId::from("O-ACTIVITY-OLD");
6944 let fresh_id = ClientOrderId::from("O-ACTIVITY-FRESH");
6945 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6946 let cache = Rc::new(RefCell::new(Cache::default()));
6947
6948 let mut manager = ExecutionManager::new(
6949 clock,
6950 cache,
6951 ExecutionManagerConfig {
6952 open_check_threshold_ns: DurationNanos::from_millis(100),
6953 ..Default::default()
6954 },
6955 )
6956 .expect("valid config");
6957
6958 manager.record_local_activity(old_id);
6959 dst::time::sleep(Duration::from_millis(101)).await;
6960 manager.record_local_activity(fresh_id);
6961
6962 manager.prune_order_local_activity();
6963
6964 assert!(!manager.order_activity.contains_key(&old_id));
6965 assert!(manager.order_activity.contains_key(&fresh_id));
6966 }
6967
6968 #[rstest]
6969 #[cfg_attr(
6970 not(all(feature = "simulation", madsim)),
6971 tokio::test(start_paused = true)
6972 )]
6973 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6974 async fn test_unapplied_fill_report_expires_after_position_check_threshold() {
6975 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6976 let cache = Rc::new(RefCell::new(Cache::default()));
6977
6978 let mut manager = ExecutionManager::new(
6979 clock,
6980 cache,
6981 ExecutionManagerConfig {
6982 position_check_threshold_ns: DurationNanos::from_millis(100),
6983 ..Default::default()
6984 },
6985 )
6986 .expect("valid config");
6987
6988 let report = unapplied_fill_report("T-UNAPPLIED");
6989 let mut expired = vec![manager.is_unapplied_fill_report_expired(&report)];
6990
6991 manager.record_unapplied_fill_report(&report);
6992 dst::time::sleep(Duration::from_millis(60)).await;
6993 manager.record_unapplied_fill_report(&report);
6994 expired.push(manager.is_unapplied_fill_report_expired(&report));
6995 dst::time::sleep(Duration::from_millis(40)).await;
6996 expired.push(manager.is_unapplied_fill_report_expired(&report));
6997
6998 assert_eq!(expired, vec![false, false, true]);
6999 }
7000
7001 #[rstest]
7002 #[cfg_attr(
7003 not(all(feature = "simulation", madsim)),
7004 tokio::test(start_paused = true)
7005 )]
7006 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7007 async fn test_record_unapplied_fill_report_prunes_outside_position_check_lookback() {
7008 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7009 let cache = Rc::new(RefCell::new(Cache::default()));
7010
7011 let mut manager = ExecutionManager::new(
7012 clock,
7013 cache,
7014 ExecutionManagerConfig {
7015 position_check_lookback_mins: 1,
7016 position_check_threshold_ns: DurationNanos::from_secs(5),
7017 ..Default::default()
7018 },
7019 )
7020 .expect("valid config");
7021
7022 let old_report = unapplied_fill_report("T-UNAPPLIED-OLD");
7023 let recent_report = unapplied_fill_report("T-UNAPPLIED-RECENT");
7024 let fresh_report = unapplied_fill_report("T-UNAPPLIED-FRESH");
7025
7026 manager.record_unapplied_fill_report(&old_report);
7027 dst::time::sleep(Duration::from_secs(30)).await;
7028 manager.record_unapplied_fill_report(&recent_report);
7029 dst::time::sleep(Duration::from_secs(31)).await;
7030 manager.record_unapplied_fill_report(&fresh_report);
7031
7032 assert!(!manager.fills_unapplied.contains_key(&(
7033 old_report.account_id,
7034 old_report.instrument_id,
7035 old_report.trade_id,
7036 )));
7037 assert!(manager.fills_unapplied.contains_key(&(
7038 recent_report.account_id,
7039 recent_report.instrument_id,
7040 recent_report.trade_id,
7041 )));
7042 assert!(manager.fills_unapplied.contains_key(&(
7043 fresh_report.account_id,
7044 fresh_report.instrument_id,
7045 fresh_report.trade_id,
7046 )));
7047 }
7048
7049 fn unapplied_fill_report(trade_id: &str) -> FillReport {
7050 FillReport::new(
7051 AccountId::from("SIM-001"),
7052 InstrumentId::from("ETHUSDT-PERP.BINANCE"),
7053 VenueOrderId::from("V-UNAPPLIED"),
7054 TradeId::from(trade_id),
7055 OrderSide::Buy,
7056 Quantity::from("1.0"),
7057 Price::from("100.0"),
7058 Money::from("0.10 USDT"),
7059 LiquiditySide::Taker,
7060 None,
7061 None,
7062 UnixNanos::from(1),
7063 UnixNanos::from(1),
7064 None,
7065 )
7066 }
7067
7068 #[rstest]
7069 fn test_prepare_open_order_report_check_builds_bulk_command_with_config() {
7070 let lookback_mins = 5_u64;
7071 let lookback = DurationNanos::from_mins(lookback_mins);
7072 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7073 let cache = Rc::new(RefCell::new(Cache::default()));
7074
7075 let mut manager = ExecutionManager::new(
7076 clock.clone(),
7077 cache.clone(),
7078 ExecutionManagerConfig {
7079 open_check_lookback_mins: Some(lookback_mins),
7080 open_check_open_only: false,
7081 reconciliation_instrument_ids: IndexSet::from([crypto_perpetual_ethusdt().id()]),
7082 ..Default::default()
7083 },
7084 )
7085 .expect("valid config");
7086
7087 let included_id = ClientOrderId::from("O-REPORT-001");
7088 let excluded_id = ClientOrderId::from("O-REPORT-002");
7089 let included_instrument_id = crypto_perpetual_ethusdt().id();
7090 let excluded_instrument_id = btcusd_bybit().id();
7091
7092 cache
7093 .borrow_mut()
7094 .add_instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
7095 .unwrap();
7096 cache
7097 .borrow_mut()
7098 .add_instrument(InstrumentAny::CryptoPerpetual(btcusd_bybit()))
7099 .unwrap();
7100 insert_accepted_limit_order(
7101 &cache,
7102 included_id,
7103 VenueOrderId::from("V-REPORT-001"),
7104 included_instrument_id,
7105 ClientId::from("BINANCE"),
7106 );
7107 insert_accepted_limit_order(
7108 &cache,
7109 excluded_id,
7110 VenueOrderId::from("V-REPORT-002"),
7111 excluded_instrument_id,
7112 ClientId::from("BYBIT"),
7113 );
7114 clock
7115 .borrow_mut()
7116 .advance_time(UnixNanos::default().saturating_add(lookback * 2), true);
7117
7118 let ts_now = clock.borrow().timestamp_ns();
7119 let command_id = UUID4::new();
7120 let check = manager.prepare_open_order_report_check(command_id, &[]);
7121
7122 assert_eq!(check.command.command_id, command_id);
7123 assert_eq!(check.command.ts_init, ts_now);
7124 assert!(!check.command.open_only);
7125 assert_eq!(check.command.instrument_id, None);
7126 assert_eq!(check.command.start, Some(ts_now.saturating_sub(lookback)));
7127 assert_eq!(check.command.end, None);
7128 assert_eq!(check.command.log_receipt_level, LogLevel::Debug);
7129 assert_eq!(check.filtered_orders.len(), 1);
7130 assert_eq!(check.filtered_orders[0].client_order_id(), included_id);
7131 }
7132
7133 #[rstest]
7134 fn test_prepare_position_report_check_builds_bulk_command_with_coverage() {
7135 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7136 let cache = Rc::new(RefCell::new(Cache::default()));
7137
7138 let manager = ExecutionManager::new(
7139 clock.clone(),
7140 cache.clone(),
7141 ExecutionManagerConfig {
7142 reconciliation_instrument_ids: IndexSet::from([crypto_perpetual_ethusdt().id()]),
7143 ..Default::default()
7144 },
7145 )
7146 .expect("valid config");
7147
7148 let included_instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7149 let excluded_instrument = InstrumentAny::CryptoPerpetual(btcusd_bybit());
7150
7151 cache
7152 .borrow_mut()
7153 .add_instrument(included_instrument.clone())
7154 .unwrap();
7155 cache
7156 .borrow_mut()
7157 .add_instrument(excluded_instrument.clone())
7158 .unwrap();
7159 let included_position = insert_open_position(
7160 &cache,
7161 &included_instrument,
7162 PositionId::from("P-REPORT-001"),
7163 OrderSide::Buy,
7164 "5.0",
7165 "3000.00",
7166 );
7167 insert_open_position(
7168 &cache,
7169 &excluded_instrument,
7170 PositionId::from("P-REPORT-002"),
7171 OrderSide::Buy,
7172 "2.0",
7173 "40000.00",
7174 );
7175
7176 let ts_now = clock.borrow().timestamp_ns();
7177 let command_id = UUID4::new();
7178 let check = manager.prepare_position_report_check(command_id, &[]);
7179 let key = (
7180 included_position.instrument_id,
7181 included_position.account_id,
7182 );
7183
7184 assert_eq!(check.command.command_id, command_id);
7185 assert_eq!(check.command.ts_init, ts_now);
7186 assert_eq!(check.command.instrument_id, None);
7187 assert_eq!(check.command.start, None);
7188 assert_eq!(check.command.end, None);
7189 assert_eq!(check.command.log_receipt_level, LogLevel::Debug);
7190 assert_eq!(check.client_coverage.len(), 1);
7191 assert_eq!(
7192 check.client_coverage.get(&key),
7193 Some(&ReportClientCoverage::Unresolved)
7194 );
7195 assert_eq!(check.activity_revisions.get(&key), Some(&0));
7196 }
7197
7198 #[rstest]
7199 fn test_position_reconciliation_preserves_unavailable_spot_coverage() {
7200 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7201 let cache = Rc::new(RefCell::new(Cache::default()));
7202 let mut manager =
7203 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
7204 .expect("valid config");
7205 let derivative = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7206 let spot = test_bybit_spot_instrument();
7207 cache
7208 .borrow_mut()
7209 .add_instrument(derivative.clone())
7210 .unwrap();
7211 cache.borrow_mut().add_instrument(spot.clone()).unwrap();
7212 let derivative_position = insert_open_position(
7213 &cache,
7214 &derivative,
7215 PositionId::from("P-DERIVATIVE-RECONCILE"),
7216 OrderSide::Buy,
7217 "5.0",
7218 "3000.00",
7219 );
7220 let spot_position = insert_open_position(
7221 &cache,
7222 &spot,
7223 PositionId::from("P-SPOT-PRESERVED"),
7224 OrderSide::Buy,
7225 "2.0",
7226 "2000.00",
7227 );
7228 let client = PositionCoverageStubClient;
7229 let check = manager.prepare_position_report_check(UUID4::new(), &[&client]);
7230 let queried_clients = IndexSet::from([client.client_id()]);
7231
7232 let events = manager.reconcile_position_reports(
7233 &check,
7234 Vec::new(),
7235 &queried_clients,
7236 &IndexSet::new(),
7237 );
7238
7239 assert!(events.iter().any(|event| {
7240 matches!(event, OrderEventAny::Filled(fill) if fill.instrument_id == derivative_position.instrument_id)
7241 }));
7242 assert!(!events.iter().any(|event| {
7243 matches!(event, OrderEventAny::Filled(fill) if fill.instrument_id == spot_position.instrument_id)
7244 }));
7245 }
7246
7247 #[rstest]
7248 fn test_position_reconciliation_preserves_spot_position_when_client_query_fails() {
7249 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7250 let cache = Rc::new(RefCell::new(Cache::default()));
7251 let mut manager =
7252 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
7253 .expect("valid config");
7254 let instrument = test_bybit_spot_instrument();
7255 cache
7256 .borrow_mut()
7257 .add_instrument(instrument.clone())
7258 .unwrap();
7259 let position = insert_open_position(
7260 &cache,
7261 &instrument,
7262 PositionId::from("P-SPOT-QUERY-FAILED"),
7263 OrderSide::Buy,
7264 "5.0",
7265 "3000.00",
7266 );
7267 let key = (position.instrument_id, position.account_id);
7268 let client_id = ClientId::from("BYBIT");
7269 let mut check = manager.prepare_position_report_check(UUID4::new(), &[]);
7270 check.client_coverage.insert(
7271 key,
7272 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
7273 );
7274 let queried_clients = IndexSet::from([client_id]);
7275 let failed_clients = IndexSet::from([client_id]);
7276
7277 let events = manager.reconcile_position_reports(
7278 &check,
7279 Vec::new(),
7280 &queried_clients,
7281 &failed_clients,
7282 );
7283
7284 assert!(
7285 !events
7286 .iter()
7287 .any(|event| matches!(event, OrderEventAny::Filled(_))),
7288 "a failed bulk query must not generate a synthetic closing fill",
7289 );
7290 let cached_position = cache.borrow().position(&position.id).unwrap().clone();
7291 assert!(cached_position.is_open());
7292 assert_eq!(cached_position.quantity, Quantity::from("5.0"));
7293 }
7294
7295 #[rstest]
7296 #[cfg_attr(
7297 not(all(feature = "simulation", madsim)),
7298 tokio::test(start_paused = true)
7299 )]
7300 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7301 async fn test_position_report_check_defers_activity_recorded_during_delayed_request() {
7302 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7303 let cache = Rc::new(RefCell::new(Cache::default()));
7304
7305 let mut manager = ExecutionManager::new(
7306 clock,
7307 cache.clone(),
7308 ExecutionManagerConfig {
7309 position_check_threshold_ns: DurationNanos::from_secs(5),
7310 ..Default::default()
7311 },
7312 )
7313 .expect("valid config");
7314
7315 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7316 let instrument_id = instrument.id();
7317 let position = insert_open_position(
7318 &cache,
7319 &instrument,
7320 PositionId::from("P-ACTIVITY-DURING-REQUEST"),
7321 OrderSide::Buy,
7322 "5.0",
7323 "3000.00",
7324 );
7325 cache
7326 .borrow_mut()
7327 .add_instrument(instrument.clone())
7328 .unwrap();
7329 let account_id = position.account_id;
7330 let check = manager.prepare_position_report_check(UUID4::new(), &[]);
7331
7332 let report = PositionStatusReport::new(
7333 account_id,
7334 instrument_id,
7335 PositionSide::Long,
7336 Quantity::from("5.0"),
7337 UnixNanos::from(1_000_000),
7338 UnixNanos::from(1_000_000),
7339 None,
7340 None,
7341 Some(Decimal::from(3000)),
7342 );
7343
7344 let closed_position = close_long_position(
7345 position,
7346 &instrument,
7347 TradeId::from("T-ACTIVITY-DURING-REQUEST"),
7348 );
7349 cache
7350 .borrow_mut()
7351 .update_position(&closed_position)
7352 .unwrap();
7353 manager.record_position_activity(instrument_id, account_id);
7354
7355 dst::time::sleep(Duration::from_secs(6)).await;
7357
7358 let events = manager.reconcile_position_reports(
7359 &check,
7360 vec![report],
7361 &IndexSet::new(),
7362 &IndexSet::new(),
7363 );
7364
7365 assert!(
7366 !events.iter().any(|event| {
7367 matches!(
7368 event,
7369 OrderEventAny::Filled(fill)
7370 if fill.order_side == OrderSide::Buy
7371 && fill.last_qty == Quantity::from("5.0")
7372 )
7373 }),
7374 "activity recorded after the request started must defer A's stale report",
7375 );
7376 }
7377
7378 #[rstest]
7379 fn test_position_report_check_does_not_defer_activity_recorded_before_request() {
7380 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7381 let cache = Rc::new(RefCell::new(Cache::default()));
7382
7383 let mut manager = ExecutionManager::new(
7384 clock,
7385 cache.clone(),
7386 ExecutionManagerConfig {
7387 position_check_threshold_ns: DurationNanos::ZERO,
7388 ..Default::default()
7389 },
7390 )
7391 .expect("valid config");
7392
7393 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7394 let instrument_id = instrument.id();
7395 let position = insert_open_position(
7396 &cache,
7397 &instrument,
7398 PositionId::from("P-ACTIVITY-BEFORE-REQUEST"),
7399 OrderSide::Buy,
7400 "5.0",
7401 "3000.00",
7402 );
7403 cache.borrow_mut().add_instrument(instrument).unwrap();
7404 let account_id = position.account_id;
7405 manager.record_position_activity(instrument_id, account_id);
7406 let check = manager.prepare_position_report_check(UUID4::new(), &[]);
7407
7408 let report = PositionStatusReport::new(
7409 account_id,
7410 instrument_id,
7411 PositionSide::Long,
7412 Quantity::from("10.0"),
7413 UnixNanos::from(1_000_000),
7414 UnixNanos::from(1_000_000),
7415 None,
7416 None,
7417 Some(Decimal::from(3000)),
7418 );
7419
7420 let events = manager.reconcile_position_reports(
7421 &check,
7422 vec![report],
7423 &IndexSet::new(),
7424 &IndexSet::new(),
7425 );
7426
7427 let fills: Vec<_> = events
7428 .iter()
7429 .filter_map(|event| match event {
7430 OrderEventAny::Filled(fill) => Some(fill),
7431 _ => None,
7432 })
7433 .collect();
7434
7435 assert_eq!(fills.len(), 1);
7436 assert_eq!(fills[0].order_side, OrderSide::Buy);
7437 assert_eq!(fills[0].last_qty, Quantity::from("5.0"));
7438 assert_eq!(fills[0].commission, None);
7439 }
7440
7441 #[rstest]
7442 fn test_mass_status_projects_companion_fill_before_void_correction() {
7443 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7444 let cache = Rc::new(RefCell::new(Cache::default()));
7445 let mut manager =
7446 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
7447 .expect("valid config");
7448 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7449 let client_order_id = ClientOrderId::from("O-MASS-VOID-001");
7450 let venue_order_id = VenueOrderId::from("V-MASS-VOID-001");
7451 let account_id = AccountId::from("TEST-001");
7452 cache
7453 .borrow_mut()
7454 .add_instrument(instrument.clone())
7455 .unwrap();
7456 insert_accepted_limit_order(
7457 &cache,
7458 client_order_id,
7459 venue_order_id,
7460 instrument.id(),
7461 ClientId::from("BINANCE"),
7462 );
7463 let order = cache.borrow().order_owned(&client_order_id).unwrap();
7464 let initial_fill = TestOrderEventStubs::filled(
7465 &order,
7466 &instrument,
7467 Some(TradeId::from("T-MASS-VOID-INITIAL")),
7468 None,
7469 Some(Price::from("100.0")),
7470 Some(Quantity::from("6.0")),
7471 Some(LiquiditySide::Taker),
7472 None,
7473 None,
7474 Some(account_id),
7475 );
7476 cache.borrow_mut().update_order(&initial_fill).unwrap();
7477 let order = cache.borrow().order_owned(&client_order_id).unwrap();
7478
7479 let report = OrderStatusReport::new(
7480 account_id,
7481 instrument.id(),
7482 Some(client_order_id),
7483 venue_order_id,
7484 OrderSide::Buy.into(),
7485 OrderType::Limit,
7486 TimeInForce::Gtc,
7487 OrderStatus::Canceled,
7488 Quantity::from("10.0"),
7489 Quantity::from("5.0"),
7490 UnixNanos::from(1_000),
7491 UnixNanos::from(1_000),
7492 UnixNanos::from(1_000),
7493 None,
7494 );
7495
7496 let companion_fill = FillReport::new(
7497 account_id,
7498 instrument.id(),
7499 venue_order_id,
7500 TradeId::from("T-MASS-VOID-COMPANION"),
7501 OrderSide::Buy,
7502 Quantity::from("1.0"),
7503 Price::from("100.0"),
7504 Money::zero(instrument.quote_currency()),
7505 LiquiditySide::Taker,
7506 Some(client_order_id),
7507 None,
7508 UnixNanos::from(900),
7509 UnixNanos::from(1_000),
7510 None,
7511 );
7512
7513 let mut fill_queue = ReconciliationFillQueue::default();
7514 let events = manager.reconcile_order_with_fills(
7515 true,
7516 &order,
7517 &report,
7518 &[&companion_fill],
7519 Some(&instrument),
7520 &mut fill_queue,
7521 None,
7522 );
7523 let mut projected = order;
7524
7525 for event in &events {
7526 projected.apply(event.clone()).unwrap();
7527 }
7528
7529 assert!(matches!(events[0], OrderEventAny::Filled(_)));
7530 assert_eq!(
7531 events
7532 .iter()
7533 .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
7534 .count(),
7535 2
7536 );
7537 assert_eq!(projected.status(), OrderStatus::Canceled);
7538 assert_eq!(projected.filled_qty(), Quantity::from("5.0"));
7539 assert_eq!(projected.voided_qty(), Quantity::from("2.0"));
7540 }
7541
7542 fn insert_accepted_limit_order(
7543 cache: &Rc<RefCell<Cache>>,
7544 client_order_id: ClientOrderId,
7545 venue_order_id: VenueOrderId,
7546 instrument_id: InstrumentId,
7547 client_id: ClientId,
7548 ) {
7549 let account_id = AccountId::from("TEST-001");
7550 let order = OrderTestBuilder::new(OrderType::Limit)
7551 .client_order_id(client_order_id)
7552 .instrument_id(instrument_id)
7553 .quantity(Quantity::from("10.0"))
7554 .price(Price::from("100.0"))
7555 .build();
7556 let submitted = TestOrderEventStubs::submitted(&order, account_id);
7557 cache
7558 .borrow_mut()
7559 .add_order(order, None, Some(client_id), false)
7560 .unwrap();
7561 let order = cache.borrow_mut().update_order(&submitted).unwrap();
7562 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
7563 cache.borrow_mut().update_order(&accepted).unwrap();
7564 }
7565
7566 fn test_bybit_spot_instrument() -> InstrumentAny {
7567 InstrumentAny::CurrencyPair(
7568 CurrencyPair::builder()
7569 .instrument_id(InstrumentId::from("ETHUSDT-SPOT.BYBIT"))
7570 .raw_symbol(Symbol::from("ETHUSDT"))
7571 .base_currency(Currency::from("ETH"))
7572 .quote_currency(Currency::from("USDT"))
7573 .price_precision(2)
7574 .size_precision(5)
7575 .price_increment(Price::from("0.01"))
7576 .size_increment(Quantity::from("0.00001"))
7577 .ts_event(UnixNanos::default())
7578 .ts_init(UnixNanos::default())
7579 .build()
7580 .unwrap(),
7581 )
7582 }
7583
7584 fn insert_open_position(
7585 cache: &Rc<RefCell<Cache>>,
7586 instrument: &InstrumentAny,
7587 position_id: PositionId,
7588 side: OrderSide,
7589 quantity: &str,
7590 price: &str,
7591 ) -> Position {
7592 let order = OrderTestBuilder::new(OrderType::Market)
7593 .instrument_id(instrument.id())
7594 .side(side)
7595 .quantity(Quantity::from(quantity))
7596 .build();
7597 let fill = TestOrderEventStubs::filled(
7598 &order,
7599 instrument,
7600 Some(TradeId::new("T-REPORT-001")),
7601 Some(position_id),
7602 Some(Price::from(price)),
7603 Some(Quantity::from(quantity)),
7604 None,
7605 None,
7606 None,
7607 Some(AccountId::from("TEST-001")),
7608 );
7609 let order_filled: OrderFilled = fill.into();
7610 let position = Position::new(instrument, order_filled);
7611 cache
7612 .borrow_mut()
7613 .add_position(&position, OmsType::Hedging)
7614 .unwrap();
7615 position
7616 }
7617
7618 fn close_long_position(
7619 mut position: Position,
7620 instrument: &InstrumentAny,
7621 trade_id: TradeId,
7622 ) -> Position {
7623 let order = OrderTestBuilder::new(OrderType::Market)
7624 .instrument_id(instrument.id())
7625 .side(OrderSide::Sell)
7626 .quantity(position.quantity)
7627 .build();
7628 let fill = TestOrderEventStubs::filled(
7629 &order,
7630 instrument,
7631 Some(trade_id),
7632 Some(position.id),
7633 Some(Price::from("3000.00")),
7634 Some(position.quantity),
7635 None,
7636 None,
7637 None,
7638 Some(position.account_id),
7639 );
7640 let order_filled: OrderFilled = fill.into();
7641 position.apply(&order_filled);
7642 position
7643 }
7644
7645 #[cfg(feature = "node")]
7646 mod node {
7647 use super::*;
7648 use crate::execution::client::LiveExecutionClient;
7649
7650 #[rstest]
7651 fn test_plan_position_fill_reports_uses_configured_lookback() {
7652 let lookback_mins = 7_u64;
7653 let lookback = DurationNanos::from_mins(lookback_mins);
7654 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7655 let cache = Rc::new(RefCell::new(Cache::default()));
7656
7657 let mut manager = ExecutionManager::new(
7658 clock.clone(),
7659 cache.clone(),
7660 ExecutionManagerConfig {
7661 position_check_lookback_mins: lookback_mins,
7662 position_check_threshold_ns: DurationNanos::ZERO,
7663 ..Default::default()
7664 },
7665 )
7666 .expect("valid config");
7667
7668 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7669 cache
7670 .borrow_mut()
7671 .add_instrument(instrument.clone())
7672 .unwrap();
7673 let position = insert_open_position(
7674 &cache,
7675 &instrument,
7676 PositionId::from("P-FILL-LOOKBACK"),
7677 OrderSide::Buy,
7678 "1.0",
7679 "3000.00",
7680 );
7681 clock
7682 .borrow_mut()
7683 .advance_time(UnixNanos::default().saturating_add(lookback * 2), true);
7684 let client = PositionCoverageStubClient;
7685 let clients: [&dyn ExecutionClient; 1] = [&client];
7686 let mut check = manager.prepare_position_report_check(UUID4::new(), &clients);
7687 let query_end = clock.borrow().timestamp_ns();
7688
7689 let report = PositionStatusReport::new(
7690 position.account_id,
7691 position.instrument_id,
7692 PositionSide::Long,
7693 Quantity::from("2.0"),
7694 query_end,
7695 query_end,
7696 None,
7697 None,
7698 Some(dec!(3000.00)),
7699 );
7700 let queried_clients = IndexSet::from([client.client_id()]);
7701
7702 let plan = manager.plan_position_fill_reports(
7703 &mut check,
7704 &[report],
7705 &queried_clients,
7706 &IndexSet::new(),
7707 &clients,
7708 );
7709
7710 assert_eq!(
7711 plan.discrepancy_keys,
7712 IndexSet::from([(position.instrument_id, position.account_id)])
7713 );
7714 assert_eq!(plan.queries.len(), 1);
7715 let query = &plan.queries[0];
7716 assert_eq!(
7717 (query.key, query.client_id),
7718 (
7719 (position.instrument_id, position.account_id),
7720 client.client_id()
7721 )
7722 );
7723 assert_eq!(query.command.instrument_id, Some(position.instrument_id));
7724 assert_eq!(query.command.venue_order_id, None);
7725 assert_eq!(
7726 query.command.start,
7727 Some(query_end.saturating_sub(lookback))
7728 );
7729 assert_eq!(query.command.end, Some(query_end));
7730 assert_eq!(query.command.correlation_id, Some(check.command.command_id));
7731 assert_eq!(query.command.log_receipt_level, LogLevel::Debug);
7732 }
7733
7734 #[rstest]
7735 #[case::reported(Some("2.0"), true)]
7736 #[case::unreported(None, false)]
7737 fn test_plan_position_fill_reports_unavailable_coverage_requires_venue_report(
7738 #[case] venue_qty: Option<&str>,
7739 #[case] expect_query: bool,
7740 ) {
7741 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7742 let cache = Rc::new(RefCell::new(Cache::default()));
7743
7744 let mut manager = ExecutionManager::new(
7745 clock.clone(),
7746 cache.clone(),
7747 ExecutionManagerConfig {
7748 position_check_threshold_ns: DurationNanos::ZERO,
7749 ..Default::default()
7750 },
7751 )
7752 .expect("valid config");
7753
7754 let spot = test_bybit_spot_instrument();
7755 cache.borrow_mut().add_instrument(spot.clone()).unwrap();
7756 let position = insert_open_position(
7757 &cache,
7758 &spot,
7759 PositionId::from("P-SPOT-UNAVAILABLE"),
7760 OrderSide::Buy,
7761 "1.0",
7762 "2000.00",
7763 );
7764 let key = (position.instrument_id, position.account_id);
7765 let client = PositionCoverageStubClient;
7766 let clients: [&dyn ExecutionClient; 1] = [&client];
7767 let mut check = manager.prepare_position_report_check(UUID4::new(), &clients);
7768 let ts_now = clock.borrow().timestamp_ns();
7769
7770 let reports: Vec<PositionStatusReport> = venue_qty
7771 .into_iter()
7772 .map(|qty| {
7773 PositionStatusReport::new(
7774 position.account_id,
7775 position.instrument_id,
7776 PositionSide::Long,
7777 Quantity::from(qty),
7778 ts_now,
7779 ts_now,
7780 None,
7781 None,
7782 Some(dec!(2000.00)),
7783 )
7784 })
7785 .collect();
7786
7787 let queried_clients = IndexSet::from([client.client_id()]);
7788
7789 let plan = manager.plan_position_fill_reports(
7790 &mut check,
7791 &reports,
7792 &queried_clients,
7793 &IndexSet::new(),
7794 &clients,
7795 );
7796
7797 let expected_queries = if expect_query {
7798 vec![(key, client.client_id())]
7799 } else {
7800 Vec::new()
7801 };
7802
7803 assert_eq!(
7804 check.client_coverage.get(&key),
7805 Some(&ReportClientCoverage::Unavailable(IndexSet::from([
7806 client.client_id()
7807 ])))
7808 );
7809 assert_eq!(plan.discrepancy_keys, IndexSet::from([key]));
7810 assert_eq!(
7811 plan.queries
7812 .iter()
7813 .map(|query| (query.key, query.client_id))
7814 .collect::<Vec<_>>(),
7815 expected_queries
7816 );
7817 }
7818
7819 #[rstest]
7820 fn test_plan_position_fill_reports_defers_position_opened_during_request() {
7821 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7822 let cache = Rc::new(RefCell::new(Cache::default()));
7823
7824 let mut manager = ExecutionManager::new(
7825 clock,
7826 cache.clone(),
7827 ExecutionManagerConfig {
7828 position_check_threshold_ns: DurationNanos::ZERO,
7829 ..Default::default()
7830 },
7831 )
7832 .expect("valid config");
7833
7834 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7835 cache
7836 .borrow_mut()
7837 .add_instrument(instrument.clone())
7838 .unwrap();
7839 let client = PositionCoverageStubClient;
7840 let clients: [&dyn ExecutionClient; 1] = [&client];
7841 let mut check = manager.prepare_position_report_check(UUID4::new(), &clients);
7842 let position = insert_open_position(
7843 &cache,
7844 &instrument,
7845 PositionId::from("P-FILL-DURING-REQUEST"),
7846 OrderSide::Buy,
7847 "1.0",
7848 "3000.00",
7849 );
7850 manager.record_position_activity(position.instrument_id, position.account_id);
7851
7852 let report = PositionStatusReport::new(
7853 position.account_id,
7854 position.instrument_id,
7855 PositionSide::Long,
7856 Quantity::from("2.0"),
7857 UnixNanos::from(1_000_000),
7858 UnixNanos::from(1_000_000),
7859 None,
7860 None,
7861 Some(dec!(3000.00)),
7862 );
7863
7864 let plan = manager.plan_position_fill_reports(
7865 &mut check,
7866 &[report],
7867 &IndexSet::from([client.client_id()]),
7868 &IndexSet::new(),
7869 &clients,
7870 );
7871
7872 assert_eq!(
7873 plan.discrepancy_keys,
7874 IndexSet::from([(position.instrument_id, position.account_id)])
7875 );
7876 assert!(plan.queries.is_empty());
7877 assert_eq!(
7878 check
7879 .activity_revisions
7880 .get(&(position.instrument_id, position.account_id)),
7881 Some(&0)
7882 );
7883 }
7884
7885 #[rstest]
7886 fn test_prepare_position_report_check_uses_live_client_bulk_coverage() {
7887 let clock = Rc::new(RefCell::new(VirtualClock::new()));
7888 let cache = Rc::new(RefCell::new(Cache::default()));
7889 let manager =
7890 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default())
7891 .expect("valid config");
7892 let derivative = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7893 let spot = test_bybit_spot_instrument();
7894 cache
7895 .borrow_mut()
7896 .add_instrument(derivative.clone())
7897 .unwrap();
7898 cache.borrow_mut().add_instrument(spot.clone()).unwrap();
7899 let derivative_position = insert_open_position(
7900 &cache,
7901 &derivative,
7902 PositionId::from("P-DERIVATIVE-COVERAGE"),
7903 OrderSide::Buy,
7904 "5.0",
7905 "3000.00",
7906 );
7907 let spot_position = insert_open_position(
7908 &cache,
7909 &spot,
7910 PositionId::from("P-SPOT-COVERAGE"),
7911 OrderSide::Buy,
7912 "2.0",
7913 "2000.00",
7914 );
7915 let client = LiveExecutionClient::new(Box::new(PositionCoverageStubClient));
7916 let client: &dyn ExecutionClient = &client;
7917
7918 let check = manager.prepare_position_report_check(UUID4::new(), &[client]);
7919 let client_id = ClientId::from("BYBIT");
7920
7921 assert_eq!(
7922 check.client_coverage.get(&(
7923 derivative_position.instrument_id,
7924 derivative_position.account_id
7925 )),
7926 Some(&ReportClientCoverage::Resolved(IndexSet::from([client_id])))
7927 );
7928 assert_eq!(
7929 check
7930 .client_coverage
7931 .get(&(spot_position.instrument_id, spot_position.account_id)),
7932 Some(&ReportClientCoverage::Unavailable(IndexSet::from([
7933 client_id
7934 ])))
7935 );
7936 }
7937 }
7938}