1use std::{
29 collections::{BTreeMap, BTreeSet},
30 future::Future,
31 sync::{
32 Arc,
33 atomic::{AtomicBool, AtomicU64, Ordering},
34 },
35 time::Duration,
36};
37
38use ahash::AHashSet;
39use anyhow::Context;
40use async_trait::async_trait;
41#[cfg(test)]
42use nautilus_common::live::get_runtime;
43use nautilus_common::{
44 clients::ExecutionClient,
45 enums::{LogColor, LogLevel},
46 live::runner::get_exec_event_sender,
47 log_debug,
48 messages::execution::{
49 BatchCancelOrders, CancelAllOrders, CancelOrder, GenerateFillReports,
50 GenerateOrderStatusReport, GenerateOrderStatusReports, GeneratePositionStatusReports,
51 ModifyOrder, QueryAccount, QueryOrder, SubmitOrder, SubmitOrderList,
52 },
53};
54use nautilus_core::{
55 DurationNanos, UUID4, UnixNanos,
56 datetime::unix_nanos_to_iso8601,
57 params::Params,
58 string::secret::SecretString,
59 time::{AtomicTime, get_atomic_clock_realtime},
60};
61use nautilus_live::{
62 ExecutionClientCore, ExecutionEventEmitter, SocketControlFactory,
63 book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS,
64 execution::failure::CommandFailure,
65 task::{TaskGroup, TaskGroupGuard, TaskJoinOutcome, TaskSlot, TaskSpawner, finish_task},
66};
67use nautilus_model::{
68 accounts::AccountAny,
69 enums::{AccountType, OmsType, OrderSide, OrderType, PositionSide},
70 events::{OrderAccepted, OrderDeniedReason, OrderEventAny},
71 identifiers::{
72 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TraderId, Venue, VenueOrderId,
73 },
74 instruments::{Instrument, InstrumentAny},
75 orders::{Order, OrderAny},
76 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
77 types::{AccountBalance, MarginBalance, Quantity},
78};
79use nautilus_network::error::SendError;
80use parking_lot::Mutex;
81use rust_decimal::Decimal;
82use serde::Deserialize;
83use tokio_util::sync::CancellationToken;
84use zeroize::Zeroizing;
85
86use crate::{
87 common::{
88 consts::{DISCONNECT_TIMEOUT, LIGHTER_ERROR_CODE_INVALID_NONCE, LIGHTER_MAX_BATCH_TX},
89 credential::{Credential, scrub_auth},
90 deployment,
91 enums::{
92 LighterAccountTier, LighterPositionMarginMode, LighterProductType, LighterTxStatus,
93 LighterTxType,
94 },
95 rate_limit::{LighterTxRateLimiter, await_tx_quota, build_tx_rate_limiter, resolve_quota},
96 symbol::{MarketRegistry, product_type_from_instrument_id},
97 },
98 config::LighterExecutionClientConfig,
99 http::{
100 client::{LIGHTER_REST_PAGE_SIZE, LighterHttpClient, LighterRawHttpClient},
101 error::LighterHttpError,
102 models::{LighterAccountDetail, LighterSendTxRequest},
103 query::{
104 LighterAccountActiveOrdersQuery, LighterAccountInactiveOrdersQuery,
105 LighterSortDirection, LighterTradeSortBy, LighterTradesQuery,
106 },
107 },
108 signing::{
109 auth_token::{build_auth_token_for, fresh_k},
110 nonce::NonceError,
111 tx::{
112 ApproveIntegratorTxInfo, CancelOrderTxInfo, CreateOrderTxInfo, L2TxAttributes,
113 ModifyOrderTxInfo, OrderInfo, TxContext, TxInfoJson, UpdateLeverageTxInfo, sign_tx,
114 },
115 },
116 websocket::{
117 LighterWsError, USER_STREAMS_ENDPOINT,
118 client::{LighterWebSocketClient, RetainedTaskSlot, TaskRetentionGuard},
119 dispatch::{
120 LIGHTER_INSTRUMENT_CACHE, MAX_RECONCILIATION_PAGES, OrderIdentity, PendingOrderAction,
121 PendingSendTx, PendingSendTxKind, TradeDedupSource, WsDispatchState,
122 cache_instruments_for_reports, derive_market_order_price_ticks,
123 evict_terminal_mappings, lookup_create_order_status_report, lookup_order_status_report,
124 nautilus_to_lighter_order_type, nautilus_to_lighter_tif, order_expiry_for,
125 parse_http_order_to_report, price_to_ticks, quantity_to_ticks,
126 },
127 messages::{
128 AccountStream, CANCEL_BATCH_ID_PREFIX, ExecutionReport, LighterWsChannel,
129 NautilusWsMessage, SendTxRejectionSource,
130 },
131 parse::{
132 LighterCommissionError, OpenFrameContext, ParsedOrderEvent, lighter_order_shape,
133 parse_lighter_order_event, parse_lighter_order_filled, parse_lighter_trade_id,
134 parse_ws_fill_report, parse_ws_order_status_report,
135 },
136 },
137};
138
139const DEFAULT_TX_EXPIRY_MS: i64 = 5 * 60 * 1_000;
142
143const ACKED_ORDER_LOOKUP_DELAY: Duration = Duration::from_secs(2);
145const ACKED_ORDER_PROBE_ATTEMPTS: usize = 3;
146const BATCH_ACK_TIMEOUT: Duration = Duration::from_secs(10);
147
148const STRATEGY_REASON_MAX_CHARS: usize = 512;
149
150const AUTH_TOKEN_REFRESH_INTERVAL: std::time::Duration =
155 std::time::Duration::from_secs(6 * 60 * 60);
156
157const _: () = assert!(
159 AUTH_TOKEN_REFRESH_INTERVAL.as_secs()
160 < crate::signing::auth_token::DEFAULT_AUTH_TOKEN_TTL_SECS as u64,
161 "AUTH_TOKEN_REFRESH_INTERVAL must stay below DEFAULT_AUTH_TOKEN_TTL_SECS",
162);
163
164const AUTH_TOKEN_REFRESH_RETRY_WINDOW: Duration = Duration::from_secs(60 * 60);
167const AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(30);
168
169const AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(5 * 60);
171const AUTH_TOKEN_REFRESH_BACKOFF: AuthTokenRefreshBackoff = AuthTokenRefreshBackoff {
172 initial_delay: AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY,
173 max_delay: AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY,
174 window: AUTH_TOKEN_REFRESH_RETRY_WINDOW,
175};
176const NONCE_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1);
177const NONCE_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(30);
178const ACCOUNT_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
181const REFERRAL_ATTRIBUTION_TIMEOUT: Duration = Duration::from_secs(10);
182
183const SENDTX_BARE_ERROR_WINDOW_MS: u64 = 1_000;
188const INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS: i64 = 5 * 365 * 24 * 60 * 60 * 1_000;
189const INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK: u32 = 0;
190const NONCE_CONNECTION_EPOCH_UNAVAILABLE: u64 = u64::MAX;
191
192#[derive(Debug)]
193pub struct LighterExecutionClient {
194 core: ExecutionClientCore,
195 clock: &'static AtomicTime,
196 config: LighterExecutionClientConfig,
197 account_tier: Option<LighterAccountTier>,
198 emitter: ExecutionEventEmitter,
199 credential: Option<Credential>,
200 http_client: LighterHttpClient,
201 ws_client: LighterWebSocketClient,
202 tx_rate_limiter: Arc<LighterTxRateLimiter>,
203 tx_send_sequencer: TxSendSequencer,
204 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
205 nonce_ready_connection_epoch: Arc<AtomicU64>,
206 registry: Arc<MarketRegistry>,
207 socket_factory: SocketControlFactory,
208 pending_tasks: TaskGroup,
209 ws_stream_handle: TaskSlot<()>,
210 ws_disconnect_handle: TaskSlot<Result<(), LighterWsError>>,
211 ws_handler_retained: Arc<RetainedTaskSlot>,
212 auth_refresh_handle: TaskSlot<()>,
213 shutdown_errors: Vec<String>,
214 cancellation_token: CancellationToken,
215 dispatch: WsDispatchState,
216 nonce_recovery_inflight: Arc<AtomicBool>,
217 auth_refresh_notify: Arc<tokio::sync::Notify>,
218}
219
220impl LighterExecutionClient {
221 fn log_report_receipt(count: usize, report_type: &str, log_level: LogLevel) {
222 let plural = if count == 1 { "" } else { "s" };
223 let message = format!("Received {count} {report_type}{plural}");
224
225 match log_level {
226 LogLevel::Off => {}
227 LogLevel::Trace => log::trace!("{message}"),
228 LogLevel::Debug => log::debug!("{message}"),
229 LogLevel::Info => log::info!("{message}"),
230 LogLevel::Warning => log::warn!("{message}"),
231 LogLevel::Error => log::error!("{message}"),
232 }
233 }
234
235 pub fn new(
248 core: ExecutionClientCore,
249 config: LighterExecutionClientConfig,
250 ) -> anyhow::Result<Self> {
251 anyhow::ensure!(
252 core.venue == config.resolved_venue(),
253 "Lighter execution core venue {} does not match configured venue {}",
254 core.venue,
255 config.resolved_venue(),
256 );
257
258 anyhow::ensure!(
259 config.account_id.get_issuer() == core.venue,
260 "Lighter account ID issuer {} does not match configured venue {}",
261 config.account_id.get_issuer(),
262 core.venue,
263 );
264
265 let credential = Credential::resolve_for_deployment(
266 config.private_key.clone().map(SecretString::into_inner),
267 config.account_index,
268 config.api_key_index,
269 config.deployment,
270 config.environment,
271 )
272 .context("failed to resolve Lighter credentials")?;
273
274 let registry = Arc::new(MarketRegistry::new_with_venue_and_settlement_currency(
275 core.venue,
276 config.settlement_currency(),
277 ));
278 let socket_factory = SocketControlFactory::new(core.client_id, Some(core.venue));
279
280 let tx_rate_limiter = build_tx_rate_limiter(config.sendtx_quota_per_min);
283
284 let raw_http = LighterRawHttpClient::new_with_quotas(
285 config.environment,
286 Some(config.http_url()),
287 config.http_timeout_secs,
288 config
289 .proxy_url
290 .as_ref()
291 .map(|value| value.expose_secret().to_owned()),
292 resolve_quota(config.rest_quota_per_min),
293 Some(Arc::clone(&tx_rate_limiter)),
294 )
295 .context("failed to construct Lighter raw HTTP client")?;
296
297 let http_client =
298 LighterHttpClient::from_raw_with_registry(raw_http, Arc::clone(®istry));
299
300 let ws_client = Self::create_ws_client(&config, Arc::clone(®istry), &socket_factory);
301
302 let clock = get_atomic_clock_realtime();
303 let emitter = ExecutionEventEmitter::new(
304 clock,
305 core.trader_id,
306 core.account_id,
307 AccountType::Margin,
308 None,
309 );
310 let pending_tasks = TaskGroup::new();
311
312 Ok(Self {
313 core,
314 clock,
315 config,
316 account_tier: None,
317 emitter,
318 credential,
319 http_client,
320 ws_client,
321 tx_rate_limiter,
322 tx_send_sequencer: TxSendSequencer::new(),
323 nonce_submission_gate: Arc::new(tokio::sync::RwLock::new(())),
324 nonce_ready_connection_epoch: Arc::new(AtomicU64::new(0)),
325 registry,
326 socket_factory,
327 cancellation_token: pending_tasks.cancellation_token(),
328 pending_tasks,
329 ws_stream_handle: TaskSlot::new(),
330 ws_disconnect_handle: TaskSlot::new(),
331 ws_handler_retained: Arc::new(RetainedTaskSlot::new()),
332 auth_refresh_handle: TaskSlot::new(),
333 shutdown_errors: Vec::new(),
334 dispatch: WsDispatchState::new(),
335 nonce_recovery_inflight: Arc::new(AtomicBool::new(false)),
336 auth_refresh_notify: Arc::new(tokio::sync::Notify::new()),
337 })
338 }
339
340 #[must_use]
342 pub fn config(&self) -> &LighterExecutionClientConfig {
343 &self.config
344 }
345
346 #[must_use]
348 pub fn has_credentials(&self) -> bool {
349 self.credential.is_some()
350 }
351
352 #[must_use]
355 pub fn pending_tasks_all_finished(&self) -> bool {
356 self.pending_tasks.all_finished()
357 }
358
359 fn create_ws_client(
360 config: &LighterExecutionClientConfig,
361 registry: Arc<MarketRegistry>,
362 socket_factory: &SocketControlFactory,
363 ) -> LighterWebSocketClient {
364 let ws_client = LighterWebSocketClient::new(
365 Some(config.ws_url()),
366 config.environment,
367 registry,
368 config.transport_backend,
369 config.ws_timeout_secs,
370 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
372 config
373 .proxy_url
374 .as_ref()
375 .map(|value| value.expose_secret().to_owned()),
376 );
377
378 ws_client.with_socket_control(socket_factory.control(USER_STREAMS_ENDPOINT))
379 }
380
381 fn take_ws_client(&mut self) -> LighterWebSocketClient {
382 let cached_instruments = self.ws_client.instruments_cache();
383 let ws_cache = cached_instruments
384 .iter()
385 .map(|entry| (*entry.key(), entry.value().clone()))
386 .collect();
387 let replacement = Self::create_ws_client(
388 &self.config,
389 Arc::clone(&self.registry),
390 &self.socket_factory,
391 );
392 replacement.cache_instruments(ws_cache);
393
394 std::mem::replace(&mut self.ws_client, replacement)
395 }
396
397 fn spawn_task<F>(&self, description: &'static str, fut: F)
398 where
399 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
400 {
401 let future = async move {
402 if let Err(e) = fut.await {
403 log::warn!("{description} failed: {e:?}");
404 }
405 };
406
407 if let Err(e) = self.pending_tasks.spawn(future) {
408 log::warn!("Skipping Lighter {description} after shutdown began: {e}");
409 }
410 }
411
412 fn begin_session_shutdown(&mut self) {
413 self.pending_tasks.begin_shutdown();
414 self.cancellation_token.cancel();
415 self.ws_client.begin_shutdown();
416 self.core.set_disconnected();
417
418 if self.ws_disconnect_handle.is_none()
419 && (self.ws_client.is_active() || self.ws_stream_handle.is_some())
420 {
421 let ws_client = self.take_ws_client();
422 let retained = Arc::clone(&self.ws_handler_retained);
423
424 if let Err(e) = self
425 .ws_disconnect_handle
426 .spawn(ws_client.disconnect_with_task_retention(retained))
427 {
428 log::error!("Failed to start Lighter WebSocket disconnect task: {e}");
429 }
430 }
431 }
432
433 fn session_tasks_finished(&self) -> bool {
434 self.ws_stream_handle.is_none()
435 && self.ws_disconnect_handle.is_none()
436 && self.ws_handler_retained.is_empty()
437 && self.auth_refresh_handle.is_none()
438 && self.pending_tasks.is_empty()
439 && self.shutdown_errors.is_empty()
440 }
441
442 async fn finish_session_shutdown(&mut self) -> anyhow::Result<()> {
443 self.pending_tasks.begin_shutdown();
444
445 Self::finish_disconnect_task(
446 &mut self.ws_disconnect_handle,
447 "WebSocket disconnect",
448 &mut self.shutdown_errors,
449 )
450 .await;
451
452 if let Err(e) = self.ws_handler_retained.finish().await {
453 self.shutdown_errors.push(e.to_string());
454 }
455 Self::finish_owned_task(
456 &mut self.ws_stream_handle,
457 "execution consumer",
458 &mut self.shutdown_errors,
459 )
460 .await;
461 Self::finish_owned_task(
462 &mut self.auth_refresh_handle,
463 "auth-token refresh",
464 &mut self.shutdown_errors,
465 )
466 .await;
467
468 if let Err(e) = self
469 .pending_tasks
470 .finish_shutdown(Duration::from_secs(1), DISCONNECT_TIMEOUT)
471 .await
472 {
473 self.shutdown_errors
474 .push(format!("client-owned tasks failed: {e}"));
475 }
476
477 let stale = self.dispatch.take_pending_sendtx();
478 warn_pending_sendtx_unknown(&stale, "session shutdown");
479 self.nonce_recovery_inflight.store(false, Ordering::Release);
480
481 if !self.shutdown_errors.is_empty() {
482 let errors = std::mem::take(&mut self.shutdown_errors);
483 anyhow::bail!("Failed to terminate Lighter tasks: {}", errors.join("; "));
484 }
485 Ok(())
486 }
487
488 async fn finish_owned_task(
489 slot: &mut TaskSlot<()>,
490 description: &str,
491 errors: &mut Vec<String>,
492 ) {
493 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
494 return;
495 };
496
497 match outcome {
498 TaskJoinOutcome::Completed(()) => {
499 log::debug!("Lighter {description} task completed");
500 }
501 TaskJoinOutcome::Aborted => {
502 log::debug!("Lighter {description} task cancelled");
503 }
504 TaskJoinOutcome::Failed(e) => {
505 errors.push(format!("{description} task failed: {e}"));
506 }
507 TaskJoinOutcome::Incomplete => {
508 errors.push(format!("{description} task did not stop after abort"));
509 }
510 }
511 }
512
513 async fn finish_disconnect_task(
514 slot: &mut TaskSlot<Result<(), LighterWsError>>,
515 description: &str,
516 errors: &mut Vec<String>,
517 ) {
518 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
519 return;
520 };
521
522 match outcome {
523 TaskJoinOutcome::Completed(Ok(())) | TaskJoinOutcome::Aborted => {}
524 TaskJoinOutcome::Completed(Err(e)) => {
525 errors.push(format!("{description} failed: {e}"));
526 }
527 TaskJoinOutcome::Failed(e) => {
528 errors.push(format!("{description} task failed: {e}"));
529 }
530 TaskJoinOutcome::Incomplete => {
531 errors.push(format!("{description} task did not stop after abort"));
532 }
533 }
534 }
535
536 async fn ensure_instruments_initialized_async(&self) -> anyhow::Result<()> {
537 if self.core.instruments_initialized() {
538 return Ok(());
539 }
540
541 let instruments = self
542 .http_client
543 .request_instruments()
544 .await
545 .context("failed to request Lighter instruments")?;
546
547 let ws_cache: Vec<(i64, InstrumentAny)> = instruments
548 .iter()
549 .filter_map(|instrument| {
550 self.registry
551 .market_index(&instrument.id())
552 .map(|market_index| (market_index, instrument.clone()))
553 })
554 .collect();
555 self.ws_client.cache_instruments(ws_cache);
556 cache_instruments_for_reports(&instruments);
557
558 log::debug!(
559 "Bootstrapped {} Lighter instruments ({} registry entries)",
560 instruments.len(),
561 self.registry.len(),
562 );
563
564 self.core.set_instruments_initialized();
565 Ok(())
566 }
567
568 async fn await_account_streams_ready(&self, timeout_secs: f64) -> anyhow::Result<()> {
569 let timeout = Duration::from_secs_f64(timeout_secs);
570 self.dispatch.account_streams_ready.await_all(timeout).await
571 }
572
573 async fn refresh_nonce(&self) -> anyhow::Result<()> {
574 let Some(credential) = &self.credential else {
575 return Ok(());
576 };
577 self.nonce_ready_connection_epoch
578 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
579 let _refresh_guard = self.nonce_submission_gate.write().await;
580
581 let response = self
582 .http_client
583 .get_next_nonce(credential.account_index(), credential.api_key_index())
584 .await
585 .context("failed to fetch Lighter nextNonce")?;
586
587 self.dispatch.nonce_manager.refresh(
588 credential.account_index(),
589 credential.api_key_index(),
590 response.nonce,
591 );
592
593 self.nonce_recovery_inflight.store(false, Ordering::Release);
595
596 log::debug!(
597 "Refreshed Lighter nonce baseline: account_index={}, api_key_index={}, next_nonce={}",
598 credential.account_index(),
599 credential.api_key_index(),
600 response.nonce,
601 );
602 Ok(())
603 }
604
605 async fn sync_nonce_from_venue(&self) -> anyhow::Result<()> {
606 let Some(credential) = &self.credential else {
607 return Ok(());
608 };
609
610 let _refresh_guard = self.nonce_submission_gate.write().await;
611 let response = self
612 .http_client
613 .get_next_nonce(credential.account_index(), credential.api_key_index())
614 .await
615 .context("failed to fetch Lighter nextNonce")?;
616 self.dispatch.nonce_manager.sync_from_venue(
617 credential.account_index(),
618 credential.api_key_index(),
619 response.nonce,
620 )?;
621 Ok(())
622 }
623
624 async fn fetch_account_detail(&self) -> Option<LighterAccountDetail> {
625 let Some(credential) = &self.credential else {
626 return None;
627 };
628 let account_index = credential.account_index();
629
630 match tokio::time::timeout(
631 ACCOUNT_SNAPSHOT_TIMEOUT,
632 self.http_client.get_account_detail(account_index),
633 )
634 .await
635 {
636 Ok(Ok(detail)) => Some(detail),
637 Ok(Err(e)) => {
638 log::warn!(
639 "Failed to fetch Lighter account detail for account_index={account_index}; \
640 continuing startup without account metadata: {e}"
641 );
642 None
643 }
644 Err(_) => {
645 log::warn!(
646 "Lighter account detail timed out after {}s for account_index={account_index}; \
647 continuing startup without account metadata",
648 ACCOUNT_SNAPSHOT_TIMEOUT.as_secs()
649 );
650 None
651 }
652 }
653 }
654
655 fn detect_account_tier(&self, detail: &LighterAccountDetail) -> LighterAccountTier {
660 let account_index = detail.account_index;
661 let code = detail.account_type;
662 let tier = LighterAccountTier::from_code(code);
663 let standard_rest = LighterAccountTier::Standard
664 .documented_rest_quota_per_min()
665 .unwrap_or(60);
666 let (active_rest, cross_check) =
667 tier_quota_report(tier, self.config.rest_quota_per_min, standard_rest);
668
669 log_debug!(
670 "Lighter execution account {account_index} reported tier {tier} \
671 (account_type={code}); active REST quota {active_rest} req/min",
672 color = LogColor::Blue
673 );
674
675 match cross_check {
676 Some(TierCrossCheck::AboveTier { documented }) => {
677 log::warn!(
678 "Configured Lighter rest_quota_per_min={active_rest} exceeds the {tier} tier \
679 limit of {documented} req/min; the venue may reject requests unless the \
680 caller IP is registered for the higher limit"
681 );
682 }
683 Some(TierCrossCheck::RaiseHint { documented }) => {
684 log_debug!(
685 "Lighter {tier} tier permits up to {documented} REST req/min; set \
686 rest_quota_per_min (and register the caller IP with Lighter) to use it",
687 color = LogColor::Blue
688 );
689 }
690 None => {}
691 }
692
693 tier
694 }
695
696 fn integrator_account_index(&self) -> Option<u64> {
697 if matches!(
698 self.account_tier,
699 Some(LighterAccountTier::Plus | LighterAccountTier::Premium)
700 ) {
701 deployment::integrator_account_index(self.config.deployment, self.config.environment)
702 } else {
703 None
704 }
705 }
706
707 async fn apply_referral_attribution(
708 &self,
709 detail: &LighterAccountDetail,
710 ) -> anyhow::Result<()> {
711 let Some(referral_code) =
712 deployment::referral_code(self.config.deployment, self.config.environment)
713 else {
714 return Ok(());
715 };
716
717 let Some(credential) = &self.credential else {
718 return Ok(());
719 };
720
721 anyhow::ensure!(
722 !detail.l1_address.trim().is_empty(),
723 "Lighter account detail returned an empty L1 address"
724 );
725
726 let auth_token = build_auth_token_for(credential)
727 .context("failed to mint Lighter auth token for referral attribution")?;
728 let referral_code = Zeroizing::new(referral_code.to_string());
729
730 self.http_client
731 .use_referral(&detail.l1_address, referral_code.as_str(), &auth_token)
732 .await
733 .context("failed to apply Lighter referral code")?;
734
735 log::debug!("Applied Robinhood Chain referral attribution");
736 Ok(())
737 }
738
739 async fn is_maker_only_api_key(&self, credential: &Credential) -> anyhow::Result<bool> {
743 let auth_token = build_auth_token_for(credential)
744 .context("failed to mint Lighter auth token for maker-only check")?;
745 let response = self
746 .http_client
747 .get_maker_only_api_keys(credential.account_index(), auth_token)
748 .await
749 .context("failed to query getMakerOnlyApiKeys")?;
750 let api_key_index = i64::from(credential.api_key_index());
751 Ok(response.api_key_indexes.contains(&api_key_index))
752 }
753
754 async fn submit_integrator_auto_approval(&self) -> anyhow::Result<()> {
755 let Some(integrator_account_index) = self.integrator_account_index() else {
756 return Ok(());
757 };
758
759 let Some(credential) = &self.credential else {
760 return Ok(());
761 };
762
763 let mut maker_only_check_failed = false;
764
765 match self.is_maker_only_api_key(credential).await {
766 Ok(true) => {
767 log::warn!(
768 "Skipping Lighter integrator auto-approval: api_key_index={} is maker-only; \
769 ensure the account has been approved by a non-maker-only key",
770 credential.api_key_index(),
771 );
772 return Ok(());
773 }
774 Ok(false) => {}
775 Err(e) => {
776 maker_only_check_failed = true;
777
778 log::debug!(
779 "Unable to determine whether the Lighter API key is maker-only; proceeding \
780 with integrator auto-approval: {e:?}"
781 );
782 }
783 }
784
785 let mut approval =
786 self.prepare_integrator_auto_approval(credential, integrator_account_index)?;
787
788 let request = LighterSendTxRequest::new(
789 LighterTxType::ApproveIntegrator as u8,
790 approval.tx_info.clone(),
791 );
792
793 approval.send_reservation.wait_for_turn().await;
794
795 let response = match self.http_client.send_tx(&request).await {
796 Ok(response) => response,
797 Err(e) => {
798 if lighter_http_error_is_definite_api_rejection(&e) {
799 let _ = self.dispatch.nonce_manager.ack_failure_if_latest(
800 credential.account_index(),
801 approval.api_key_index,
802 approval.nonce,
803 );
804 }
805 approval.send_reservation.release();
806 let hint = if maker_only_check_failed {
807 " (maker-only pre-flight check failed earlier; venue may reject with 62007 \
808 if this key is maker-only)"
809 } else {
810 ""
811 };
812 return Err(anyhow::Error::new(e).context(format!(
813 "failed to submit Lighter integrator approval nonce={} api_key_index={}{hint}",
814 approval.nonce, approval.api_key_index,
815 )));
816 }
817 };
818
819 let _ = self.dispatch.nonce_manager.ack_success(
820 credential.account_index(),
821 approval.api_key_index,
822 approval.nonce,
823 );
824 approval.send_reservation.release();
825
826 log::debug!(
827 "Submitted Lighter integrator approval: integrator={}, nonce={}, \
828 api_key_index={}, approval_expiry={}, tx_hash={}",
829 integrator_account_index,
830 approval.nonce,
831 approval.api_key_index,
832 approval.approval_expiry,
833 response.tx_hash,
834 );
835 Ok(())
836 }
837
838 fn prepare_integrator_auto_approval(
839 &self,
840 credential: &Credential,
841 integrator_account_index: u64,
842 ) -> anyhow::Result<PreparedIntegratorApproval> {
843 let ReservedTxContext {
844 context,
845 send_reservation,
846 } = self.build_tx_context(credential)?;
847
848 let now_ms = (self.clock.get_time_ns().as_u64() as i64) / 1_000_000;
849 let approval_expiry = now_ms.saturating_add(INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS);
850 let nonce = context.nonce;
851 let api_key_index = context.api_key_index;
852
853 let tx = ApproveIntegratorTxInfo {
854 context,
855 integrator_account_index: integrator_account_index as i64,
856 max_perps_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
857 max_perps_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
858 max_spot_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
859 max_spot_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
860 approval_expiry,
861 skip_nonce: 0,
862 };
863
864 let signed = sign_tx(
865 &tx,
866 self.config.chain_id(),
867 &credential.private_key()?,
868 fresh_k(),
869 );
870
871 let tx_info = TxInfoJson::approve_integrator(&tx, &signed, "");
872
873 Ok(PreparedIntegratorApproval {
874 tx_info,
875 nonce,
876 api_key_index,
877 approval_expiry,
878 send_reservation,
879 })
880 }
881
882 async fn spawn_ws_consumer(&mut self) -> anyhow::Result<()> {
883 let mut ws_guard = TaskRetentionGuard::new(
887 self.ws_client.clone(),
888 Arc::clone(&self.ws_handler_retained),
889 );
890 ws_guard
891 .client_mut()
892 .connect_with_cancellation(self.cancellation_token.clone())
893 .await
894 .context("failed to connect to Lighter WebSocket")?;
895
896 let post_connect = async {
900 ws_guard
901 .client_mut()
902 .wait_until_active()
903 .await
904 .context("Lighter WebSocket did not reach active state")?;
905
906 if let Some(credential) = &self.credential {
907 let auth_token = build_auth_token_for(credential)
908 .context("failed to mint Lighter auth token")?;
909 let account_index = credential.account_index();
910
911 ws_guard
912 .client_mut()
913 .set_execution_context(self.core.account_id, account_index)
914 .await
915 .map_err(|e| anyhow::anyhow!("failed to set Lighter execution context: {e}"))?;
916
917 let channels = [
922 LighterWsChannel::AccountAllOrders(account_index),
923 LighterWsChannel::AccountAllTrades(account_index),
924 LighterWsChannel::AccountAllPositions(account_index),
925 LighterWsChannel::AccountAllAssets(account_index),
926 LighterWsChannel::UserStats(account_index),
927 ];
928
929 for channel in channels {
930 ws_guard
931 .client_mut()
932 .subscribe_account(channel.clone(), auth_token.clone())
933 .await
934 .map_err(|e| {
935 anyhow::anyhow!(
936 "failed to subscribe to Lighter account channel {channel:?}: {e}",
937 )
938 })?;
939 }
940
941 log::debug!("Subscribed to Lighter account streams: account_index={account_index}",);
942 } else {
943 log::warn!(
944 "Lighter execution client has no credentials: account streams not subscribed; \
945 typed execution reports will not flow"
946 );
947 }
948
949 Ok::<(), anyhow::Error>(())
950 };
951
952 if let Err(e) = post_connect.await {
953 log::warn!("Lighter post-connect setup failed, tearing down WS: {e}");
954 let ws_client = ws_guard.disarm();
955 let mut rollback_errors = Vec::new();
956
957 if let Err(e) = ws_client
958 .disconnect_with_task_retention(Arc::clone(&self.ws_handler_retained))
959 .await
960 {
961 rollback_errors.push(e.to_string());
962 }
963
964 if let Err(e) = self.ws_handler_retained.finish().await {
965 rollback_errors.push(e.to_string());
966 }
967
968 if rollback_errors.is_empty() {
969 return Err(e);
970 }
971 return Err(e.context(format!(
972 "Lighter post-connect rollback failed: {}",
973 rollback_errors.join("; ")
974 )));
975 }
976
977 let mut ws_client = ws_guard.disarm();
978 self.ws_client.set_task_slot(ws_client.take_task_slot());
979
980 let cancellation_token = self.cancellation_token.clone();
981 let emitter = self.emitter.clone();
982 let dispatch = self.dispatch.clone();
983 let credential_for_loop = self.credential.clone();
984 let http_client_for_loop = self.http_client.clone();
985 let registry_for_loop = Arc::clone(&self.registry);
986 let auth_refresh_notify = Arc::clone(&self.auth_refresh_notify);
987 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
988 let nonce_ready_connection_epoch = Arc::clone(&self.nonce_ready_connection_epoch);
989 let pending_tasks_for_loop = self
990 .pending_tasks
991 .spawner()
992 .context("Lighter task admission is closed")?;
993 let account_id_for_loop = self.core.account_id;
994 let clock_for_loop = self.clock;
995 let nonce_refresh_retry = NonceRefreshRetry {
996 http_client: self.http_client.clone(),
997 dispatch: self.dispatch.clone(),
998 credential: self.credential.clone(),
999 submission_gate: Arc::clone(&self.nonce_submission_gate),
1000 ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
1001 ws_client: ws_client.clone(),
1002 cancellation_token: self.cancellation_token.clone(),
1003 pending_tasks: pending_tasks_for_loop.clone(),
1004 };
1005
1006 self.ws_stream_handle.spawn(async move {
1007 log::debug!("Lighter execution WebSocket consumption loop started");
1008
1009 loop {
1010 tokio::select! {
1011 () = cancellation_token.cancelled() => {
1012 log::debug!("Lighter execution consumption loop cancelled");
1013 break;
1014 }
1015 msg_opt = ws_client.next_event() => {
1016 match msg_opt {
1017 Some(NautilusWsMessage::ExecutionReports(reports)) => {
1018 let mut order_count = 0_usize;
1019 let mut fill_count = 0_usize;
1020 let trader_id = emitter.trader_id();
1021 let account_index = credential_for_loop
1022 .as_ref()
1023 .map(|c| c.account_index());
1024
1025 for report in reports {
1026 match report {
1027 ExecutionReport::Order(order) => {
1028 order_count += 1;
1029 dispatch_lighter_order(
1030 &order,
1031 &dispatch,
1032 &emitter,
1033 ®istry_for_loop,
1034 account_id_for_loop,
1035 trader_id,
1036 clock_for_loop.get_time_ns(),
1037 );
1038 }
1039 ExecutionReport::Fill(trade) => {
1040 fill_count += 1;
1041 dispatch_lighter_trade(
1042 &trade,
1043 &dispatch,
1044 &emitter,
1045 ®istry_for_loop,
1046 account_id_for_loop,
1047 trader_id,
1048 account_index,
1049 clock_for_loop.get_time_ns(),
1050 );
1051 }
1052 }
1053 }
1054 log::debug!(
1055 "Lighter execution batch: orders={order_count} fills={fill_count}",
1056 );
1057 }
1058 Some(NautilusWsMessage::PositionSnapshot {
1059 reports,
1060 skipped_market_ids,
1061 }) => {
1062 for r in &reports {
1065 if let Some(idx) =
1066 registry_for_loop.market_index(&r.instrument_id)
1067 {
1068 dispatch.note_active_market(idx);
1069 }
1070 }
1071 let position_count = reports.len();
1072 let retained_positions: Vec<InstrumentId> = skipped_market_ids
1073 .iter()
1074 .filter_map(|market_id| {
1075 registry_for_loop.instrument_id(*market_id)
1076 })
1077 .collect();
1078 let removed = dispatch.replace_position_snapshot(
1079 &reports,
1080 &retained_positions,
1081 &skipped_market_ids,
1082 );
1083 log::debug!(
1084 "Lighter position snapshot: positions={position_count}, skipped_markets={}, removed={}",
1085 skipped_market_ids.len(),
1086 removed.len(),
1087 );
1088 emit_lighter_position_reports(
1089 reports,
1090 removed,
1091 &emitter,
1092 account_id_for_loop,
1093 clock_for_loop.get_time_ns(),
1094 );
1095 }
1096 Some(NautilusWsMessage::PositionUpdate {
1097 reports,
1098 closed_market_ids,
1099 skipped_market_ids,
1100 }) => {
1101 let mut covered_market_ids = closed_market_ids.clone();
1102
1103 for report in &reports {
1104 if let Some(market_id) =
1105 registry_for_loop.market_index(&report.instrument_id)
1106 {
1107 dispatch.note_active_market(market_id);
1108 covered_market_ids.push(market_id);
1109 }
1110 }
1111 let position_count = reports.len();
1112 let closed_positions: Vec<InstrumentId> = closed_market_ids
1113 .iter()
1114 .filter_map(|market_id| {
1115 registry_for_loop.instrument_id(*market_id)
1116 })
1117 .collect();
1118 let removed = dispatch.apply_position_update(
1119 &reports,
1120 &closed_positions,
1121 &covered_market_ids,
1122 &skipped_market_ids,
1123 );
1124 log::debug!(
1125 "Lighter position update: positions={position_count}, closed_markets={}, skipped_markets={}, removed={}",
1126 closed_market_ids.len(),
1127 skipped_market_ids.len(),
1128 removed.len(),
1129 );
1130 emit_lighter_position_reports(
1131 reports,
1132 removed,
1133 &emitter,
1134 account_id_for_loop,
1135 clock_for_loop.get_time_ns(),
1136 );
1137 }
1138 Some(NautilusWsMessage::AccountState(state)) => {
1139 log::debug!(
1140 "Lighter AccountState: balances={} margins={}",
1141 state.balances.len(),
1142 state.margins.len(),
1143 );
1144 dispatch.cache_account_state((*state).clone());
1150 emitter.send_account_state(*state);
1151 }
1152 Some(NautilusWsMessage::Reconnected { connection_epoch }) => {
1153 dispatch.invalidate_position_snapshot();
1154 nonce_ready_connection_epoch.store(
1155 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1156 Ordering::Release,
1157 );
1158 log::debug!("Lighter WebSocket reconnected (execution stream)");
1166 auth_refresh_notify.notify_one();
1167
1168 let _refresh_guard = nonce_submission_gate.write().await;
1182 let disconnected_epoch = connection_epoch.saturating_sub(1);
1183 let stale = dispatch.drain_pending_sendtx(disconnected_epoch);
1184 warn_pending_sendtx_unknown(&stale, "reconnect");
1185
1186 if let Some(credential) = &credential_for_loop {
1187 match http_client_for_loop
1188 .get_next_nonce(
1189 credential.account_index(),
1190 credential.api_key_index(),
1191 )
1192 .await
1193 {
1194 Ok(response) => {
1195 dispatch.nonce_manager.refresh(
1196 credential.account_index(),
1197 credential.api_key_index(),
1198 response.nonce,
1199 );
1200 nonce_ready_connection_epoch
1201 .store(connection_epoch, Ordering::Release);
1202 log::debug!(
1203 "Refreshed Lighter nonce after reconnect: \
1204 account_index={}, next_nonce={}",
1205 credential.account_index(),
1206 response.nonce,
1207 );
1208 }
1209 Err(e) => {
1210 log::error!(
1211 "Failed to refresh Lighter nonce after reconnect: {e}",
1212 );
1213 nonce_refresh_retry
1214 .clone()
1215 .spawn(connection_epoch);
1216 }
1217 }
1218 }
1219 }
1220 Some(message @ (NautilusWsMessage::SendTxAck { .. } | NautilusWsMessage::SendTxBatchResult { .. })) => {
1221 let account_index = credential_for_loop.as_ref().map(|c| c.account_index());
1222 let acked = match message {
1223 NautilusWsMessage::SendTxAck { connection_epoch, tx_hash, code } => {
1224 handle_send_tx_ack_for_connection(&dispatch, account_index, connection_epoch, code, tx_hash.as_deref()).into_iter().collect()
1225 }
1226 NautilusWsMessage::SendTxBatchResult { connection_epoch, id, code, message, tx_hashes } => {
1227 let (acked, resync) = handle_send_tx_batch_result(
1228 &dispatch, &emitter, account_index, connection_epoch,
1229 clock_for_loop.get_time_ns(), &id, code, &message, &tx_hashes,
1230 );
1231
1232 if resync && ws_client.connection_epoch() == connection_epoch {
1233 nonce_ready_connection_epoch.store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
1234
1235 nonce_refresh_retry.clone().spawn(connection_epoch);
1236 }
1237 acked
1238 }
1239 _ => unreachable!("matched transaction response"),
1240 };
1241
1242 for pending in acked {
1243 let Some(credential) = credential_for_loop.clone() else { continue; };
1244 spawn_acked_order_probe(
1245 &pending,
1246 AckedOrderProbeContext {
1247 http_client: http_client_for_loop.clone(),
1248 registry: Arc::clone(®istry_for_loop),
1249 credential,
1250 dispatch: dispatch.clone(),
1251 account_id: account_id_for_loop,
1252 clock: clock_for_loop,
1253 emitter: emitter.clone(),
1254 connection_epoch: ws_client.connection_epoch_atomic(),
1255 cancellation_token: cancellation_token.clone(),
1256 pending_tasks: pending_tasks_for_loop.clone(),
1257 },
1258 );
1259 }
1260 }
1261 Some(NautilusWsMessage::SendTxRejected {
1262 connection_epoch,
1263 source,
1264 code,
1265 message,
1266 tx_hash,
1267 }) => {
1268 let account_index = credential_for_loop
1269 .as_ref()
1270 .map(|c| c.account_index());
1271 let needs_nonce_resync = handle_send_tx_rejection_for_connection(
1272 &dispatch,
1273 &emitter,
1274 account_index,
1275 connection_epoch,
1276 clock_for_loop.get_time_ns(),
1277 source,
1278 code,
1279 &message,
1280 tx_hash.as_deref(),
1281 );
1282
1283 if needs_nonce_resync
1287 && let Some(credential) = &credential_for_loop
1288 {
1289 if ws_client.connection_epoch() != connection_epoch {
1290 log::warn!(
1291 "Skipping stale Lighter nonce rejection from connection \
1292 epoch {connection_epoch}",
1293 );
1294 continue;
1295 }
1296 nonce_ready_connection_epoch.store(
1297 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1298 Ordering::Release,
1299 );
1300 let _refresh_guard = nonce_submission_gate.write().await;
1301
1302 match http_client_for_loop
1303 .get_next_nonce(
1304 credential.account_index(),
1305 credential.api_key_index(),
1306 )
1307 .await
1308 {
1309 Ok(response) => {
1310 dispatch.nonce_manager.refresh(
1311 credential.account_index(),
1312 credential.api_key_index(),
1313 response.nonce,
1314 );
1315 nonce_ready_connection_epoch
1316 .store(connection_epoch, Ordering::Release);
1317 log::debug!(
1318 "Hard-refreshed Lighter nonce after invalid-nonce \
1319 rejection: account_index={}, next_nonce={}",
1320 credential.account_index(),
1321 response.nonce,
1322 );
1323 }
1324 Err(e) => {
1325 log::error!(
1326 "Failed to refresh Lighter nonce after \
1327 invalid-nonce rejection: {e}",
1328 );
1329 nonce_refresh_retry
1330 .clone()
1331 .spawn(connection_epoch);
1332 }
1333 }
1334 }
1335 }
1336 Some(NautilusWsMessage::Raw(value)) => {
1337 log::debug!("Unhandled Lighter raw frame on execution stream: {value}");
1338 }
1339 Some(NautilusWsMessage::AccountStreamFirstFrame(stream)) => {
1340 match stream {
1346 AccountStream::Orders => {
1347 dispatch.account_streams_ready.mark_orders();
1348 }
1349 AccountStream::Trades => {
1350 dispatch.account_streams_ready.mark_trades();
1351 }
1352 AccountStream::Positions => {
1353 dispatch.account_streams_ready.mark_positions();
1354 }
1355 AccountStream::Assets => {
1356 dispatch.account_streams_ready.mark_assets();
1357 }
1358 AccountStream::UserStats => {
1359 dispatch.account_streams_ready.mark_user_stats();
1360 }
1361 }
1362 }
1363 Some(
1367 NautilusWsMessage::Trades(_)
1368 | NautilusWsMessage::Quote(_)
1369 | NautilusWsMessage::Deltas(_)
1370 | NautilusWsMessage::Depth(_)
1371 | NautilusWsMessage::Bar(_)
1372 | NautilusWsMessage::MarkPrice(_)
1373 | NautilusWsMessage::IndexPrice(_)
1374 | NautilusWsMessage::FundingRate(_),
1375 ) => {}
1376 None => {
1377 log::debug!("Lighter execution next_event returned None");
1378 tokio::select! {
1379 () = cancellation_token.cancelled() => {
1380 log::debug!(
1381 "Lighter execution consumption loop cancelled"
1382 );
1383 break;
1384 }
1385 () = tokio::time::sleep(Duration::from_secs(1)) => {}
1386 }
1387 }
1388 }
1389 }
1390 }
1391 }
1392
1393 log::debug!("Lighter execution WebSocket consumption loop finished");
1394 })?;
1395
1396 if let Some(credential) = &self.credential {
1397 self.spawn_auth_token_refresh(credential.clone())?;
1398 }
1399
1400 Ok(())
1401 }
1402
1403 fn spawn_auth_token_refresh(&mut self, credential: Credential) -> anyhow::Result<()> {
1404 let ws_client = self.ws_client.clone();
1405 let cancellation_token = self.cancellation_token.clone();
1406 let account_index = credential.account_index();
1407 let channels = auth_token_rotation_channels(account_index);
1408 let refresh_notify = Arc::clone(&self.auth_refresh_notify);
1409
1410 self.auth_refresh_handle.spawn(async move {
1411 log::debug!(
1412 "Lighter auth-token refresh task started: interval={}s, account_index={account_index}",
1413 AUTH_TOKEN_REFRESH_INTERVAL.as_secs(),
1414 );
1415
1416 let mut next_refresh_delay = AUTH_TOKEN_REFRESH_INTERVAL;
1417
1418 loop {
1419 tokio::select! {
1420 biased;
1421 () = cancellation_token.cancelled() => {
1422 log::debug!("Lighter auth-token refresh task cancelled");
1423 break;
1424 }
1425 () = refresh_notify.notified() => {
1426 log::debug!(
1427 "Refreshing Lighter auth token immediately after reconnect: account_index={account_index}",
1428 );
1429 }
1430 () = tokio::time::sleep(next_refresh_delay) => {}
1431 }
1432
1433 let outcome = refresh_auth_token_until_rotated(
1434 &credential,
1435 &channels,
1436 &cancellation_token,
1437 AUTH_TOKEN_REFRESH_BACKOFF,
1438 build_auth_token_for,
1439 |channel, token| {
1440 let ws_client = ws_client.clone();
1441 async move { ws_client.subscribe_account(channel, token).await }
1442 },
1443 )
1444 .await;
1445
1446 if let Some(delay) = auth_token_refresh_next_delay(outcome) {
1447 if outcome == AuthTokenRefreshOutcome::Exhausted {
1448 log::error!(
1449 "Lighter auth-token rotation exhausted retry window; retrying again in {}s: account_index={account_index}",
1450 delay.as_secs(),
1451 );
1452 }
1453 next_refresh_delay = delay;
1454 } else {
1455 log::debug!("Lighter auth-token refresh task cancelled");
1456 break;
1457 }
1458 }
1459 })?;
1460 Ok(())
1461 }
1462
1463 fn resolve_slippage_bps(&self, params: Option<&Params>) -> u32 {
1465 params
1466 .and_then(|p| p.get_u64("market_order_slippage_bps"))
1467 .map_or(self.config.market_order_slippage_bps, |v| v as u32)
1468 }
1469
1470 fn build_tx_context(&self, credential: &Credential) -> anyhow::Result<ReservedTxContext> {
1471 let connection_epoch = self.ws_client.connection_epoch();
1472 anyhow::ensure!(
1473 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1474 "Lighter nonce refresh is pending for connection epoch {connection_epoch}",
1475 );
1476 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
1477 .try_read_owned()
1478 .context("Lighter nonce refresh is in progress")?;
1479 anyhow::ensure!(
1480 self.ws_client.connection_epoch() == connection_epoch
1481 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1482 "Lighter connection or nonce state changed during transaction preparation",
1483 );
1484 let nonce = match self
1485 .dispatch
1486 .nonce_manager
1487 .next_nonce(credential.account_index(), credential.api_key_index())
1488 {
1489 Ok(nonce) => nonce,
1490 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
1491 self.spawn_nonce_window_recovery(credential);
1494 anyhow::bail!("failed to allocate Lighter nonce: {e}");
1495 }
1496 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
1497 };
1498
1499 let now_ns = self.clock.get_time_ns().as_u64() as i64;
1500 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
1501 let send_reservation = self
1502 .tx_send_sequencer
1503 .reserve(
1504 credential.account_index(),
1505 credential.api_key_index(),
1506 nonce,
1507 )
1508 .with_nonce_ownership(nonce_guard, connection_epoch);
1509
1510 let context = TxContext {
1511 account_index: credential.account_index(),
1512 api_key_index: credential.api_key_index(),
1513 nonce,
1514 expired_at,
1515 };
1516
1517 Ok(ReservedTxContext {
1518 context,
1519 send_reservation,
1520 })
1521 }
1522
1523 fn spawn_nonce_window_recovery(&self, credential: &Credential) {
1524 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
1525 return;
1526 }
1527
1528 let inflight = Arc::clone(&self.nonce_recovery_inflight);
1529 let http_client = self.http_client.clone();
1530 let dispatch = self.dispatch.clone();
1531 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1532 let account_index = credential.account_index();
1533 let api_key_index = credential.api_key_index();
1534
1535 self.spawn_task("nonce_window_recovery", async move {
1536 let _refresh_guard = nonce_submission_gate.write().await;
1537 let result = http_client
1538 .get_next_nonce(account_index, api_key_index)
1539 .await;
1540 inflight.store(false, Ordering::Release);
1541
1542 match result {
1543 Ok(response) => {
1544 let _ = dispatch.nonce_manager.sync_from_venue(
1547 account_index,
1548 api_key_index,
1549 response.nonce,
1550 );
1551 log::debug!(
1552 "Resynced Lighter nonce baseline after skip-window exhaustion: \
1553 account_index={account_index}, api_key_index={api_key_index}, \
1554 next_nonce={}",
1555 response.nonce,
1556 );
1557 }
1558 Err(e) => {
1559 log::error!(
1560 "Failed to resync Lighter nonce after skip-window exhaustion: {e}",
1561 );
1562 }
1563 }
1564 Ok(())
1565 });
1566 }
1567
1568 fn dispatch_create_order_plan(
1569 &self,
1570 plan: CreateOrderPlan,
1571 credential: &Credential,
1572 ) -> anyhow::Result<()> {
1573 let context = self.fanout_dispatch_context(credential)?;
1574 let prepared = context.sign_create_order(plan)?;
1575 self.spawn_task("submit_order", async move {
1576 context.send_create_order(prepared).await;
1577 Ok(())
1578 });
1579
1580 Ok(())
1581 }
1582
1583 fn prepare_create_order_plan(
1584 &self,
1585 order: &OrderAny,
1586 slippage_bps: u32,
1587 ) -> anyhow::Result<CreateOrderPlan> {
1588 let instrument_id = order.instrument_id();
1589 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
1590 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
1591 })?;
1592
1593 let instrument = self.core.cache().try_instrument(&instrument_id)?.clone();
1594
1595 let order_kind = nautilus_to_lighter_order_type(order.order_type())?;
1596
1597 let tif = nautilus_to_lighter_tif(
1598 order.order_type(),
1599 order.time_in_force(),
1600 order.is_post_only(),
1601 )?;
1602 let now_ms = (self.clock.get_time_ns().as_u64() / 1_000_000) as i64;
1603 let order_expiry = order_expiry_for(
1604 order.order_type(),
1605 &order.time_in_force(),
1606 order.expire_time(),
1607 now_ms,
1608 self.config.use_gtd,
1609 )?;
1610
1611 let base_amount = quantity_to_ticks(&order.quantity(), instrument.size_precision())?;
1612
1613 anyhow::ensure!(
1615 base_amount > 0,
1616 "quantity `{}` rounds to 0 ticks at size_precision {}",
1617 order.quantity(),
1618 instrument.size_precision(),
1619 );
1620 let price_precision = instrument.price_precision();
1621 let is_buy = matches!(order.order_side(), OrderSide::Buy);
1622
1623 let price_ticks = match order.order_type() {
1626 OrderType::Market => {
1627 let quote = self
1628 .core
1629 .cache()
1630 .quote(&instrument_id)
1631 .copied()
1632 .ok_or_else(|| {
1633 anyhow::anyhow!(
1634 "no cached quote for {instrument_id}: subscribe to quotes before submitting MARKET orders",
1635 )
1636 })?;
1637 let base = if is_buy {
1638 quote.ask_price.as_decimal()
1639 } else {
1640 quote.bid_price.as_decimal()
1641 };
1642 derive_market_order_price_ticks(base, is_buy, price_precision, slippage_bps)?
1643 }
1644 OrderType::StopMarket | OrderType::MarketIfTouched => {
1645 let trigger = order.trigger_price().ok_or_else(|| {
1646 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type(),)
1647 })?;
1648 derive_market_order_price_ticks(
1649 trigger.as_decimal(),
1650 is_buy,
1651 price_precision,
1652 slippage_bps,
1653 )?
1654 }
1655 _ => order
1656 .price()
1657 .map(|p| price_to_ticks(&p, price_precision))
1658 .transpose()?
1659 .unwrap_or(0),
1660 };
1661
1662 let trigger_price_ticks = order
1663 .trigger_price()
1664 .map(|p| price_to_ticks(&p, price_precision))
1665 .transpose()?
1666 .unwrap_or(0);
1667
1668 if matches!(
1671 order.order_type(),
1672 OrderType::StopMarket
1673 | OrderType::StopLimit
1674 | OrderType::MarketIfTouched
1675 | OrderType::LimitIfTouched
1676 ) {
1677 anyhow::ensure!(
1678 trigger_price_ticks > 0,
1679 "trigger_price `{:?}` rounds to 0 ticks at precision {price_precision}",
1680 order.trigger_price(),
1681 );
1682 }
1683 validate_order_amount(&instrument, order.quantity(), price_ticks, price_precision)?;
1684
1685 Ok(CreateOrderPlan {
1686 order: order.clone(),
1687 market_index,
1688 base_amount,
1689 price: price_ticks,
1690 order_type: order_kind as u8,
1691 time_in_force: tif as u8,
1692 trigger_price: trigger_price_ticks,
1693 order_expiry,
1694 })
1695 }
1696
1697 fn fanout_dispatch_context(
1698 &self,
1699 credential: &Credential,
1700 ) -> anyhow::Result<FanoutDispatchContext> {
1701 let pending_tasks = self
1702 .pending_tasks
1703 .spawner()
1704 .context("Lighter task admission is closed")?;
1705 Ok(FanoutDispatchContext {
1706 clock: self.clock,
1707 chain_id: self.config.chain_id(),
1708 integrator_account_index: self.integrator_account_index(),
1709 emitter: self.emitter.clone(),
1710 credential: credential.clone(),
1711 http_client: self.http_client.clone(),
1712 ws_client: self.ws_client.clone(),
1713 tx_rate_limiter: Arc::clone(&self.tx_rate_limiter),
1714 tx_send_sequencer: self.tx_send_sequencer.clone(),
1715 nonce_submission_gate: Arc::clone(&self.nonce_submission_gate),
1716 nonce_ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
1717 dispatch: self.dispatch.clone(),
1718 nonce_recovery_inflight: Arc::clone(&self.nonce_recovery_inflight),
1719 pending_tasks,
1720 })
1721 }
1722
1723 fn dispatch_signed_cancel_order(&self, cmd: &CancelOrder, credential: &Credential) {
1724 let context = match self.fanout_dispatch_context(credential) {
1725 Ok(context) => context,
1726 Err(e) => {
1727 log::warn!("Skipping Lighter cancel_order after shutdown began: {e}");
1728 return;
1729 }
1730 };
1731 self.dispatch
1732 .set_pending_order_action(cmd.client_order_id, PendingOrderAction::Cancel);
1733 let emit_cancel_rejected = self.can_emit_order_cancel_rejected(&cmd.client_order_id);
1734 if !emit_cancel_rejected {
1735 self.dispatch
1736 .clear_pending_order_action_if(&cmd.client_order_id, PendingOrderAction::Cancel);
1737 }
1738
1739 let prepared = match self
1740 .prepare_cancel_order_plan(cmd)
1741 .and_then(|plan| context.sign_cancel_order(&plan))
1742 {
1743 Ok(prepared) => prepared,
1744 Err(e) => {
1745 self.dispatch.clear_pending_order_action_if(
1746 &cmd.client_order_id,
1747 PendingOrderAction::Cancel,
1748 );
1749 let reason = format!("Lighter cancel_order failed: {e}");
1750 if emit_cancel_rejected {
1751 log::warn!("{reason} for {}", cmd.client_order_id);
1752 self.emitter.emit_order_cancel_rejected_event(
1753 cmd.strategy_id,
1754 cmd.instrument_id,
1755 cmd.client_order_id,
1756 cmd.venue_order_id,
1757 &reason,
1758 self.clock.get_time_ns(),
1759 );
1760 } else {
1761 log::warn!(
1762 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
1763 cmd.client_order_id,
1764 );
1765 }
1766 return;
1767 }
1768 };
1769
1770 self.spawn_task("cancel_order", async move {
1771 context
1772 .send_cancel_order(prepared, emit_cancel_rejected)
1773 .await;
1774 Ok(())
1775 });
1776 }
1777
1778 fn dispatch_cancel_orders(&self, cancels: &[CancelOrder]) -> anyhow::Result<()> {
1779 let credential = self.credential.as_ref().ok_or_else(|| {
1780 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
1781 })?;
1782
1783 if cancels.is_empty() {
1784 log::debug!("batch_cancel_orders called with empty cancel list");
1785 return Ok(());
1786 }
1787
1788 let context = self.fanout_dispatch_context(credential)?;
1789
1790 for cancel in cancels {
1791 self.dispatch
1792 .set_pending_order_action(cancel.client_order_id, PendingOrderAction::Cancel);
1793 }
1794
1795 let emit_cancel_rejected: Vec<bool> = cancels
1796 .iter()
1797 .map(|cancel| self.can_emit_order_cancel_rejected(&cancel.client_order_id))
1798 .collect();
1799
1800 for (cancel, emit) in cancels.iter().zip(&emit_cancel_rejected) {
1801 if !emit {
1802 self.dispatch.clear_pending_order_action_if(
1803 &cancel.client_order_id,
1804 PendingOrderAction::Cancel,
1805 );
1806 }
1807 }
1808
1809 let mut plans = Vec::with_capacity(cancels.len());
1810
1811 for (cancel, emit_cancel_rejected) in cancels.iter().zip(emit_cancel_rejected) {
1812 match self.prepare_cancel_order_plan(cancel) {
1813 Ok(plan) => plans.push((plan, emit_cancel_rejected)),
1814 Err(e) => {
1815 self.dispatch.clear_pending_order_action_if(
1816 &cancel.client_order_id,
1817 PendingOrderAction::Cancel,
1818 );
1819 let reason = format!("Lighter cancel_order failed: {e}");
1820
1821 if emit_cancel_rejected {
1822 self.emitter.emit_order_cancel_rejected_event(
1823 cancel.strategy_id,
1824 cancel.instrument_id,
1825 cancel.client_order_id,
1826 cancel.venue_order_id,
1827 &reason,
1828 self.clock.get_time_ns(),
1829 );
1830 } else {
1831 log::warn!(
1832 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
1833 cancel.client_order_id,
1834 );
1835 }
1836 }
1837 }
1838 }
1839
1840 if plans.is_empty() {
1841 return Ok(());
1842 }
1843
1844 self.spawn_task("batch_cancel_orders", async move {
1845 for chunk in plans.chunks(LIGHTER_MAX_BATCH_TX) {
1846 let mut batch = Vec::with_capacity(chunk.len());
1847 let mut remaining = chunk.iter().peekable();
1848 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
1849
1850 while let Some((plan, emit_cancel_rejected)) = remaining.peek() {
1851 let emit_cancel_rejected = *emit_cancel_rejected;
1852 let failure = (
1853 plan.strategy_id,
1854 plan.instrument_id,
1855 plan.client_order_id,
1856 plan.venue_order_id,
1857 );
1858
1859 match context.sign_cancel_order(plan) {
1860 Ok(prepared) => {
1861 batch.push((prepared, emit_cancel_rejected));
1862 retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
1863 }
1864 Err(e)
1865 if matches!(
1866 e.downcast_ref::<NonceError>(),
1867 Some(NonceError::SkipWindowExhausted { .. })
1868 ) || e.is::<tokio::sync::TryLockError>()
1869 || context.nonce_ready_connection_epoch.load(Ordering::Acquire)
1870 != context.ws_client.connection_epoch() =>
1871 {
1872 context.send_cancel_batch(std::mem::take(&mut batch)).await;
1874 tokio::time::sleep(retry_delay).await;
1875 retry_delay = (retry_delay * 2).min(NONCE_REFRESH_RETRY_MAX_DELAY);
1876 continue;
1877 }
1878 Err(e) if emit_cancel_rejected => {
1879 context.dispatch.clear_pending_order_action_if(
1880 &failure.2,
1881 PendingOrderAction::Cancel,
1882 );
1883 context.emitter.emit_order_cancel_rejected_event(
1884 failure.0,
1885 failure.1,
1886 failure.2,
1887 failure.3,
1888 &format!("Lighter cancel_order failed: {e}"),
1889 context.clock.get_time_ns(),
1890 );
1891 }
1892 Err(e) => {
1893 context.dispatch.clear_pending_order_action_if(
1894 &failure.2,
1895 PendingOrderAction::Cancel,
1896 );
1897
1898 log::warn!(
1899 "Lighter cancel_order failed: {e} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
1900 failure.2,
1901 );
1902 }
1903 }
1904
1905 remaining.next();
1906 }
1907
1908 context.send_cancel_batch(batch).await;
1909 }
1910
1911 Ok(())
1912 });
1913
1914 Ok(())
1915 }
1916
1917 fn can_emit_order_cancel_rejected(&self, client_order_id: &ClientOrderId) -> bool {
1918 self.core
1919 .cache()
1920 .order(client_order_id)
1921 .is_none_or(|order| order.is_pending_cancel())
1922 }
1923
1924 fn prepare_cancel_order_plan(&self, cmd: &CancelOrder) -> anyhow::Result<CancelOrderPlan> {
1925 let market_index = self
1926 .registry
1927 .market_index(&cmd.instrument_id)
1928 .ok_or_else(|| {
1929 anyhow::anyhow!(
1930 "no Lighter market_index registered for instrument {}",
1931 cmd.instrument_id,
1932 )
1933 })?;
1934
1935 self.core.cache().try_order(&cmd.client_order_id)?;
1936
1937 let voi = cmd
1940 .venue_order_id
1941 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
1942 .ok_or_else(|| {
1943 anyhow::anyhow!(
1944 "cannot cancel Lighter order {}: venue order_id not yet known \
1945 (await OrderAccepted before issuing cancel)",
1946 cmd.client_order_id,
1947 )
1948 })?;
1949
1950 let venue_index: i64 = voi
1951 .as_str()
1952 .parse()
1953 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
1954
1955 Ok(CancelOrderPlan {
1956 client_order_id: cmd.client_order_id,
1957 strategy_id: cmd.strategy_id,
1958 instrument_id: cmd.instrument_id,
1959 venue_order_id: Some(voi),
1960 market_index,
1961 venue_index,
1962 })
1963 }
1964
1965 fn dispatch_signed_modify_order(&self, cmd: &ModifyOrder, credential: &Credential) {
1966 self.dispatch
1967 .set_pending_order_action(cmd.client_order_id, PendingOrderAction::Modify);
1968 let prepared = match self.prepare_signed_modify_order(cmd, credential) {
1969 Ok(prepared) => prepared,
1970 Err(e) => {
1971 self.dispatch.clear_pending_order_action_if(
1972 &cmd.client_order_id,
1973 PendingOrderAction::Modify,
1974 );
1975 let reason = format!("Lighter modify_order failed: {e}");
1976 log::warn!("{reason} for {}", cmd.client_order_id);
1977 self.emitter.emit_order_modify_rejected_event(
1978 cmd.strategy_id,
1979 cmd.instrument_id,
1980 cmd.client_order_id,
1981 cmd.venue_order_id,
1982 &reason,
1983 self.clock.get_time_ns(),
1984 );
1985 return;
1986 }
1987 };
1988 let PreparedModifyOrder {
1989 client_order_id,
1990 strategy_id,
1991 instrument_id,
1992 venue_order_id,
1993 tx_info,
1994 nonce,
1995 api_key_index,
1996 tx_hash,
1997 mut send_reservation,
1998 } = prepared;
1999 let connection_epoch = send_reservation.connection_epoch;
2000
2001 let ws_client = self.ws_client.clone();
2002 let dispatch = self.dispatch.clone();
2003 let credential = credential.clone();
2004 let emitter = self.emitter.clone();
2005 let clock = self.clock;
2006
2007 let tx_rate_limiter = self.tx_rate_limiter.clone();
2008
2009 self.spawn_task("modify_order", async move {
2010 send_reservation.wait_for_turn().await;
2011 await_tx_quota(&tx_rate_limiter).await;
2012 dispatch.enqueue_pending_sendtx(PendingSendTx {
2013 batch_id: None,
2014 connection_epoch,
2015 kind: PendingSendTxKind::Modify {
2016 strategy_id,
2017 instrument_id,
2018 client_order_id,
2019 venue_order_id,
2020 },
2021 submitted_at: clock.get_time_ns(),
2022 nonce,
2023 api_key_index,
2024 tx_hash,
2025 });
2026
2027 if let Err(e) = ws_client
2028 .send_tx_on_connection(LighterTxType::ModifyOrder as u8, tx_info, connection_epoch)
2029 .await
2030 {
2031 let failure = classify_lighter_ws_command_failure("modify_order", &e);
2032 let reason = command_failure_reason(&failure);
2033 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2034 log::warn!(
2035 "Lighter modify_order dispatch outcome unknown for {client_order_id}: {reason}; \
2036 retaining pending state for venue reconciliation; diagnostic={e:?}",
2037 );
2038 } else {
2039 log::error!("{reason} for {client_order_id}; diagnostic={e:?}");
2040 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, nonce);
2041 dispatch.clear_pending_order_action_if(
2042 &client_order_id,
2043 PendingOrderAction::Modify,
2044 );
2045 rollback_tx_dispatch(&dispatch, &credential, None, nonce);
2046 emitter.emit_order_modify_rejected_event(
2047 strategy_id,
2048 instrument_id,
2049 client_order_id,
2050 venue_order_id,
2051 reason,
2052 clock.get_time_ns(),
2053 );
2054 }
2055 }
2056 send_reservation.release();
2057 Ok(())
2058 });
2059 }
2060
2061 fn prepare_signed_modify_order(
2062 &self,
2063 cmd: &ModifyOrder,
2064 credential: &Credential,
2065 ) -> anyhow::Result<PreparedModifyOrder> {
2066 let market_index = self
2067 .registry
2068 .market_index(&cmd.instrument_id)
2069 .ok_or_else(|| {
2070 anyhow::anyhow!(
2071 "no Lighter market_index registered for instrument {}",
2072 cmd.instrument_id,
2073 )
2074 })?;
2075
2076 let voi = cmd
2077 .venue_order_id
2078 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2079 .ok_or_else(|| {
2080 anyhow::anyhow!(
2081 "cannot modify Lighter order {}: venue order_id not yet known \
2082 (await OrderAccepted before issuing modify)",
2083 cmd.client_order_id,
2084 )
2085 })?;
2086
2087 let venue_index: i64 = voi
2088 .as_str()
2089 .parse()
2090 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2091
2092 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
2093 let instrument = self
2094 .core
2095 .cache()
2096 .try_instrument(&cmd.instrument_id)?
2097 .clone();
2098
2099 let new_qty = cmd.quantity.unwrap_or(order.quantity());
2100 let price_precision = instrument.price_precision();
2101 let new_trigger = cmd.trigger_price.or(order.trigger_price());
2102
2103 let price_ticks = match order.order_type() {
2106 OrderType::StopMarket | OrderType::MarketIfTouched => {
2107 let trigger = new_trigger.ok_or_else(|| {
2108 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type())
2109 })?;
2110 let is_buy = matches!(order.order_side(), OrderSide::Buy);
2111 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
2112
2113 derive_market_order_price_ticks(
2114 trigger.as_decimal(),
2115 is_buy,
2116 price_precision,
2117 slippage_bps,
2118 )?
2119 }
2120 _ => {
2121 let new_price = cmd.price.or(order.price()).ok_or_else(|| {
2122 anyhow::anyhow!("modify_order requires a price (none on order or command)")
2123 })?;
2124
2125 price_to_ticks(&new_price, price_precision)?
2126 }
2127 };
2128
2129 let base_amount = quantity_to_ticks(&new_qty, instrument.size_precision())?;
2130 anyhow::ensure!(
2131 base_amount > 0,
2132 "quantity `{new_qty}` rounds to 0 ticks at size_precision {}",
2133 instrument.size_precision(),
2134 );
2135 let trigger_price_ticks = match new_trigger {
2136 Some(trigger) if !trigger.is_zero() => price_to_ticks(&trigger, price_precision)?,
2137 _ => 0,
2138 };
2139
2140 if matches!(
2141 order.order_type(),
2142 OrderType::StopMarket
2143 | OrderType::StopLimit
2144 | OrderType::MarketIfTouched
2145 | OrderType::LimitIfTouched
2146 ) {
2147 anyhow::ensure!(
2148 trigger_price_ticks > 0,
2149 "trigger_price `{new_trigger:?}` rounds to 0 ticks at precision {price_precision}",
2150 );
2151 }
2152 validate_order_amount(&instrument, new_qty, price_ticks, price_precision)?;
2153
2154 let ReservedTxContext {
2155 context,
2156 send_reservation,
2157 } = self.build_tx_context(credential)?;
2158
2159 let captured_nonce = context.nonce;
2160 let captured_api_key_index = context.api_key_index;
2161
2162 let mut rollback_guard =
2163 TxDispatchGuard::new(self.dispatch.clone(), credential, None, captured_nonce);
2164
2165 let tx = ModifyOrderTxInfo {
2166 context,
2167 market_index,
2168 index: venue_index,
2169 base_amount,
2170 price: price_ticks,
2171 trigger_price: trigger_price_ticks,
2172 attributes: integrator_attributes(self.integrator_account_index()),
2173 };
2174
2175 let signed = sign_tx(
2176 &tx,
2177 self.config.chain_id(),
2178 &credential.private_key()?,
2179 fresh_k(),
2180 );
2181
2182 let tx_info_str = TxInfoJson::modify_order(&tx, &signed);
2183 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
2184 .context("failed to wrap signed Lighter modify tx_info JSON")?;
2185
2186 rollback_guard.disarm();
2187
2188 Ok(PreparedModifyOrder {
2189 client_order_id: cmd.client_order_id,
2190 strategy_id: cmd.strategy_id,
2191 instrument_id: cmd.instrument_id,
2192 venue_order_id: Some(voi),
2193 tx_info,
2194 nonce: captured_nonce,
2195 api_key_index: captured_api_key_index,
2196 tx_hash: signed.tx_hash_hex(),
2197 send_reservation,
2198 })
2199 }
2200
2201 pub fn update_leverage(
2220 &self,
2221 instrument_id: InstrumentId,
2222 initial_margin_fraction: u16,
2223 margin_mode: LighterPositionMarginMode,
2224 ) -> anyhow::Result<()> {
2225 let credential = self.credential.as_ref().ok_or_else(|| {
2226 anyhow::anyhow!("Lighter execution client cannot update leverage without credentials")
2227 })?;
2228
2229 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
2230 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
2231 })?;
2232
2233 anyhow::ensure!(
2234 (1..=10_000).contains(&initial_margin_fraction),
2235 "initial_margin_fraction must be in 1..=10_000, was {initial_margin_fraction}",
2236 );
2237
2238 let ReservedTxContext {
2239 context,
2240 mut send_reservation,
2241 } = self.build_tx_context(credential)?;
2242
2243 let connection_epoch = send_reservation.connection_epoch;
2244
2245 let captured_nonce = context.nonce;
2246 let captured_api_key_index = context.api_key_index;
2247 let mut rollback_guard =
2248 TxDispatchGuard::new(self.dispatch.clone(), credential, None, captured_nonce);
2249
2250 let tx = UpdateLeverageTxInfo {
2251 context,
2252 market_index,
2253 initial_margin_fraction,
2254 margin_mode: margin_mode as u8,
2255 skip_nonce: 0,
2256 };
2257
2258 let signed = sign_tx(
2259 &tx,
2260 self.config.chain_id(),
2261 &credential.private_key()?,
2262 fresh_k(),
2263 );
2264
2265 let tx_info_str = TxInfoJson::update_leverage(&tx, &signed);
2266 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
2267 .context("failed to wrap signed Lighter update_leverage tx_info JSON")?;
2268
2269 rollback_guard.disarm();
2270 let captured_tx_hash = signed.tx_hash_hex();
2271
2272 let ws_client = self.ws_client.clone();
2273 let dispatch = self.dispatch.clone();
2274 let credential = credential.clone();
2275 let clock = self.clock;
2276
2277 let tx_rate_limiter = self.tx_rate_limiter.clone();
2278
2279 self.spawn_task("update_leverage", async move {
2280 send_reservation.wait_for_turn().await;
2281 await_tx_quota(&tx_rate_limiter).await;
2282 dispatch.enqueue_pending_sendtx(PendingSendTx {
2283 batch_id: None,
2284 connection_epoch,
2285 kind: PendingSendTxKind::Other,
2286 submitted_at: clock.get_time_ns(),
2287 nonce: captured_nonce,
2288 api_key_index: captured_api_key_index,
2289 tx_hash: captured_tx_hash,
2290 });
2291
2292 if let Err(e) = ws_client
2293 .send_tx_on_connection(
2294 LighterTxType::UpdateLeverage as u8,
2295 tx_info,
2296 connection_epoch,
2297 )
2298 .await
2299 {
2300 let failure = classify_lighter_ws_command_failure("update_leverage", &e);
2301 let reason = command_failure_reason(&failure);
2302 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2303 log::warn!(
2304 "Lighter update_leverage dispatch outcome unknown for {instrument_id}: \
2305 {reason}; retaining pending nonce for venue reconciliation; diagnostic={e:?}",
2306 );
2307 } else {
2308 log::error!("{reason} for {instrument_id}; diagnostic={e:?}");
2309 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, captured_nonce);
2310 rollback_tx_dispatch(&dispatch, &credential, None, captured_nonce);
2311 }
2312 }
2313 send_reservation.release();
2314 Ok(())
2315 });
2316
2317 Ok(())
2318 }
2319}
2320
2321fn emit_lighter_position_reports(
2322 reports: Vec<PositionStatusReport>,
2323 removed: Vec<InstrumentId>,
2324 emitter: &ExecutionEventEmitter,
2325 account_id: AccountId,
2326 now: UnixNanos,
2327) {
2328 for report in reports {
2329 log::debug!(
2330 "Lighter PositionStatusReport: instrument={} side={:?} qty={}",
2331 report.instrument_id,
2332 report.position_side,
2333 report.quantity,
2334 );
2335 emitter.send_position_report(report);
2336 }
2337
2338 for instrument_id in removed {
2340 let flat = PositionStatusReport::new(
2341 account_id,
2342 instrument_id,
2343 PositionSide::Flat,
2344 Quantity::zero(0),
2345 now,
2346 now,
2347 Some(UUID4::new()),
2348 None,
2349 None,
2350 );
2351 emitter.send_position_report(flat);
2352 }
2353}
2354
2355#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2356enum AuthTokenRefreshOutcome {
2357 Rotated,
2358 Cancelled,
2359 Exhausted,
2360}
2361
2362#[derive(Debug, Clone, Copy)]
2363struct AuthTokenRefreshBackoff {
2364 initial_delay: Duration,
2365 max_delay: Duration,
2366 window: Duration,
2367}
2368
2369#[derive(Clone)]
2370struct NonceRefreshRetry {
2371 http_client: LighterHttpClient,
2372 dispatch: WsDispatchState,
2373 credential: Option<Credential>,
2374 submission_gate: Arc<tokio::sync::RwLock<()>>,
2375 ready_connection_epoch: Arc<AtomicU64>,
2376 ws_client: LighterWebSocketClient,
2377 cancellation_token: CancellationToken,
2378 pending_tasks: TaskSpawner,
2379}
2380
2381impl NonceRefreshRetry {
2382 fn spawn(self, connection_epoch: u64) {
2383 let pending_tasks = self.pending_tasks.clone();
2384
2385 let future = async move {
2386 let Some(credential) = self.credential else {
2387 return;
2388 };
2389 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
2390
2391 loop {
2392 tokio::select! {
2393 () = self.cancellation_token.cancelled() => return,
2394 () = tokio::time::sleep(retry_delay) => {}
2395 }
2396
2397 if self.ws_client.connection_epoch() != connection_epoch
2398 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2399 {
2400 return;
2401 }
2402
2403 let _refresh_guard = tokio::select! {
2404 () = self.cancellation_token.cancelled() => return,
2405 guard = self.submission_gate.write() => guard,
2406 };
2407
2408 if self.ws_client.connection_epoch() != connection_epoch
2409 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2410 {
2411 return;
2412 }
2413
2414 let result = tokio::select! {
2415 () = self.cancellation_token.cancelled() => return,
2416 result = self.http_client.get_next_nonce(
2417 credential.account_index(),
2418 credential.api_key_index(),
2419 ) => result,
2420 };
2421
2422 match result {
2423 Ok(response) => {
2424 if self.ws_client.connection_epoch() != connection_epoch {
2425 return;
2426 }
2427 self.dispatch.nonce_manager.refresh(
2428 credential.account_index(),
2429 credential.api_key_index(),
2430 response.nonce,
2431 );
2432 self.ready_connection_epoch
2433 .store(connection_epoch, Ordering::Release);
2434 log::info!(
2435 "Recovered Lighter nonce refresh: account_index={}, \
2436 connection_epoch={connection_epoch}, next_nonce={}",
2437 credential.account_index(),
2438 response.nonce,
2439 );
2440 return;
2441 }
2442 Err(e) => {
2443 log::error!(
2444 "Failed to retry Lighter nonce refresh for connection epoch \
2445 {connection_epoch}: {e}",
2446 );
2447 }
2448 }
2449
2450 retry_delay = retry_delay
2451 .saturating_mul(2)
2452 .min(NONCE_REFRESH_RETRY_MAX_DELAY);
2453 }
2454 };
2455
2456 if let Err(e) = pending_tasks.spawn(future) {
2457 log::debug!("Skipping Lighter nonce refresh retry during shutdown: {e}");
2458 }
2459 }
2460}
2461
2462fn auth_token_rotation_channels(account_index: i64) -> [LighterWsChannel; 5] {
2463 [
2464 LighterWsChannel::AccountAllOrders(account_index),
2465 LighterWsChannel::AccountAllTrades(account_index),
2466 LighterWsChannel::AccountAllPositions(account_index),
2467 LighterWsChannel::AccountAllAssets(account_index),
2468 LighterWsChannel::UserStats(account_index),
2469 ]
2470}
2471
2472async fn refresh_auth_token_until_rotated<MintToken, Subscribe, SubscribeFuture>(
2473 credential: &Credential,
2474 channels: &[LighterWsChannel],
2475 cancellation_token: &CancellationToken,
2476 backoff: AuthTokenRefreshBackoff,
2477 mut mint_token: MintToken,
2478 mut subscribe: Subscribe,
2479) -> AuthTokenRefreshOutcome
2480where
2481 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2482 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2483 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2484{
2485 let retry_started = tokio::time::Instant::now();
2486 let mut retry_delay = backoff.initial_delay.min(backoff.max_delay);
2487 let mut attempt = 1_u32;
2488
2489 loop {
2490 match rotate_auth_token_once(credential, channels, &mut mint_token, &mut subscribe).await {
2491 Ok(()) => {
2492 log::debug!(
2493 "Lighter auth-token rotated for account_index={}, attempts={attempt}",
2494 credential.account_index(),
2495 );
2496 return AuthTokenRefreshOutcome::Rotated;
2497 }
2498 Err(e) => {
2499 log::warn!("Lighter auth-token rotation attempt {attempt} failed: {e:#}");
2500 }
2501 }
2502
2503 let Some(remaining) = backoff.window.checked_sub(retry_started.elapsed()) else {
2504 log::error!(
2505 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2506 credential.account_index(),
2507 );
2508 return AuthTokenRefreshOutcome::Exhausted;
2509 };
2510
2511 if remaining.as_nanos() == 0 {
2512 log::error!(
2513 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2514 credential.account_index(),
2515 );
2516 return AuthTokenRefreshOutcome::Exhausted;
2517 }
2518
2519 let delay = retry_delay.min(remaining);
2520 log::warn!(
2521 "Retrying Lighter auth-token rotation in {:.3}s: account_index={}, attempts={attempt}",
2522 delay.as_secs_f64(),
2523 credential.account_index(),
2524 );
2525
2526 if !sleep_or_auth_token_refresh_cancelled(delay, cancellation_token).await {
2527 return AuthTokenRefreshOutcome::Cancelled;
2528 }
2529
2530 retry_delay = next_auth_token_refresh_retry_delay(retry_delay, backoff.max_delay);
2531 attempt = attempt.saturating_add(1);
2532 }
2533}
2534
2535async fn rotate_auth_token_once<MintToken, Subscribe, SubscribeFuture>(
2536 credential: &Credential,
2537 channels: &[LighterWsChannel],
2538 mint_token: &mut MintToken,
2539 subscribe: &mut Subscribe,
2540) -> anyhow::Result<()>
2541where
2542 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2543 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2544 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2545{
2546 let token =
2547 mint_token(credential).context("failed to mint Lighter auth token during rotation")?;
2548 let mut first_error = None;
2549
2550 for channel in channels {
2551 if let Err(e) = subscribe(channel.clone(), token.clone()).await {
2552 log::debug!("Lighter auth-token rotation: re-subscribe failed for {channel:?}: {e}",);
2553 first_error.get_or_insert_with(|| format!("{channel:?}: {e}"));
2554 }
2555 }
2556
2557 if let Some(error) = first_error {
2558 anyhow::bail!("failed to re-subscribe Lighter account channels: {error}");
2559 }
2560
2561 Ok(())
2562}
2563
2564async fn sleep_or_auth_token_refresh_cancelled(
2565 duration: Duration,
2566 cancellation_token: &CancellationToken,
2567) -> bool {
2568 tokio::select! {
2569 () = cancellation_token.cancelled() => false,
2570 () = tokio::time::sleep(duration) => true,
2571 }
2572}
2573
2574fn auth_token_refresh_next_delay(outcome: AuthTokenRefreshOutcome) -> Option<Duration> {
2575 match outcome {
2576 AuthTokenRefreshOutcome::Rotated => Some(AUTH_TOKEN_REFRESH_INTERVAL),
2577 AuthTokenRefreshOutcome::Cancelled => None,
2578 AuthTokenRefreshOutcome::Exhausted => Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY),
2579 }
2580}
2581
2582fn next_auth_token_refresh_retry_delay(current: Duration, max: Duration) -> Duration {
2583 current.checked_mul(2).unwrap_or(max).min(max)
2584}
2585
2586#[derive(Debug)]
2587struct ReservedTxContext {
2588 context: TxContext,
2589 send_reservation: TxSendReservation,
2590}
2591
2592#[derive(Debug, Clone)]
2593struct TxSendSequencer {
2594 state: Arc<Mutex<TxSendSequencerState>>,
2595 version: Arc<AtomicU64>,
2596 changed: tokio::sync::watch::Sender<u64>,
2597}
2598
2599impl TxSendSequencer {
2600 fn new() -> Self {
2601 let (changed, _) = tokio::sync::watch::channel(0);
2602 Self {
2603 state: Arc::new(Mutex::new(TxSendSequencerState::default())),
2604 version: Arc::new(AtomicU64::new(0)),
2605 changed,
2606 }
2607 }
2608
2609 fn reserve(&self, account_index: i64, api_key_index: u8, nonce: i64) -> TxSendReservation {
2610 let key = TxSendKey {
2611 account_index,
2612 api_key_index,
2613 };
2614 self.state
2615 .lock()
2616 .pending
2617 .entry(key)
2618 .or_default()
2619 .insert(nonce);
2620 self.notify_waiters();
2621
2622 TxSendReservation {
2623 sequencer: self.clone(),
2624 key,
2625 nonce,
2626 released: false,
2627 nonce_guard: None,
2628 connection_epoch: 0,
2629 }
2630 }
2631
2632 async fn wait_for_turn(&self, key: TxSendKey, nonce: i64) {
2633 let mut changed = self.changed.subscribe();
2634
2635 loop {
2636 if self.ready_to_send(key, nonce) {
2637 return;
2638 }
2639
2640 if changed.changed().await.is_err() {
2641 tokio::task::yield_now().await;
2642 }
2643 }
2644 }
2645
2646 fn release(&self, key: TxSendKey, nonce: i64) {
2647 let mut state = self.state.lock();
2648 let should_notify = if let Some(pending) = state.pending.get_mut(&key) {
2649 let removed = pending.remove(&nonce);
2650 if pending.is_empty() {
2651 state.pending.remove(&key);
2652 }
2653 removed
2654 } else {
2655 false
2656 };
2657 drop(state);
2658
2659 if should_notify {
2660 self.notify_waiters();
2661 }
2662 }
2663
2664 fn ready_to_send(&self, key: TxSendKey, nonce: i64) -> bool {
2665 let state = self.state.lock();
2666 state
2667 .pending
2668 .get(&key)
2669 .and_then(|pending| pending.first())
2670 .is_none_or(|first| *first >= nonce)
2671 }
2672
2673 fn notify_waiters(&self) {
2674 let version = self.version.fetch_add(1, Ordering::AcqRel) + 1;
2675 let _ = self.changed.send(version);
2676 }
2677}
2678
2679#[derive(Debug, Default)]
2680struct TxSendSequencerState {
2681 pending: BTreeMap<TxSendKey, BTreeSet<i64>>,
2682}
2683
2684#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
2685struct TxSendKey {
2686 account_index: i64,
2687 api_key_index: u8,
2688}
2689
2690#[derive(Debug)]
2691struct TxSendReservation {
2692 sequencer: TxSendSequencer,
2693 key: TxSendKey,
2694 nonce: i64,
2695 released: bool,
2696 nonce_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
2697 connection_epoch: u64,
2698}
2699
2700impl TxSendReservation {
2701 fn with_nonce_ownership(
2702 mut self,
2703 nonce_guard: tokio::sync::OwnedRwLockReadGuard<()>,
2704 connection_epoch: u64,
2705 ) -> Self {
2706 self.nonce_guard = Some(nonce_guard);
2707 self.connection_epoch = connection_epoch;
2708 self
2709 }
2710
2711 async fn wait_for_turn(&self) {
2712 self.sequencer.wait_for_turn(self.key, self.nonce).await;
2713 }
2714
2715 fn release(&mut self) {
2716 if self.released {
2717 return;
2718 }
2719
2720 self.sequencer.release(self.key, self.nonce);
2721 self.nonce_guard.take();
2722 self.released = true;
2723 }
2724}
2725
2726impl Drop for TxSendReservation {
2727 fn drop(&mut self) {
2728 self.release();
2729 }
2730}
2731
2732#[cfg(test)]
2733async fn wait_for_tx_send_reservations(reservations: &[&TxSendReservation]) {
2734 let Some(first) = reservations.first() else {
2735 return;
2736 };
2737
2738 debug_assert!(
2739 reservations
2740 .iter()
2741 .all(|reservation| reservation.key == first.key),
2742 "batch send reservations must share one nonce stream",
2743 );
2744
2745 let nonce = reservations
2746 .iter()
2747 .map(|reservation| reservation.nonce)
2748 .min()
2749 .expect("reservations is non-empty");
2750 first.sequencer.wait_for_turn(first.key, nonce).await;
2751}
2752
2753struct PreparedCreateOrder {
2754 order: OrderAny,
2755 client_order_index: i64,
2756 tx_info: Box<serde_json::value::RawValue>,
2757 nonce: i64,
2758 api_key_index: u8,
2759 tx_hash: String,
2760 send_reservation: TxSendReservation,
2761}
2762
2763struct CreateOrderPlan {
2764 order: OrderAny,
2765 market_index: i64,
2766 base_amount: i64,
2767 price: u32,
2768 order_type: u8,
2769 time_in_force: u8,
2770 trigger_price: u32,
2771 order_expiry: i64,
2772}
2773
2774struct PreparedCancelOrder {
2775 client_order_id: ClientOrderId,
2776 strategy_id: StrategyId,
2777 instrument_id: InstrumentId,
2778 venue_order_id: Option<VenueOrderId>,
2779 tx_info: Box<serde_json::value::RawValue>,
2780 nonce: i64,
2781 api_key_index: u8,
2782 tx_hash: String,
2783 send_reservation: TxSendReservation,
2784}
2785
2786struct CancelOrderPlan {
2787 client_order_id: ClientOrderId,
2788 strategy_id: StrategyId,
2789 instrument_id: InstrumentId,
2790 venue_order_id: Option<VenueOrderId>,
2791 market_index: i64,
2792 venue_index: i64,
2793}
2794
2795#[derive(Clone)]
2796struct FanoutDispatchContext {
2797 clock: &'static AtomicTime,
2798 chain_id: u32,
2799 integrator_account_index: Option<u64>,
2800 emitter: ExecutionEventEmitter,
2801 credential: Credential,
2802 http_client: LighterHttpClient,
2803 ws_client: LighterWebSocketClient,
2804 tx_rate_limiter: Arc<LighterTxRateLimiter>,
2805 tx_send_sequencer: TxSendSequencer,
2806 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
2807 nonce_ready_connection_epoch: Arc<AtomicU64>,
2808 dispatch: WsDispatchState,
2809 nonce_recovery_inflight: Arc<AtomicBool>,
2810 pending_tasks: TaskSpawner,
2811}
2812
2813impl FanoutDispatchContext {
2814 fn build_tx_context(&self) -> anyhow::Result<ReservedTxContext> {
2815 let connection_epoch = self.ws_client.connection_epoch();
2816 anyhow::ensure!(
2817 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
2818 "Lighter nonce refresh is pending for connection epoch {connection_epoch}",
2819 );
2820 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
2821 .try_read_owned()
2822 .context("Lighter nonce refresh is in progress")?;
2823 anyhow::ensure!(
2824 self.ws_client.connection_epoch() == connection_epoch
2825 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
2826 "Lighter connection or nonce state changed during transaction preparation",
2827 );
2828 let nonce = match self.dispatch.nonce_manager.next_nonce(
2829 self.credential.account_index(),
2830 self.credential.api_key_index(),
2831 ) {
2832 Ok(nonce) => nonce,
2833 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
2834 self.spawn_nonce_window_recovery();
2835 let reason = format!("failed to allocate Lighter nonce: {e}");
2836 return Err(anyhow::Error::new(e).context(reason));
2837 }
2838 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
2839 };
2840
2841 let now_ns = self.clock.get_time_ns().as_u64() as i64;
2842 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
2843 let send_reservation = self
2844 .tx_send_sequencer
2845 .reserve(
2846 self.credential.account_index(),
2847 self.credential.api_key_index(),
2848 nonce,
2849 )
2850 .with_nonce_ownership(nonce_guard, connection_epoch);
2851
2852 Ok(ReservedTxContext {
2853 context: TxContext {
2854 account_index: self.credential.account_index(),
2855 api_key_index: self.credential.api_key_index(),
2856 nonce,
2857 expired_at,
2858 },
2859 send_reservation,
2860 })
2861 }
2862
2863 fn spawn_nonce_window_recovery(&self) {
2864 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
2865 return;
2866 }
2867
2868 let inflight = Arc::clone(&self.nonce_recovery_inflight);
2869 let http_client = self.http_client.clone();
2870 let dispatch = self.dispatch.clone();
2871 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
2872 let account_index = self.credential.account_index();
2873 let api_key_index = self.credential.api_key_index();
2874
2875 let future = async move {
2876 let _refresh_guard = nonce_submission_gate.write().await;
2877 let result = http_client
2878 .get_next_nonce(account_index, api_key_index)
2879 .await;
2880 inflight.store(false, Ordering::Release);
2881
2882 match result {
2883 Ok(response) => {
2884 let _ = dispatch.nonce_manager.sync_from_venue(
2885 account_index,
2886 api_key_index,
2887 response.nonce,
2888 );
2889 log::debug!(
2890 "Resynced Lighter nonce baseline after skip-window exhaustion: \
2891 account_index={account_index}, api_key_index={api_key_index}, \
2892 next_nonce={}",
2893 response.nonce,
2894 );
2895 }
2896 Err(e) => {
2897 log::error!("Failed to resync Lighter nonce after skip-window exhaustion: {e}");
2898 }
2899 }
2900 };
2901
2902 if let Err(e) = self.pending_tasks.spawn(future) {
2903 self.nonce_recovery_inflight.store(false, Ordering::Release);
2904 log::warn!("Skipping Lighter nonce recovery after shutdown began: {e}");
2905 }
2906 }
2907
2908 fn sign_create_order(&self, plan: CreateOrderPlan) -> anyhow::Result<PreparedCreateOrder> {
2909 let cloid = plan.order.client_order_id();
2910 let client_order_index = self.dispatch.register_create_identity(&plan.order)?;
2911
2912 let ReservedTxContext {
2913 context,
2914 send_reservation,
2915 } = match self.build_tx_context() {
2916 Ok(context) => context,
2917 Err(e) => {
2918 self.dispatch.forget_cloid(client_order_index);
2919 self.dispatch.forget_order_identity(&cloid);
2920 return Err(e);
2921 }
2922 };
2923
2924 let nonce = context.nonce;
2925 let api_key_index = context.api_key_index;
2926 let mut rollback_guard = TxDispatchGuard::new(
2927 self.dispatch.clone(),
2928 &self.credential,
2929 Some(client_order_index),
2930 nonce,
2931 )
2932 .with_order_identity(cloid);
2933
2934 let tx = CreateOrderTxInfo {
2935 context,
2936 order: OrderInfo {
2937 market_index: plan.market_index,
2938 client_order_index,
2939 base_amount: plan.base_amount,
2940 price: plan.price,
2941 is_ask: matches!(plan.order.order_side(), OrderSide::Sell),
2942 order_type: plan.order_type,
2943 time_in_force: plan.time_in_force,
2944 reduce_only: plan.order.is_reduce_only(),
2945 trigger_price: plan.trigger_price,
2946 order_expiry: plan.order_expiry,
2947 },
2948 attributes: integrator_attributes(self.integrator_account_index),
2949 };
2950
2951 let signed = sign_tx(
2952 &tx,
2953 self.chain_id,
2954 &self.credential.private_key()?,
2955 fresh_k(),
2956 );
2957
2958 let tx_info =
2959 serde_json::value::RawValue::from_string(TxInfoJson::create_order(&tx, &signed))
2960 .context("failed to wrap signed Lighter tx_info JSON")?;
2961
2962 rollback_guard.disarm();
2963
2964 Ok(PreparedCreateOrder {
2965 order: plan.order,
2966 client_order_index,
2967 tx_info,
2968 nonce,
2969 api_key_index,
2970 tx_hash: signed.tx_hash_hex(),
2971 send_reservation,
2972 })
2973 }
2974
2975 async fn send_create_order(&self, prepared: PreparedCreateOrder) {
2976 let PreparedCreateOrder {
2977 order,
2978 client_order_index,
2979 tx_info,
2980 nonce,
2981 api_key_index,
2982 tx_hash,
2983 mut send_reservation,
2984 } = prepared;
2985 let connection_epoch = send_reservation.connection_epoch;
2986 let client_order_id = order.client_order_id();
2987
2988 self.emitter.emit_order_submitted(&order);
2989 log::debug!("Lighter submit_order: queueing CreateOrder tx for {client_order_id}");
2990 send_reservation.wait_for_turn().await;
2991 await_tx_quota(&self.tx_rate_limiter).await;
2992 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
2993 batch_id: None,
2994 connection_epoch,
2995 kind: PendingSendTxKind::Create {
2996 order: Box::new(order.clone()),
2997 client_order_index,
2998 },
2999 submitted_at: self.clock.get_time_ns(),
3000 nonce,
3001 api_key_index,
3002 tx_hash,
3003 });
3004
3005 if let Err(e) = self
3006 .ws_client
3007 .send_tx_on_connection(LighterTxType::CreateOrder as u8, tx_info, connection_epoch)
3008 .await
3009 {
3010 let failure = classify_lighter_ws_command_failure("submit_order", &e);
3011 let reason = command_failure_reason(&failure);
3012 if matches!(&failure, CommandFailure::Ambiguous(_)) {
3013 log::warn!(
3014 "Lighter submit_order dispatch outcome unknown for {client_order_id}: {reason}; \
3015 retaining pending state for venue reconciliation; diagnostic={e:?}",
3016 );
3017 } else {
3018 log::error!("{reason} for {client_order_id}; diagnostic={e:?}");
3019 self.dispatch
3020 .remove_pending_sendtx_by_nonce(connection_epoch, nonce);
3021 rollback_tx_dispatch_create(
3022 &self.dispatch,
3023 &self.credential,
3024 Some(client_order_index),
3025 &client_order_id,
3026 nonce,
3027 );
3028 self.emitter
3029 .emit_order_rejected(&order, reason, self.clock.get_time_ns(), false);
3030 }
3031 }
3032 send_reservation.release();
3033 }
3034
3035 fn sign_cancel_order(&self, plan: &CancelOrderPlan) -> anyhow::Result<PreparedCancelOrder> {
3036 let ReservedTxContext {
3037 context,
3038 send_reservation,
3039 } = self.build_tx_context()?;
3040
3041 let nonce = context.nonce;
3042 let api_key_index = context.api_key_index;
3043 let mut rollback_guard =
3044 TxDispatchGuard::new(self.dispatch.clone(), &self.credential, None, nonce);
3045
3046 let tx = CancelOrderTxInfo {
3047 context,
3048 market_index: plan.market_index,
3049 index: plan.venue_index,
3050 skip_nonce: 0,
3051 };
3052
3053 let signed = sign_tx(
3054 &tx,
3055 self.chain_id,
3056 &self.credential.private_key()?,
3057 fresh_k(),
3058 );
3059
3060 let tx_info =
3061 serde_json::value::RawValue::from_string(TxInfoJson::cancel_order(&tx, &signed))
3062 .context("failed to wrap signed Lighter cancel tx_info JSON")?;
3063
3064 rollback_guard.disarm();
3065
3066 Ok(PreparedCancelOrder {
3067 client_order_id: plan.client_order_id,
3068 strategy_id: plan.strategy_id,
3069 instrument_id: plan.instrument_id,
3070 venue_order_id: plan.venue_order_id,
3071 tx_info,
3072 nonce,
3073 api_key_index,
3074 tx_hash: signed.tx_hash_hex(),
3075 send_reservation,
3076 })
3077 }
3078
3079 async fn send_cancel_order(
3080 &self,
3081 mut prepared: PreparedCancelOrder,
3082 emit_cancel_rejected: bool,
3083 ) {
3084 let connection_epoch = prepared.send_reservation.connection_epoch;
3085 prepared.send_reservation.wait_for_turn().await;
3086 await_tx_quota(&self.tx_rate_limiter).await;
3087 self.enqueue_cancel_order(&prepared, emit_cancel_rejected, None);
3088
3089 if let Err(e) = self
3090 .ws_client
3091 .send_tx_on_connection(
3092 LighterTxType::CancelOrder as u8,
3093 prepared.tx_info.clone(),
3094 connection_epoch,
3095 )
3096 .await
3097 {
3098 self.handle_cancel_send_failure(&prepared, emit_cancel_rejected, &e);
3099 }
3100
3101 prepared.send_reservation.release();
3102 }
3103
3104 async fn send_cancel_batch(&self, mut batch: Vec<(PreparedCancelOrder, bool)>) {
3105 let mut batch_ids = Vec::new();
3106
3107 while !batch.is_empty() {
3108 let count = batch
3111 .windows(2)
3112 .position(|pair| {
3113 pair[1].0.nonce != pair[0].0.nonce + 1
3114 || pair[1].0.send_reservation.connection_epoch
3115 != pair[0].0.send_reservation.connection_epoch
3116 })
3117 .map_or(batch.len(), |index| index + 1);
3118
3119 let mut group: Vec<_> = batch.drain(..count).collect();
3120 let connection_epoch = group[0].0.send_reservation.connection_epoch;
3121 let id = format!("{CANCEL_BATCH_ID_PREFIX}{}", group[0].0.tx_hash);
3122 group[0].0.send_reservation.wait_for_turn().await;
3123 await_tx_quota(&self.tx_rate_limiter).await;
3124
3125 let data = crate::websocket::messages::LighterWsSendTxBatch {
3126 id: id.clone(),
3127 tx_types: serde_json::to_string(&vec![
3128 LighterTxType::CancelOrder as u8;
3129 group.len()
3130 ])
3131 .expect("transaction types serialize"),
3132 tx_infos: serde_json::to_string(
3133 &group
3134 .iter()
3135 .map(|(prepared, _)| prepared.tx_info.get())
3136 .collect::<Vec<_>>(),
3137 )
3138 .expect("transaction strings serialize"),
3139 };
3140
3141 for (prepared, emit) in &group {
3142 self.enqueue_cancel_order(prepared, *emit, Some(&id));
3143 }
3144
3145 if let Err(e) = self
3146 .ws_client
3147 .send_tx_batch_on_connection(data, connection_epoch)
3148 .await
3149 {
3150 for (prepared, emit) in group.iter().rev() {
3151 self.handle_cancel_send_failure(prepared, *emit, &e);
3152 }
3153 }
3154
3155 for (prepared, _) in &mut group {
3156 prepared.send_reservation.release();
3157 }
3158
3159 batch_ids.push(id);
3160 }
3161
3162 let wait = async {
3164 while self.dispatch.pending_sendtx.lock().iter().any(|pending| {
3165 pending
3166 .batch_id
3167 .as_ref()
3168 .is_some_and(|id| batch_ids.contains(id))
3169 }) {
3170 tokio::time::sleep(Duration::from_millis(10)).await;
3171 }
3172 };
3173
3174 if tokio::time::timeout(BATCH_ACK_TIMEOUT, wait).await.is_err() {
3175 log::warn!("Lighter batch acknowledgement timed out: ids={batch_ids:?}");
3176 }
3177 }
3178
3179 fn enqueue_cancel_order(
3180 &self,
3181 prepared: &PreparedCancelOrder,
3182 emit_cancel_rejected: bool,
3183 batch_id: Option<&str>,
3184 ) {
3185 self.dispatch.set_pending_cancel_nonce(
3186 &prepared.client_order_id,
3187 prepared.send_reservation.connection_epoch,
3188 prepared.nonce,
3189 );
3190 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
3191 batch_id: batch_id.map(str::to_string),
3192 connection_epoch: prepared.send_reservation.connection_epoch,
3193 kind: if emit_cancel_rejected {
3194 PendingSendTxKind::Cancel {
3195 strategy_id: prepared.strategy_id,
3196 instrument_id: prepared.instrument_id,
3197 client_order_id: prepared.client_order_id,
3198 venue_order_id: prepared.venue_order_id,
3199 }
3200 } else {
3201 PendingSendTxKind::Other
3202 },
3203 submitted_at: self.clock.get_time_ns(),
3204 nonce: prepared.nonce,
3205 api_key_index: prepared.api_key_index,
3206 tx_hash: prepared.tx_hash.clone(),
3207 });
3208 }
3209
3210 fn handle_cancel_send_failure(
3211 &self,
3212 prepared: &PreparedCancelOrder,
3213 emit_cancel_rejected: bool,
3214 error: &LighterWsError,
3215 ) {
3216 let failure = classify_lighter_ws_command_failure("cancel_order", error);
3217 let reason = command_failure_reason(&failure);
3218
3219 if matches!(failure, CommandFailure::Ambiguous(_)) {
3220 log::warn!(
3221 "Lighter cancel_order outcome unknown for {}: {reason}; retaining pending state for venue reconciliation",
3222 prepared.client_order_id
3223 );
3224 return;
3225 }
3226
3227 log::error!(
3228 "{reason} for {}; diagnostic={error:?}",
3229 prepared.client_order_id
3230 );
3231
3232 self.dispatch.remove_pending_sendtx_by_nonce(
3233 prepared.send_reservation.connection_epoch,
3234 prepared.nonce,
3235 );
3236 self.dispatch
3237 .clear_pending_order_action_if(&prepared.client_order_id, PendingOrderAction::Cancel);
3238 rollback_tx_dispatch(&self.dispatch, &self.credential, None, prepared.nonce);
3239
3240 if emit_cancel_rejected {
3241 self.emitter.emit_order_cancel_rejected_event(
3242 prepared.strategy_id,
3243 prepared.instrument_id,
3244 prepared.client_order_id,
3245 prepared.venue_order_id,
3246 reason,
3247 self.clock.get_time_ns(),
3248 );
3249 } else {
3250 log::warn!(
3251 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
3252 prepared.client_order_id
3253 );
3254 }
3255 }
3256}
3257
3258struct PreparedModifyOrder {
3259 client_order_id: ClientOrderId,
3260 strategy_id: StrategyId,
3261 instrument_id: InstrumentId,
3262 venue_order_id: Option<VenueOrderId>,
3263 tx_info: Box<serde_json::value::RawValue>,
3264 nonce: i64,
3265 api_key_index: u8,
3266 tx_hash: String,
3267 send_reservation: TxSendReservation,
3268}
3269
3270struct PreparedIntegratorApproval {
3271 tx_info: String,
3272 nonce: i64,
3273 api_key_index: u8,
3274 approval_expiry: i64,
3275 send_reservation: TxSendReservation,
3276}
3277
3278#[derive(Debug, PartialEq, Eq)]
3282enum TierCrossCheck {
3283 AboveTier { documented: u32 },
3284 RaiseHint { documented: u32 },
3285}
3286
3287fn tier_quota_report(
3292 tier: LighterAccountTier,
3293 rest_quota_per_min: Option<u32>,
3294 standard_rest: u32,
3295) -> (u32, Option<TierCrossCheck>) {
3296 let configured = rest_quota_per_min.filter(|&n| n > 0);
3297 let active_rest = configured.unwrap_or(standard_rest);
3298 let cross_check = match (tier.documented_rest_quota_per_min(), configured) {
3299 (Some(documented), Some(configured)) if configured > documented => {
3300 Some(TierCrossCheck::AboveTier { documented })
3301 }
3302 (Some(documented), None) if documented > standard_rest => {
3303 Some(TierCrossCheck::RaiseHint { documented })
3304 }
3305 _ => None,
3306 };
3307 (active_rest, cross_check)
3308}
3309
3310struct TxDispatchGuard {
3311 dispatch: WsDispatchState,
3312 account_index: i64,
3313 api_key_index: u8,
3314 client_order_index: Option<i64>,
3315 client_order_id: Option<ClientOrderId>,
3316 nonce: i64,
3317 armed: bool,
3318}
3319
3320impl TxDispatchGuard {
3321 fn new(
3322 dispatch: WsDispatchState,
3323 credential: &Credential,
3324 client_order_index: Option<i64>,
3325 nonce: i64,
3326 ) -> Self {
3327 Self {
3328 dispatch,
3329 account_index: credential.account_index(),
3330 api_key_index: credential.api_key_index(),
3331 client_order_index,
3332 client_order_id: None,
3333 nonce,
3334 armed: true,
3335 }
3336 }
3337
3338 fn with_order_identity(mut self, client_order_id: ClientOrderId) -> Self {
3339 self.client_order_id = Some(client_order_id);
3340 self
3341 }
3342
3343 fn disarm(&mut self) {
3344 self.armed = false;
3345 }
3346}
3347
3348impl Drop for TxDispatchGuard {
3349 fn drop(&mut self) {
3350 if self.armed {
3351 rollback_tx_dispatch_indices(
3352 &self.dispatch,
3353 self.account_index,
3354 self.api_key_index,
3355 self.client_order_index,
3356 self.client_order_id.as_ref(),
3357 self.nonce,
3358 );
3359 }
3360 }
3361}
3362
3363fn handle_send_tx_ack_for_connection(
3368 dispatch: &WsDispatchState,
3369 account_index: Option<i64>,
3370 connection_epoch: u64,
3371 code: i64,
3372 tx_hash: Option<&str>,
3373) -> Option<PendingSendTx> {
3374 let popped = match tx_hash {
3375 Some(hash) => {
3376 let matched = dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash);
3377 if matched.is_none() {
3378 log::warn!("Lighter sendTx ack unmatched: tx_hash={hash} code={code}");
3379 }
3380 matched
3381 }
3382 None => dispatch.pop_pending_sendtx_if_only(connection_epoch),
3383 };
3384
3385 if let (Some(pending), Some(account_index)) = (&popped, account_index) {
3386 let _ =
3387 dispatch
3388 .nonce_manager
3389 .ack_success(account_index, pending.api_key_index, pending.nonce);
3390 }
3391
3392 log::debug!(
3393 "Lighter sendTx ack: code={code} tx_hash={tx_hash:?} popped_nonce={:?}",
3394 popped.as_ref().map(|p| p.nonce),
3395 );
3396
3397 popped
3398}
3399
3400#[expect(
3401 clippy::too_many_arguments,
3402 reason = "batch response shares the single-transaction rejection sink"
3403)]
3404fn handle_send_tx_batch_result(
3405 dispatch: &WsDispatchState,
3406 emitter: &ExecutionEventEmitter,
3407 account_index: Option<i64>,
3408 connection_epoch: u64,
3409 now: UnixNanos,
3410 id: &str,
3411 code: i64,
3412 message: &str,
3413 tx_hashes: &[String],
3414) -> (Vec<PendingSendTx>, bool) {
3415 let mut pending: Vec<_> = dispatch
3416 .pending_sendtx
3417 .lock()
3418 .iter()
3419 .filter(|pending| {
3420 pending.connection_epoch == connection_epoch && pending.batch_id.as_deref() == Some(id)
3421 })
3422 .cloned()
3423 .collect();
3424
3425 if pending.is_empty() {
3426 log::warn!("Ignoring unmatched Lighter batch response: id={id}");
3427 return (Vec::new(), false);
3428 }
3429
3430 if code == 200 {
3431 let acked = pending
3432 .iter()
3433 .filter(|pending| tx_hashes.contains(&pending.tx_hash))
3434 .filter_map(|pending| {
3435 handle_send_tx_ack_for_connection(
3436 dispatch,
3437 account_index,
3438 connection_epoch,
3439 code,
3440 Some(&pending.tx_hash),
3441 )
3442 })
3443 .collect();
3444
3445 if pending
3446 .iter()
3447 .any(|pending| !tx_hashes.contains(&pending.tx_hash))
3448 {
3449 log::warn!(
3450 "Incomplete Lighter batch acknowledgement: id={id}; retaining unmatched transactions for reconciliation"
3451 );
3452 }
3453
3454 return (acked, false);
3455 }
3456
3457 pending.sort_unstable_by_key(|pending| std::cmp::Reverse(pending.nonce));
3459 let mut resync = false;
3460 for pending in pending {
3461 resync |= handle_send_tx_rejection_for_connection(
3462 dispatch,
3463 emitter,
3464 account_index,
3465 connection_epoch,
3466 now,
3467 SendTxRejectionSource::Ack,
3468 Some(code),
3469 message,
3470 Some(&pending.tx_hash),
3471 );
3472 }
3473
3474 (Vec::new(), resync)
3475}
3476
3477fn spawn_acked_order_probe(pending: &PendingSendTx, context: AckedOrderProbeContext) {
3478 let Some(probe) = AckedOrderProbe::from_pending(pending) else {
3479 return;
3480 };
3481
3482 let pending_tasks = context.pending_tasks.clone();
3483 let batch_cancel = (pending.batch_id.is_some()
3484 && matches!(pending.kind, PendingSendTxKind::Cancel { .. }))
3485 .then(|| pending.clone());
3486
3487 let future = async move {
3488 tokio::select! {
3489 () = context.cancellation_token.cancelled() => return,
3490 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
3491 }
3492
3493 if let Some(pending) = batch_cancel {
3494 if let Err(e) = probe_acked_cancel(&pending, &context).await {
3495 log::warn!("Lighter acknowledged batch cancel probe failed: {e}");
3496 }
3497
3498 return;
3499 }
3500
3501 if let Err(e) = probe_acked_order(probe, &context).await {
3502 log::warn!("Lighter acknowledged order probe failed: {e:?}");
3503 }
3504 };
3505
3506 if let Err(e) = pending_tasks.spawn(future) {
3507 log::debug!("Skipping Lighter acknowledged-order probe during shutdown: {e}");
3508 }
3509}
3510
3511#[derive(Clone)]
3512struct AckedOrderProbeContext {
3513 http_client: LighterHttpClient,
3514 registry: Arc<MarketRegistry>,
3515 credential: Credential,
3516 dispatch: WsDispatchState,
3517 account_id: AccountId,
3518 clock: &'static AtomicTime,
3519 emitter: ExecutionEventEmitter,
3520 connection_epoch: Arc<AtomicU64>,
3521 cancellation_token: CancellationToken,
3522 pending_tasks: TaskSpawner,
3523}
3524
3525async fn probe_acked_cancel(
3526 pending: &PendingSendTx,
3527 context: &AckedOrderProbeContext,
3528) -> anyhow::Result<()> {
3529 let PendingSendTxKind::Cancel {
3530 strategy_id,
3531 instrument_id,
3532 client_order_id,
3533 venue_order_id,
3534 } = pending.kind
3535 else {
3536 return Ok(());
3537 };
3538
3539 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
3540 if context.connection_epoch.load(Ordering::Acquire) != pending.connection_epoch
3541 || context.dispatch.pending_cancel_nonce(&client_order_id)
3542 != Some((pending.connection_epoch, pending.nonce))
3543 {
3544 return Ok(());
3545 }
3546
3547 match context.http_client.get_tx(pending.tx_hash.clone()).await {
3548 Ok(tx) => {
3549 anyhow::ensure!(
3550 tx_hash_matches(&tx.hash, &pending.tx_hash)
3551 && tx.tx_type == LighterTxType::CancelOrder as u8
3552 && tx.account_index == context.credential.account_index()
3553 && tx.api_key_index == pending.api_key_index
3554 && tx.nonce == pending.nonce,
3555 "Lighter transaction lookup did not match acknowledged cancel identity"
3556 );
3557 let event = parse_acked_tx_event(&tx.event_info)?;
3558
3559 if context.connection_epoch.load(Ordering::Acquire) != pending.connection_epoch {
3560 return Ok(());
3561 }
3562
3563 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
3564 if context.dispatch.clear_pending_cancel_if_nonce(
3565 &client_order_id,
3566 pending.connection_epoch,
3567 pending.nonce,
3568 ) {
3569 context.emitter.emit_order_cancel_rejected_event(
3570 strategy_id,
3571 instrument_id,
3572 client_order_id,
3573 venue_order_id,
3574 &format!("Lighter cancel transaction failed: {}", event.app_error),
3575 context.clock.get_time_ns(),
3576 );
3577 }
3578
3579 return Ok(());
3581 }
3582
3583 if tx.status == LighterTxStatus::Executed {
3584 if let Some(report) = lookup_order_status_report(
3585 &context.http_client,
3586 &context.registry,
3587 &context.credential,
3588 context.account_id,
3589 Some(instrument_id),
3590 Some(&client_order_id),
3591 venue_order_id.as_ref(),
3592 &context.dispatch,
3593 context.clock,
3594 )
3595 .await?
3596 && context.connection_epoch.load(Ordering::Acquire)
3597 == pending.connection_epoch
3598 && context.dispatch.pending_cancel_nonce(&client_order_id)
3599 == Some((pending.connection_epoch, pending.nonce))
3600 {
3601 context.emitter.send_order_status_report(report);
3602 }
3603
3604 return Ok(());
3605 }
3606 }
3607 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
3608 Err(e) => log::warn!("Lighter batch cancel transaction lookup failed: {e}"),
3609 }
3610
3611 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
3612 tokio::select! {
3613 () = context.cancellation_token.cancelled() => return Ok(()),
3614 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
3615 }
3616 }
3617 }
3618
3619 log::warn!(
3620 "Lighter batch cancel outcome unresolved for {client_order_id}; retaining pending action for reconciliation"
3621 );
3622 Ok(())
3623}
3624
3625async fn probe_acked_order(
3626 probe: AckedOrderProbe,
3627 context: &AckedOrderProbeContext,
3628) -> anyhow::Result<()> {
3629 if matches!(&probe, AckedOrderProbe::Create { .. }) {
3630 return probe_acked_create(probe, context).await;
3631 }
3632
3633 let report = lookup_order_status_report(
3634 &context.http_client,
3635 &context.registry,
3636 &context.credential,
3637 context.account_id,
3638 Some(probe.instrument_id()),
3639 Some(&probe.client_order_id()),
3640 probe.venue_order_id().as_ref(),
3641 &context.dispatch,
3642 context.clock,
3643 )
3644 .await?;
3645
3646 if let Some(report) = &report {
3647 context.dispatch.seed_accepted_from_report(report);
3648 }
3649
3650 warn_if_acked_order_missing(&probe, report.is_some());
3651 Ok(())
3652}
3653
3654async fn probe_acked_create(
3655 probe: AckedOrderProbe,
3656 context: &AckedOrderProbeContext,
3657) -> anyhow::Result<()> {
3658 let AckedOrderProbe::Create {
3659 order,
3660 client_order_index,
3661 connection_epoch,
3662 nonce,
3663 api_key_index,
3664 tx_hash,
3665 } = probe
3666 else {
3667 unreachable!("create probe called with non-create transaction")
3668 };
3669 let client_order_id = order.client_order_id();
3670 let mut transaction_executed = false;
3671
3672 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
3673 if context.connection_epoch.load(Ordering::Acquire) != connection_epoch {
3674 log::warn!(
3675 "Lighter acknowledged create outcome unresolved after reconnect for {client_order_id}; retaining identity for reconciliation",
3676 );
3677 return Ok(());
3678 }
3679
3680 if !context.dispatch.create_submission_is_pending(
3681 &client_order_id,
3682 client_order_index,
3683 nonce,
3684 ) {
3685 return Ok(());
3686 }
3687
3688 let report = match lookup_create_order_status_report(
3689 &context.http_client,
3690 &context.registry,
3691 &context.credential,
3692 context.account_id,
3693 order.instrument_id(),
3694 client_order_id,
3695 client_order_index,
3696 nonce,
3697 &context.dispatch,
3698 context.clock,
3699 )
3700 .await
3701 {
3702 Ok(report) => report,
3703 Err(e) => {
3704 log::warn!(
3705 "Lighter acknowledged create order lookup failed for {client_order_id} on attempt {attempt}: {e:?}",
3706 );
3707 None
3708 }
3709 };
3710
3711 if let Some(report) = report {
3712 if context.dispatch.observe_create_submission(
3713 &client_order_id,
3714 client_order_index,
3715 nonce,
3716 report.venue_order_id,
3717 ) {
3718 context.dispatch.seed_accepted_from_report(&report);
3719 context.emitter.send_order_status_report(report);
3720 }
3721 return Ok(());
3722 }
3723
3724 match context.http_client.get_tx(tx_hash.clone()).await {
3725 Ok(tx) => {
3726 validate_acked_create_tx(
3727 &tx,
3728 context.credential.account_index(),
3729 api_key_index,
3730 client_order_index,
3731 nonce,
3732 &tx_hash,
3733 )?;
3734
3735 let event = parse_acked_tx_event(&tx.event_info).unwrap_or_else(|e| {
3736 log::warn!(
3737 "Lighter create transaction carried invalid event_info for {client_order_id}: {e}",
3738 );
3739 AckedTxEvent::default()
3740 });
3741
3742 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
3743 let detail = if event.app_error.is_empty() {
3744 "transaction failed without an application error".to_string()
3745 } else {
3746 event.app_error
3747 };
3748 let reason = format!(
3749 "Lighter sequencer rejected acknowledged create transaction {tx_hash}: {detail}",
3750 );
3751 reject_create_order(
3752 &context.dispatch,
3753 &context.emitter,
3754 &order,
3755 client_order_index,
3756 nonce,
3757 &reason,
3758 context.clock.get_time_ns(),
3759 lighter_reason_indicates_post_only_rejection(&detail),
3760 Some((&context.connection_epoch, connection_epoch)),
3761 );
3762 return Ok(());
3763 }
3764 transaction_executed |= tx.status == LighterTxStatus::Executed;
3765 }
3766 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
3767 Err(e) => {
3768 log::warn!(
3769 "Lighter acknowledged create transaction lookup failed for {client_order_id} on attempt {attempt}: {e}",
3770 );
3771 }
3772 }
3773
3774 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
3775 tokio::select! {
3776 () = context.cancellation_token.cancelled() => return Ok(()),
3777 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
3778 }
3779 }
3780 }
3781
3782 if transaction_executed {
3783 let _ =
3784 context
3785 .dispatch
3786 .confirm_create_submission(&client_order_id, client_order_index, nonce);
3787 log::warn!(
3788 "Lighter acknowledged create transaction executed without a queryable order for {client_order_id}; retaining identity for reconciliation",
3789 );
3790 } else {
3791 log::warn!(
3792 "Lighter acknowledged create outcome unresolved after {ACKED_ORDER_PROBE_ATTEMPTS} transaction lookups for {client_order_id}; retaining identity for reconciliation",
3793 );
3794 }
3795 Ok(())
3796}
3797
3798#[derive(Deserialize)]
3799struct AckedCreateTxInfo {
3800 #[serde(rename = "ClientOrderIndex")]
3801 client_order_index: i64,
3802}
3803
3804#[derive(Default, Deserialize)]
3805struct AckedTxEvent {
3806 #[serde(default, rename = "ae")]
3807 app_error: String,
3808}
3809
3810fn validate_acked_create_tx(
3811 tx: &crate::http::models::LighterTx,
3812 account_index: i64,
3813 api_key_index: u8,
3814 client_order_index: i64,
3815 nonce: i64,
3816 tx_hash: &str,
3817) -> anyhow::Result<()> {
3818 anyhow::ensure!(
3819 tx_hash_matches(&tx.hash, tx_hash)
3820 && tx.tx_type == LighterTxType::CreateOrder as u8
3821 && tx.account_index == account_index
3822 && tx.api_key_index == api_key_index
3823 && tx.nonce == nonce,
3824 "Lighter transaction lookup did not match acknowledged create identity",
3825 );
3826 let info: AckedCreateTxInfo = serde_json::from_str(&tx.info)
3827 .context("failed to parse Lighter create transaction info")?;
3828 anyhow::ensure!(
3829 info.client_order_index == client_order_index,
3830 "Lighter transaction lookup returned client_order_index {} for acknowledged create {client_order_index}",
3831 info.client_order_index,
3832 );
3833 Ok(())
3834}
3835
3836fn tx_hash_matches(left: &str, right: &str) -> bool {
3837 let left = left
3838 .strip_prefix("0x")
3839 .or_else(|| left.strip_prefix("0X"))
3840 .unwrap_or(left);
3841 let right = right
3842 .strip_prefix("0x")
3843 .or_else(|| right.strip_prefix("0X"))
3844 .unwrap_or(right);
3845 left.eq_ignore_ascii_case(right)
3846}
3847
3848fn parse_acked_tx_event(event_info: &str) -> anyhow::Result<AckedTxEvent> {
3849 if event_info.trim().is_empty() {
3850 return Ok(AckedTxEvent::default());
3851 }
3852 serde_json::from_str(event_info).context("failed to parse Lighter create transaction event")
3853}
3854
3855fn warn_if_acked_order_missing(probe: &AckedOrderProbe, order_found: bool) {
3856 if order_found {
3857 return;
3858 }
3859
3860 match probe {
3861 AckedOrderProbe::Create { .. } => {}
3862 AckedOrderProbe::Cancel {
3863 client_order_id, ..
3864 } => {
3865 log::warn!(
3866 "Lighter cancel_order outcome unresolved: order not found after venue ack for {client_order_id}",
3867 );
3868 }
3869 AckedOrderProbe::Modify {
3870 client_order_id, ..
3871 } => {
3872 log::warn!(
3873 "Lighter modify_order outcome unresolved: order not found after venue ack for {client_order_id}",
3874 );
3875 }
3876 }
3877}
3878
3879#[derive(Debug, Clone)]
3880enum AckedOrderProbe {
3881 Create {
3882 order: Box<OrderAny>,
3883 client_order_index: i64,
3884 connection_epoch: u64,
3885 nonce: i64,
3886 api_key_index: u8,
3887 tx_hash: String,
3888 },
3889 Cancel {
3890 instrument_id: InstrumentId,
3891 client_order_id: ClientOrderId,
3892 venue_order_id: Option<VenueOrderId>,
3893 },
3894 Modify {
3895 instrument_id: InstrumentId,
3896 client_order_id: ClientOrderId,
3897 venue_order_id: Option<VenueOrderId>,
3898 },
3899}
3900
3901impl AckedOrderProbe {
3902 fn from_pending(pending: &PendingSendTx) -> Option<Self> {
3903 match &pending.kind {
3904 PendingSendTxKind::Create {
3905 order,
3906 client_order_index,
3907 } => Some(Self::Create {
3908 order: order.clone(),
3909 client_order_index: *client_order_index,
3910 connection_epoch: pending.connection_epoch,
3911 nonce: pending.nonce,
3912 api_key_index: pending.api_key_index,
3913 tx_hash: pending.tx_hash.clone(),
3914 }),
3915 PendingSendTxKind::Cancel {
3916 instrument_id,
3917 client_order_id,
3918 venue_order_id,
3919 ..
3920 } => Some(Self::Cancel {
3921 instrument_id: *instrument_id,
3922 client_order_id: *client_order_id,
3923 venue_order_id: *venue_order_id,
3924 }),
3925 PendingSendTxKind::Modify {
3926 instrument_id,
3927 client_order_id,
3928 venue_order_id,
3929 ..
3930 } => Some(Self::Modify {
3931 instrument_id: *instrument_id,
3932 client_order_id: *client_order_id,
3933 venue_order_id: *venue_order_id,
3934 }),
3935 PendingSendTxKind::Other => None,
3936 }
3937 }
3938
3939 fn instrument_id(&self) -> InstrumentId {
3940 match self {
3941 Self::Create { order, .. } => order.instrument_id(),
3942 Self::Cancel { instrument_id, .. } | Self::Modify { instrument_id, .. } => {
3943 *instrument_id
3944 }
3945 }
3946 }
3947
3948 fn client_order_id(&self) -> ClientOrderId {
3949 match self {
3950 Self::Create { order, .. } => order.client_order_id(),
3951 Self::Cancel {
3952 client_order_id, ..
3953 }
3954 | Self::Modify {
3955 client_order_id, ..
3956 } => *client_order_id,
3957 }
3958 }
3959
3960 fn venue_order_id(&self) -> Option<VenueOrderId> {
3961 match self {
3962 Self::Create { .. } => None,
3963 Self::Cancel { venue_order_id, .. } | Self::Modify { venue_order_id, .. } => {
3964 *venue_order_id
3965 }
3966 }
3967 }
3968}
3969
3970#[expect(
3979 clippy::too_many_arguments,
3980 reason = "shared terminal create transition keeps cleanup and event attribution together"
3981)]
3982fn reject_create_order(
3983 dispatch: &WsDispatchState,
3984 emitter: &ExecutionEventEmitter,
3985 order: &OrderAny,
3986 client_order_index: i64,
3987 nonce: i64,
3988 reason: &str,
3989 now: UnixNanos,
3990 due_post_only: bool,
3991 connection_epoch: Option<(&AtomicU64, u64)>,
3992) -> bool {
3993 let cloid = order.client_order_id();
3994 if !dispatch.reject_create_submission(&cloid, client_order_index, nonce, connection_epoch) {
3995 log::warn!(
3996 "Ignored stale Lighter create rejection for cloid={cloid} client_order_index={client_order_index} nonce={nonce}",
3997 );
3998 return false;
3999 }
4000 emitter.emit_order_rejected(order, reason, now, due_post_only);
4001 true
4002}
4003
4004#[expect(
4005 clippy::too_many_arguments,
4006 reason = "consumer-loop sink that flattens one SendTxRejected message without a wrapper struct"
4007)]
4008fn handle_send_tx_rejection_for_connection(
4009 dispatch: &WsDispatchState,
4010 emitter: &ExecutionEventEmitter,
4011 account_index: Option<i64>,
4012 connection_epoch: u64,
4013 now: UnixNanos,
4014 source: SendTxRejectionSource,
4015 code: Option<i64>,
4016 message: &str,
4017 tx_hash: Option<&str>,
4018) -> bool {
4019 let needs_nonce_resync = code == Some(LIGHTER_ERROR_CODE_INVALID_NONCE);
4020 let failure = venue_rejection_failure(code, message);
4021 let reason = command_failure_reason(&failure);
4022
4023 let pending = match tx_hash {
4024 Some(hash) => dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash),
4025 None => match source {
4026 SendTxRejectionSource::Ack => dispatch.pop_pending_sendtx_if_only(connection_epoch),
4027 SendTxRejectionSource::BareError => dispatch.pop_pending_sendtx_if_only_within(
4028 connection_epoch,
4029 now,
4030 SENDTX_BARE_ERROR_WINDOW_MS,
4031 ),
4032 },
4033 };
4034 let Some(pending) = pending else {
4035 log::warn!(
4036 "Lighter sendTx rejection unattributed (source={source:?} code={code:?}): {message:?}",
4037 );
4038 return needs_nonce_resync;
4039 };
4040
4041 match &pending.kind {
4042 PendingSendTxKind::Create {
4043 order,
4044 client_order_index,
4045 } => {
4046 let cloid = order.client_order_id();
4047 log::error!(
4048 "{reason} attributed to cloid={cloid} nonce={} api_key_index={}; diagnostic_code={code:?} diagnostic_message={message:?}",
4049 pending.nonce,
4050 pending.api_key_index,
4051 );
4052
4053 if let Some(account_index) = account_index {
4054 let _ = dispatch.nonce_manager.ack_failure_if_latest(
4055 account_index,
4056 pending.api_key_index,
4057 pending.nonce,
4058 );
4059 }
4060 reject_create_order(
4061 dispatch,
4062 emitter,
4063 order,
4064 *client_order_index,
4065 pending.nonce,
4066 reason,
4067 now,
4068 lighter_reason_indicates_post_only_rejection(message),
4069 None,
4070 );
4071 }
4072 PendingSendTxKind::Cancel {
4073 strategy_id,
4074 instrument_id,
4075 client_order_id,
4076 venue_order_id,
4077 } => {
4078 log::error!(
4079 "{reason} attributed to cancel cloid={client_order_id} nonce={} api_key_index={}; diagnostic_code={code:?} diagnostic_message={message:?}",
4080 pending.nonce,
4081 pending.api_key_index,
4082 );
4083
4084 if let Some(account_index) = account_index {
4085 let _ = dispatch.nonce_manager.ack_failure_if_latest(
4086 account_index,
4087 pending.api_key_index,
4088 pending.nonce,
4089 );
4090 }
4091 dispatch.clear_pending_order_action_if(client_order_id, PendingOrderAction::Cancel);
4092 emitter.emit_order_cancel_rejected_event(
4093 *strategy_id,
4094 *instrument_id,
4095 *client_order_id,
4096 *venue_order_id,
4097 reason,
4098 now,
4099 );
4100 }
4101 PendingSendTxKind::Modify {
4102 strategy_id,
4103 instrument_id,
4104 client_order_id,
4105 venue_order_id,
4106 } => {
4107 log::error!(
4108 "{reason} attributed to modify cloid={client_order_id} nonce={} api_key_index={}; diagnostic_code={code:?} diagnostic_message={message:?}",
4109 pending.nonce,
4110 pending.api_key_index,
4111 );
4112
4113 if let Some(account_index) = account_index {
4114 let _ = dispatch.nonce_manager.ack_failure_if_latest(
4115 account_index,
4116 pending.api_key_index,
4117 pending.nonce,
4118 );
4119 }
4120 dispatch.clear_pending_order_action_if(client_order_id, PendingOrderAction::Modify);
4121 emitter.emit_order_modify_rejected_event(
4122 *strategy_id,
4123 *instrument_id,
4124 *client_order_id,
4125 *venue_order_id,
4126 reason,
4127 now,
4128 );
4129 }
4130 PendingSendTxKind::Other => {
4131 if let Some(account_index) = account_index {
4132 let _ = dispatch.nonce_manager.ack_failure_if_latest(
4133 account_index,
4134 pending.api_key_index,
4135 pending.nonce,
4136 );
4137 }
4138 log::warn!(
4139 "{reason} on non-create sendTx (nonce={} api_key_index={}); diagnostic_code={code:?} diagnostic_message={message:?}",
4140 pending.nonce,
4141 pending.api_key_index,
4142 );
4143 }
4144 }
4145
4146 needs_nonce_resync
4147}
4148
4149fn lighter_reason_indicates_post_only_rejection(reason: &str) -> bool {
4150 let normalized: String = reason
4151 .chars()
4152 .filter_map(|ch| {
4153 if ch == '-' || ch == '_' || ch.is_whitespace() {
4154 None
4155 } else {
4156 Some(ch.to_ascii_lowercase())
4157 }
4158 })
4159 .collect();
4160
4161 normalized.contains("postonly") || normalized.contains("postwouldexecute")
4162}
4163
4164fn lighter_http_error_is_definite_api_rejection(error: &LighterHttpError) -> bool {
4165 match error {
4166 LighterHttpError::RateLimit(_) | LighterHttpError::Venue { .. } => true,
4167 LighterHttpError::Http { status, .. } => *status < 500,
4168 LighterHttpError::Network(_)
4169 | LighterHttpError::HistoryIncomplete { .. }
4170 | LighterHttpError::Parse(_) => false,
4171 }
4172}
4173
4174fn rollback_tx_dispatch(
4175 dispatch: &WsDispatchState,
4176 credential: &Credential,
4177 client_order_index: Option<i64>,
4178 nonce: i64,
4179) {
4180 rollback_tx_dispatch_indices(
4181 dispatch,
4182 credential.account_index(),
4183 credential.api_key_index(),
4184 client_order_index,
4185 None,
4186 nonce,
4187 );
4188}
4189
4190fn rollback_tx_dispatch_create(
4191 dispatch: &WsDispatchState,
4192 credential: &Credential,
4193 client_order_index: Option<i64>,
4194 client_order_id: &ClientOrderId,
4195 nonce: i64,
4196) {
4197 rollback_tx_dispatch_indices(
4198 dispatch,
4199 credential.account_index(),
4200 credential.api_key_index(),
4201 client_order_index,
4202 Some(client_order_id),
4203 nonce,
4204 );
4205}
4206
4207fn rollback_tx_dispatch_indices(
4211 dispatch: &WsDispatchState,
4212 account_index: i64,
4213 api_key_index: u8,
4214 client_order_index: Option<i64>,
4215 client_order_id: Option<&ClientOrderId>,
4216 nonce: i64,
4217) {
4218 let _ = dispatch
4219 .nonce_manager
4220 .ack_failure_if_latest(account_index, api_key_index, nonce);
4221
4222 if let Some(client_order_index) = client_order_index {
4223 dispatch.forget_cloid(client_order_index);
4224 }
4225
4226 if let Some(cloid) = client_order_id {
4227 dispatch.forget_order_identity(cloid);
4228 }
4229}
4230
4231fn integrator_attributes(integrator_account_index: Option<u64>) -> L2TxAttributes {
4232 integrator_account_index.map_or_else(L2TxAttributes::default, |integrator_account_index| {
4233 L2TxAttributes {
4234 integrator_account_index,
4235 ..Default::default()
4236 }
4237 })
4238}
4239
4240fn command_failure_reason(failure: &CommandFailure) -> &str {
4241 match failure {
4242 CommandFailure::NotSent(reason)
4243 | CommandFailure::Ambiguous(reason)
4244 | CommandFailure::VenueRejected(reason) => reason,
4245 }
4246}
4247
4248fn warn_pending_sendtx_unknown(pending_sendtx: &[PendingSendTx], context: &str) {
4249 if pending_sendtx.is_empty() {
4250 return;
4251 }
4252
4253 log::warn!(
4254 "Discarded {} pending sendTx entries during {context}; order state recovers via \
4255 reconciliation",
4256 pending_sendtx.len(),
4257 );
4258
4259 for pending in pending_sendtx {
4260 if let PendingSendTxKind::Create { order, .. } = &pending.kind {
4261 log::warn!(
4262 "Lighter sendTx outcome unknown during {context} for {}",
4263 order.client_order_id(),
4264 );
4265 }
4266 }
4267}
4268
4269fn classify_lighter_ws_command_failure(action: &str, error: &LighterWsError) -> CommandFailure {
4270 let fallback = format!("Lighter {action} dispatch failed");
4271 let reason = sanitize_strategy_reason(&format!("{fallback}: {error}"))
4272 .unwrap_or_else(|| fallback.clone());
4273
4274 match error {
4275 LighterWsError::SendTxOutcomeUnknown(_)
4276 | LighterWsError::Network(_)
4277 | LighterWsError::Parse(_)
4278 | LighterWsError::Transport(SendError::WriteTimeout | SendError::BrokenPipe(_)) => {
4279 CommandFailure::ambiguous(reason)
4280 }
4281 LighterWsError::Transport(
4282 SendError::InvalidInput(_)
4283 | SendError::Closed
4284 | SendError::Timeout
4285 | SendError::ConnectionChanged,
4286 )
4287 | LighterWsError::Authentication(_)
4288 | LighterWsError::Client(_) => CommandFailure::not_sent(reason),
4289 }
4290}
4291
4292fn venue_rejection_failure(code: Option<i64>, message: &str) -> CommandFailure {
4293 let clean_message = sanitize_strategy_reason(message);
4294 let reason = match (code, clean_message) {
4295 (Some(code), Some(message)) => format!("LIGHTER_{code}: {message}"),
4296 (Some(code), None) => format!("LIGHTER_{code}"),
4297 (None, Some(message)) => message,
4298 (None, None) => "Lighter venue rejected sendTx".to_string(),
4299 };
4300 let reason = sanitize_strategy_reason(&reason)
4301 .unwrap_or_else(|| "Lighter venue rejected sendTx".to_string());
4302 CommandFailure::venue_rejected(reason)
4303}
4304
4305fn sanitize_strategy_reason(input: &str) -> Option<String> {
4306 let mut output = String::new();
4307 let mut inside_markup = false;
4308 let mut pending_space = false;
4309 let mut char_count = 0;
4310 let mut chars = input.chars().peekable();
4311
4312 while let Some(ch) = chars.next() {
4313 if inside_markup {
4314 if ch == '>' {
4315 inside_markup = false;
4316 }
4317 continue;
4318 }
4319
4320 match ch {
4321 '<' if chars.peek().is_some_and(|next| {
4322 next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?')
4323 }) =>
4324 {
4325 inside_markup = true;
4326 pending_space = !output.is_empty();
4327 }
4328 _ if ch.is_control() || is_unicode_format_control(ch) || ch.is_whitespace() => {
4329 pending_space = !output.is_empty();
4330 }
4331 _ => {
4332 if pending_space && char_count < STRATEGY_REASON_MAX_CHARS {
4333 output.push(' ');
4334 char_count += 1;
4335 }
4336 pending_space = false;
4337
4338 if char_count == STRATEGY_REASON_MAX_CHARS {
4339 break;
4340 }
4341 output.push(ch);
4342 char_count += 1;
4343 }
4344 }
4345 }
4346
4347 let output = output.trim().to_string();
4348 (!output.is_empty()).then_some(output)
4349}
4350
4351fn is_unicode_format_control(ch: char) -> bool {
4352 matches!(
4353 ch,
4354 '\u{00ad}'
4355 | '\u{061c}'
4356 | '\u{06dd}'
4357 | '\u{070f}'
4358 | '\u{08e2}'
4359 | '\u{180e}'
4360 | '\u{feff}'
4361 | '\u{fff9}'..='\u{fffb}'
4362 | '\u{0600}'..='\u{0605}'
4363 | '\u{0890}'..='\u{0891}'
4364 | '\u{200b}'..='\u{200f}'
4365 | '\u{202a}'..='\u{202e}'
4366 | '\u{2060}'..='\u{206f}'
4367 | '\u{110bd}'
4368 | '\u{110cd}'
4369 | '\u{13430}'..='\u{1343f}'
4370 | '\u{1bca0}'..='\u{1bca3}'
4371 | '\u{1d173}'..='\u{1d17a}'
4372 | '\u{e0001}'
4373 | '\u{e0020}'..='\u{e007f}'
4374 )
4375}
4376
4377fn format_between_timestamps(
4382 start: Option<UnixNanos>,
4383 end: Option<UnixNanos>,
4384 ts_now: UnixNanos,
4385) -> Option<String> {
4386 let (start, end) = match (start, end) {
4387 (None, None) => return None,
4388 (Some(s), Some(e)) => (s, e),
4389 (Some(s), None) => (s, ts_now),
4390 (None, Some(e)) => (UnixNanos::from(0), e),
4391 };
4392 let start_secs = start.as_u64() / 1_000_000_000;
4393 let end_secs = end.as_u64() / 1_000_000_000;
4394 Some(format!("{start_secs}-{end_secs}"))
4395}
4396
4397#[async_trait(?Send)]
4398impl ExecutionClient for LighterExecutionClient {
4399 fn is_connected(&self) -> bool {
4400 self.core.is_connected()
4401 }
4402
4403 fn client_id(&self) -> ClientId {
4404 self.core.client_id
4405 }
4406
4407 fn account_id(&self) -> AccountId {
4408 self.core.account_id
4409 }
4410
4411 fn venue(&self) -> Venue {
4412 self.core.venue
4413 }
4414
4415 fn oms_type(&self) -> OmsType {
4416 self.core.oms_type
4417 }
4418
4419 fn get_account(&self) -> Option<AccountAny> {
4420 self.core.cache().account_owned(&self.core.account_id)
4421 }
4422
4423 fn generate_account_state(
4424 &self,
4425 balances: Vec<AccountBalance>,
4426 margins: Vec<MarginBalance>,
4427 reported: bool,
4428 ts_event: UnixNanos,
4429 info: Option<Params>,
4430 ) -> anyhow::Result<()> {
4431 self.emitter
4432 .emit_account_state(balances, margins, reported, ts_event, info);
4433 Ok(())
4434 }
4435
4436 fn start(&mut self) -> anyhow::Result<()> {
4437 if self.core.is_started() {
4438 return Ok(());
4439 }
4440
4441 let sender = get_exec_event_sender();
4442 self.emitter.set_sender(sender);
4443 self.core.set_started();
4444
4445 log::info!(
4446 "Started Lighter execution client: client_id={}, account_id={}, environment={:?}, has_credentials={}",
4447 self.core.client_id,
4448 self.core.account_id,
4449 self.config.environment,
4450 self.has_credentials(),
4451 );
4452
4453 Ok(())
4454 }
4455
4456 fn stop(&mut self) -> anyhow::Result<()> {
4457 if self.core.is_stopped() && self.core.is_disconnected() && self.session_tasks_finished() {
4458 return Ok(());
4459 }
4460
4461 log::info!("Stopping Lighter execution client {}", self.core.client_id);
4462
4463 self.begin_session_shutdown();
4464
4465 self.core.set_stopped();
4466
4467 log::info!("Lighter execution client stopped");
4468 Ok(())
4469 }
4470
4471 fn reset(&mut self) -> anyhow::Result<()> {
4472 log::debug!("Resetting Lighter execution client {}", self.core.client_id);
4473 self.begin_session_shutdown();
4474 Ok(())
4475 }
4476
4477 fn dispose(&mut self) -> anyhow::Result<()> {
4478 log::debug!("Disposing Lighter execution client {}", self.core.client_id);
4479 self.stop()
4480 }
4481
4482 async fn connect(&mut self) -> anyhow::Result<()> {
4483 if self.core.is_connected() && self.pending_tasks.is_open() {
4484 return Ok(());
4485 }
4486
4487 if !self.has_credentials() {
4491 anyhow::bail!(
4492 "Lighter execution client requires credentials; \
4493 set private_key, account_index, and api_key_index in the config \
4494 or use the deployment-specific credential environment variables"
4495 );
4496 }
4497
4498 log::info!(
4499 "Connecting Lighter execution client {}",
4500 self.core.client_id
4501 );
4502
4503 if !self.session_tasks_finished() || !self.pending_tasks.is_open() {
4506 self.begin_session_shutdown();
4507 self.finish_session_shutdown().await?;
4508 }
4509
4510 if !self.pending_tasks.is_open() {
4511 self.pending_tasks
4512 .start_generation()
4513 .map_err(|e| anyhow::anyhow!("Failed to start Lighter task generation: {e}"))?;
4514 self.cancellation_token = self.pending_tasks.cancellation_token();
4515 }
4516
4517 let ws_client = self.ws_client.clone();
4518 let setup_guard = TaskGroupGuard::new(&[&self.pending_tasks], move || {
4519 ws_client.begin_shutdown();
4520 });
4521 self.auth_refresh_notify = Arc::new(tokio::sync::Notify::new());
4522
4523 self.dispatch.account_streams_ready.reset();
4529 self.dispatch.clear_position_cache();
4530 self.dispatch.clear_account_state_cache();
4531
4532 self.ensure_instruments_initialized_async().await?;
4533 self.refresh_nonce().await?;
4534
4535 self.nonce_ready_connection_epoch
4537 .store(self.ws_client.connection_epoch(), Ordering::Release);
4538 let account_detail = self.fetch_account_detail().await;
4539 self.account_tier = None;
4540
4541 if let Some(detail) = &account_detail {
4542 let tier = self.detect_account_tier(detail);
4543 self.account_tier = Some(tier);
4544
4545 if !matches!(tier, LighterAccountTier::Plus | LighterAccountTier::Premium) {
4546 log_debug!(
4547 "Lighter {tier} account will omit integrator approval and order attribution",
4548 color = LogColor::Blue
4549 );
4550 }
4551
4552 match tokio::time::timeout(
4553 REFERRAL_ATTRIBUTION_TIMEOUT,
4554 self.apply_referral_attribution(detail),
4555 )
4556 .await
4557 {
4558 Ok(Ok(())) => {}
4559 Ok(Err(e)) => {
4560 log::warn!(
4561 "Robinhood Chain referral attribution failed; continuing startup: {e:?}"
4562 );
4563 }
4564 Err(_) => {
4565 log::warn!(
4566 "Robinhood Chain referral attribution timed out after {}s; continuing \
4567 startup",
4568 REFERRAL_ATTRIBUTION_TIMEOUT.as_secs()
4569 );
4570 }
4571 }
4572 }
4573
4574 if let Err(e) = self.submit_integrator_auto_approval().await {
4575 let is_unapproved = e.chain().any(|cause| {
4580 matches!(
4581 cause.downcast_ref::<LighterHttpError>(),
4582 Some(LighterHttpError::Venue { code: 21149, .. }),
4583 )
4584 });
4585
4586 if is_unapproved {
4587 self.nonce_ready_connection_epoch
4588 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
4589 return Err(e.context(
4590 "Lighter account is not integrator-approved (venue 21149); \
4591 orders cannot be placed",
4592 ));
4593 }
4594 log::error!("Lighter integrator approval failed; continuing startup: {e:?}");
4595 }
4596
4597 self.nonce_ready_connection_epoch
4598 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
4599
4600 if let Err(e) = self.sync_nonce_from_venue().await {
4601 log::debug!(
4602 "Failed to sync Lighter nonce after integrator approval; continuing startup: {e:?}"
4603 );
4604 }
4605
4606 if let Err(e) = self.spawn_ws_consumer().await {
4607 self.begin_session_shutdown();
4608 return match self.finish_session_shutdown().await {
4609 Ok(()) => Err(e),
4610 Err(shutdown_error) => Err(anyhow::anyhow!(
4611 "{e}; failed to roll back partial Lighter connection: {shutdown_error}"
4612 )),
4613 };
4614 }
4615 self.nonce_ready_connection_epoch
4616 .store(self.ws_client.connection_epoch(), Ordering::Release);
4617
4618 if let Err(e) = self.await_account_streams_ready(30.0).await {
4619 log::warn!("Connect failed after WS started, tearing down: {e}");
4620 self.begin_session_shutdown();
4621
4622 if let Err(shutdown_error) = self.finish_session_shutdown().await {
4623 log::warn!("Failed to finish partial Lighter connection: {shutdown_error}");
4624 }
4625
4626 return Err(e);
4627 }
4628
4629 setup_guard.disarm();
4630 self.core.set_connected();
4631
4632 log::info!("Connected: client_id={}", self.core.client_id);
4633 Ok(())
4634 }
4635
4636 async fn disconnect(&mut self) -> anyhow::Result<()> {
4637 if self.core.is_disconnected() && self.session_tasks_finished() {
4638 return Ok(());
4639 }
4640
4641 log::info!(
4642 "Disconnecting Lighter execution client {}",
4643 self.core.client_id
4644 );
4645
4646 self.begin_session_shutdown();
4647 let tasks_result = self.finish_session_shutdown().await;
4648
4649 self.core.set_disconnected();
4650
4651 log::info!("Disconnected: client_id={}", self.core.client_id);
4652 tasks_result
4653 }
4654
4655 fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
4656 let credential = self.credential.as_ref().ok_or_else(|| {
4657 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
4658 })?;
4659
4660 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
4661
4662 if order.is_closed() {
4663 log::warn!("Cannot submit closed order {}", order.client_order_id());
4664 return Ok(());
4665 }
4666
4667 let cached_instrument = self
4668 .core
4669 .cache()
4670 .instrument(&order.instrument_id())
4671 .cloned();
4672
4673 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
4674 self.emitter.emit_order_denied(&order, &reason);
4675 return Ok(());
4676 }
4677
4678 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
4679 let plan = match self.prepare_create_order_plan(&order, slippage_bps) {
4680 Ok(plan) => plan,
4681 Err(e) => {
4682 let reason = OrderDeniedReason::ValidationFailed {
4683 detail: format!("Lighter submit_order failed: {e}"),
4684 };
4685 self.emitter.emit_order_denied(&order, &reason.to_string());
4686 return Ok(());
4687 }
4688 };
4689
4690 if let Err(e) = self.dispatch_create_order_plan(plan, credential) {
4691 let reason = OrderDeniedReason::SubmitFailed {
4692 detail: format!("Lighter submit_order failed: {e}"),
4693 };
4694 self.emitter.emit_order_denied(&order, &reason.to_string());
4695 }
4696
4697 Ok(())
4698 }
4699
4700 fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
4701 let credential = self.credential.as_ref().ok_or_else(|| {
4702 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
4703 })?;
4704
4705 if cmd.order_list.client_order_ids.is_empty() {
4706 log::debug!("submit_order_list called with empty order list");
4707 return Ok(());
4708 }
4709
4710 let orders = self.core.get_orders_for_list(&cmd.order_list)?;
4711
4712 if orders.len() > LIGHTER_MAX_BATCH_TX {
4713 let reason = OrderDeniedReason::UnsupportedOrderList {
4714 detail: format!(
4715 "Lighter order-list fanout supports at most {LIGHTER_MAX_BATCH_TX} txs, was {}",
4716 orders.len(),
4717 ),
4718 }
4719 .to_string();
4720
4721 for order in &orders {
4722 self.emitter.emit_order_denied(order, &reason);
4723 }
4724 return Ok(());
4725 }
4726
4727 if orders.iter().any(is_grouped_order) {
4728 let reason = OrderDeniedReason::UnsupportedOrderList {
4729 detail: format!(
4730 "Lighter submit_order_list supports only independent orders; \
4731 grouped contingency lists remain out of scope (order_list_id={})",
4732 cmd.order_list.id,
4733 ),
4734 }
4735 .to_string();
4736
4737 for order in &orders {
4738 self.emitter.emit_order_denied(order, &reason);
4739 }
4740 return Ok(());
4741 }
4742
4743 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
4744 let mut plans = Vec::with_capacity(orders.len());
4745
4746 for order in orders {
4747 if order.is_closed() {
4748 log::warn!("Cannot submit closed order {}", order.client_order_id());
4749 continue;
4750 }
4751
4752 let cached_instrument = self
4753 .core
4754 .cache()
4755 .instrument(&order.instrument_id())
4756 .cloned();
4757
4758 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
4759 self.emitter.emit_order_denied(&order, &reason);
4760 continue;
4761 }
4762
4763 match self.prepare_create_order_plan(&order, slippage_bps) {
4764 Ok(plan) => plans.push(plan),
4765 Err(e) => {
4766 let reason = OrderDeniedReason::ValidationFailed {
4767 detail: format!("Lighter submit_order_list failed: {e}"),
4768 }
4769 .to_string();
4770
4771 self.emitter.emit_order_denied(&order, &reason);
4772 }
4773 }
4774 }
4775
4776 if plans.is_empty() {
4777 log::warn!(
4778 "Lighter submit_order_list: no supported orders to dispatch for {}",
4779 cmd.order_list.id,
4780 );
4781 return Ok(());
4782 }
4783
4784 let context = self.fanout_dispatch_context(credential)?;
4785 self.spawn_task("submit_order_list", async move {
4786 for plan in plans {
4787 let order = plan.order.clone();
4788 match context.sign_create_order(plan) {
4789 Ok(prepared) => context.send_create_order(prepared).await,
4790 Err(e) => {
4791 let reason = OrderDeniedReason::SubmitFailed {
4792 detail: format!("Lighter submit_order_list failed: {e}"),
4793 };
4794 context
4795 .emitter
4796 .emit_order_denied(&order, &reason.to_string());
4797 }
4798 }
4799 }
4800 Ok(())
4801 });
4802
4803 Ok(())
4804 }
4805
4806 fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
4807 let credential = self.credential.as_ref().ok_or_else(|| {
4808 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
4809 })?;
4810 self.dispatch_signed_modify_order(&cmd, credential);
4811 Ok(())
4812 }
4813
4814 fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
4815 let credential = self.credential.as_ref().ok_or_else(|| {
4816 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
4817 })?;
4818 self.dispatch_signed_cancel_order(&cmd, credential);
4819 Ok(())
4820 }
4821
4822 fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
4823 let cancels: Vec<_> = self
4826 .core
4827 .cache()
4828 .orders_open(None, Some(&cmd.instrument_id), None, None, cmd.order_side)
4829 .into_iter()
4830 .map(|order| {
4831 cancel_order_from_cancel_all(&cmd, order.client_order_id(), order.strategy_id())
4832 })
4833 .collect();
4834
4835 if cancels.is_empty() {
4836 return Ok(());
4837 }
4838
4839 self.dispatch_cancel_orders(&cancels)
4840 }
4841
4842 fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
4843 if cmd.cancels.len() > LIGHTER_MAX_BATCH_TX {
4844 let reason = format!(
4845 "Lighter batch-cancel supports at most {LIGHTER_MAX_BATCH_TX} txs, was {}",
4846 cmd.cancels.len()
4847 );
4848
4849 for cancel in &cmd.cancels {
4850 self.dispatch.clear_pending_order_action_if(
4851 &cancel.client_order_id,
4852 PendingOrderAction::Cancel,
4853 );
4854
4855 if self.can_emit_order_cancel_rejected(&cancel.client_order_id) {
4856 self.emitter.emit_order_cancel_rejected_event(
4857 cancel.strategy_id,
4858 cancel.instrument_id,
4859 cancel.client_order_id,
4860 cancel.venue_order_id,
4861 &reason,
4862 self.clock.get_time_ns(),
4863 );
4864 }
4865 }
4866
4867 return Ok(());
4868 }
4869
4870 self.dispatch_cancel_orders(&cmd.cancels)
4871 }
4872
4873 fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
4874 let cached = self.dispatch.snapshot_account_state();
4880 match cached {
4881 Some(state) => {
4882 log::debug!("Lighter query_account replaying cached AccountState");
4883 self.emitter.send_account_state(state);
4884 }
4885 None => {
4886 log::warn!(
4887 "Lighter query_account: no AccountState cached yet \
4888 (account_all_assets stream has not pushed since connect)",
4889 );
4890 }
4891 }
4892 Ok(())
4893 }
4894
4895 fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
4896 let credential = self
4897 .credential
4898 .as_ref()
4899 .ok_or_else(|| anyhow::anyhow!("Lighter query_order requires credentials"))?
4900 .clone();
4901 let registry = Arc::clone(&self.registry);
4902 let http_client = self.http_client.clone();
4903 let emitter = self.emitter.clone();
4904 let core_account_id = self.core.account_id;
4905 let dispatch = self.dispatch.clone();
4906 let clock = self.clock;
4907
4908 self.spawn_task("query_order", async move {
4909 let report = lookup_order_status_report(
4910 &http_client,
4911 ®istry,
4912 &credential,
4913 core_account_id,
4914 Some(cmd.instrument_id),
4915 Some(&cmd.client_order_id),
4916 cmd.venue_order_id.as_ref(),
4917 &dispatch,
4918 clock,
4919 )
4920 .await?;
4921
4922 match report {
4923 Some(report) => {
4924 log::debug!(
4925 "Lighter query_order returning report for {}",
4926 cmd.client_order_id
4927 );
4928 dispatch.seed_accepted_from_report(&report);
4929 emitter.send_order_status_report(report);
4930 }
4931 None => {
4932 log::warn!(
4933 "Lighter query_order: no order found for {}",
4934 cmd.client_order_id,
4935 );
4936 }
4937 }
4938 Ok(())
4939 });
4940 Ok(())
4941 }
4942
4943 async fn generate_order_status_report(
4944 &self,
4945 cmd: &GenerateOrderStatusReport,
4946 ) -> anyhow::Result<Option<OrderStatusReport>> {
4947 let Some(credential) = &self.credential else {
4948 log::warn!("Lighter generate_order_status_report: no credentials");
4949 return Ok(None);
4950 };
4951
4952 if cmd.client_order_id.is_none() && cmd.venue_order_id.is_none() {
4953 log::warn!(
4954 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id",
4955 );
4956 return Ok(None);
4957 }
4958 let report = lookup_order_status_report(
4959 &self.http_client,
4960 &self.registry,
4961 credential,
4962 self.core.account_id,
4963 cmd.instrument_id,
4964 cmd.client_order_id.as_ref(),
4965 cmd.venue_order_id.as_ref(),
4966 &self.dispatch,
4967 self.clock,
4968 )
4969 .await?;
4970
4971 if let Some(report) = &report {
4972 self.dispatch.seed_accepted_from_report(report);
4973 }
4974
4975 Ok(report)
4976 }
4977
4978 async fn generate_order_status_reports(
4979 &self,
4980 cmd: &GenerateOrderStatusReports,
4981 ) -> anyhow::Result<Vec<OrderStatusReport>> {
4982 let Some(credential) = &self.credential else {
4983 log::warn!("Lighter generate_order_status_reports: no credentials");
4984 Self::log_report_receipt(0, "OrderStatusReport", cmd.log_receipt_level);
4985 return Ok(Vec::new());
4986 };
4987
4988 let auth = build_auth_token_for(credential)
4989 .context("failed to mint Lighter auth token for report fetch")?;
4990 let ts_init = self.clock.get_time_ns();
4991
4992 if cmd.instrument_id.is_none() && self.dispatch.active_markets_snapshot().is_empty() {
5001 seed_active_markets_from_inactive_orders(
5002 &self.http_client,
5003 &self.dispatch,
5004 credential,
5005 &auth,
5006 format_between_timestamps(cmd.start, cmd.end, ts_init),
5007 )
5008 .await?;
5009 }
5010
5011 let market_indices = match cmd.instrument_id {
5012 Some(id) => match self.registry.market_index(&id) {
5013 Some(idx) => vec![idx],
5014 None => {
5015 anyhow::bail!("no Lighter market_index for order report instrument {id}",);
5016 }
5017 },
5018 None => self.dispatch.active_markets_snapshot(),
5019 };
5020
5021 if market_indices.is_empty() {
5022 log::debug!(
5023 "Lighter generate_order_status_reports: no active markets yet; returning empty",
5024 );
5025 Self::log_report_receipt(0, "OrderStatusReport", cmd.log_receipt_level);
5026 return Ok(Vec::new());
5027 }
5028
5029 let mut reports: Vec<OrderStatusReport> = Vec::new();
5030 let mut active_errors = Vec::new();
5031
5032 for market_index in market_indices {
5038 let query = Zeroizing::new(LighterAccountActiveOrdersQuery {
5039 authorization: None,
5040 auth: Some(auth.clone()),
5041 account_index: credential.account_index(),
5042 market_id: market_index,
5043 });
5044 let active = match self.http_client.get_account_active_orders(&query).await {
5045 Ok(response) => response,
5046 Err(e) => {
5047 let detail = format!(
5048 "failed to fetch Lighter active orders for market_index={market_index}: {}",
5049 scrub_auth(&format!("{e:#}")),
5050 );
5051 log::warn!("{detail}",);
5052 active_errors.push(detail);
5053 continue;
5054 }
5055 };
5056
5057 for order in &active.orders {
5058 self.dispatch.note_active_market(order.market_index);
5059
5060 let Some(report) = parse_http_order_to_report(
5061 order,
5062 &self.registry,
5063 self.core.account_id,
5064 ts_init,
5065 ) else {
5066 let detail = format!(
5067 "failed to parse Lighter active order {} for market_index={market_index}",
5068 order.order_id,
5069 );
5070 log::warn!("{detail}");
5071 active_errors.push(detail);
5072 continue;
5073 };
5074 restore_reconciled_order(
5075 &self.core,
5076 &self.dispatch,
5077 order,
5078 report.order_status.is_closed(),
5079 );
5080 let report = self.dispatch.translate_order_cloid(report);
5081 let report = self.dispatch.preserve_pending_order_status(report);
5082 self.dispatch.seed_accepted_from_report(&report);
5083 reports.push(report);
5084 }
5085 }
5086
5087 if !active_errors.is_empty() {
5088 return Err(incomplete_order_reports(reports, active_errors.join("; ")));
5089 }
5090
5091 if !cmd.open_only {
5099 let inactive_markets: Vec<i64> = match cmd.instrument_id {
5100 Some(id) => self
5101 .registry
5102 .market_index(&id)
5103 .map(|m| vec![m])
5104 .unwrap_or_default(),
5105 None => self.dispatch.active_markets_snapshot(),
5106 };
5107
5108 let between_timestamps = format_between_timestamps(cmd.start, cmd.end, ts_init);
5109
5110 for market_id in inactive_markets {
5111 let mut cursor: Option<String> = None;
5112 let mut seen_cursors = AHashSet::new();
5113 let mut pages = 0_usize;
5114
5115 loop {
5116 pages += 1;
5117 if pages > MAX_RECONCILIATION_PAGES {
5118 return Err(incomplete_order_reports(
5119 reports,
5120 format!(
5121 "Lighter inactive-order reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages for market_index={market_id}",
5122 ),
5123 ));
5124 }
5125
5126 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
5127 authorization: None,
5128 auth: Some(auth.clone()),
5129 account_index: credential.account_index(),
5130 market_id: Some(market_id),
5131 ask_filter: None,
5132 between_timestamps: between_timestamps.clone(),
5133 cursor: cursor.clone(),
5134 limit: LIGHTER_REST_PAGE_SIZE,
5135 });
5136
5137 match self.http_client.get_account_inactive_orders(&query).await {
5138 Ok(inactive) => {
5139 for order in &inactive.orders {
5140 let Some(report) = parse_http_order_to_report(
5141 order,
5142 &self.registry,
5143 self.core.account_id,
5144 ts_init,
5145 ) else {
5146 return Err(incomplete_order_reports(
5147 reports,
5148 format!(
5149 "failed to parse Lighter inactive order {} for market_index={market_id}",
5150 order.order_id,
5151 ),
5152 ));
5153 };
5154
5155 if cmd.start.is_some_and(|start| report.ts_last < start)
5156 || cmd.end.is_some_and(|end| report.ts_last > end)
5157 {
5158 continue;
5159 }
5160
5161 self.dispatch.note_active_market(order.market_index);
5162 restore_reconciled_order(
5163 &self.core,
5164 &self.dispatch,
5165 order,
5166 report.order_status.is_closed(),
5167 );
5168 let report = self.dispatch.translate_order_cloid(report);
5169 let report = self.dispatch.preserve_pending_order_status(report);
5170 self.dispatch.seed_accepted_from_report(&report);
5171 reports.push(report);
5172 }
5173
5174 match inactive.next_cursor {
5175 Some(next) if !next.is_empty() => {
5176 if !seen_cursors.insert(next.clone()) {
5177 return Err(incomplete_order_reports(
5178 reports,
5179 format!(
5180 "Lighter inactive-order reconciliation repeated cursor `{next}` for market_index={market_id}",
5181 ),
5182 ));
5183 }
5184 cursor = Some(next);
5185 }
5186 _ => break,
5187 }
5188 }
5189 Err(e) => {
5190 return Err(incomplete_order_reports(
5191 reports,
5192 format!(
5193 "failed to fetch Lighter inactive orders for market_index={market_id}: {}",
5194 scrub_auth(&format!("{e:#}")),
5195 ),
5196 ));
5197 }
5198 }
5199 }
5200 }
5201 }
5202
5203 Self::log_report_receipt(reports.len(), "OrderStatusReport", cmd.log_receipt_level);
5204 Ok(reports)
5205 }
5206
5207 async fn generate_fill_reports(
5208 &self,
5209 cmd: GenerateFillReports,
5210 ) -> anyhow::Result<Vec<FillReport>> {
5211 let reports = self.paginate_fill_reports(&cmd).await?.reports;
5212 Self::log_report_receipt(reports.len(), "FillReport", cmd.log_receipt_level);
5213 Ok(reports)
5214 }
5215
5216 async fn generate_position_status_reports(
5217 &self,
5218 cmd: &GeneratePositionStatusReports,
5219 ) -> anyhow::Result<Vec<PositionStatusReport>> {
5220 let (reports, complete, _) = self.cached_position_reports(cmd)?;
5221 anyhow::ensure!(
5222 complete,
5223 "Lighter position snapshot does not cover the requested instrument scope",
5224 );
5225 Self::log_report_receipt(reports.len(), "PositionStatusReport", cmd.log_receipt_level);
5226 Ok(reports)
5227 }
5228
5229 async fn generate_mass_status(
5230 &self,
5231 lookback_mins: Option<u64>,
5232 ) -> anyhow::Result<Option<ExecutionMassStatus>> {
5233 let ts_init = self.clock.get_time_ns();
5234
5235 let lookback_start = lookback_mins
5238 .map(DurationNanos::try_from_mins)
5239 .transpose()?
5240 .map(|lookback| ts_init.saturating_sub(lookback));
5241
5242 let mut order_cmd = GenerateOrderStatusReports::new(
5247 UUID4::new(),
5248 ts_init,
5249 false,
5250 None,
5251 lookback_start,
5252 None,
5253 None,
5254 None,
5255 );
5256 order_cmd.log_receipt_level = LogLevel::Debug;
5257 let fill_cmd = GenerateFillReports::new(
5258 UUID4::new(),
5259 ts_init,
5260 None,
5261 None,
5262 lookback_start,
5263 None,
5264 None,
5265 None,
5266 );
5267 let position_cmd =
5268 GeneratePositionStatusReports::new(UUID4::new(), ts_init, None, None, None, None, None);
5269
5270 let order_result = self.generate_order_status_reports(&order_cmd).await;
5273 let (mut order_reports, mut reports_complete) = match order_result {
5274 Ok(reports) => (reports, true),
5275 Err(e) => {
5276 log::warn!(
5277 "Lighter order report generation failed: {}",
5278 scrub_auth(&format!("{e:#}")),
5279 );
5280 let partial = partial_order_reports(&e);
5281 let reports = if partial.is_empty() {
5282 let mut active_cmd = GenerateOrderStatusReports::new(
5283 UUID4::new(),
5284 ts_init,
5285 true,
5286 None,
5287 None,
5288 None,
5289 None,
5290 None,
5291 );
5292 active_cmd.log_receipt_level = LogLevel::Debug;
5293 let active_result = self.generate_order_status_reports(&active_cmd).await;
5294 match active_result {
5295 Ok(reports) => reports,
5296 Err(e) => {
5297 log::warn!(
5298 "Lighter active order report generation incomplete: {}",
5299 scrub_auth(&format!("{e:#}")),
5300 );
5301 partial_order_reports(&e)
5302 }
5303 }
5304 } else {
5305 partial
5306 };
5307 (reports, false)
5308 }
5309 };
5310 let (mut fill_reports, fill_reports_complete) =
5311 fill_reports_for_mass_status(self.paginate_fill_reports(&fill_cmd).await)
5312 .context("Lighter fill reconciliation failed")?;
5313 reports_complete &= fill_reports_complete;
5314 Self::log_report_receipt(fill_reports.len(), "FillReport", fill_cmd.log_receipt_level);
5315
5316 let mut reported_orders: AHashSet<VenueOrderId> = order_reports
5317 .iter()
5318 .map(|report| report.venue_order_id)
5319 .collect();
5320 let mut fill_markets: Vec<i64> = fill_reports
5321 .iter()
5322 .filter(|report| !reported_orders.contains(&report.venue_order_id))
5323 .filter_map(|report| self.registry.market_index(&report.instrument_id))
5324 .collect::<AHashSet<_>>()
5325 .into_iter()
5326 .collect();
5327 fill_markets.sort_unstable();
5328
5329 for market_index in fill_markets {
5330 let Some(instrument_id) = self.registry.instrument_id(market_index) else {
5331 reports_complete = false;
5332 continue;
5333 };
5334 let mut order_cmd = GenerateOrderStatusReports::new(
5335 UUID4::new(),
5336 ts_init,
5337 false,
5338 Some(instrument_id),
5339 lookback_start,
5340 None,
5341 None,
5342 None,
5343 );
5344 order_cmd.log_receipt_level = LogLevel::Debug;
5345 let order_result = self.generate_order_status_reports(&order_cmd).await;
5346 reports_complete &= order_result.is_ok();
5347 let reports = match order_result {
5348 Ok(reports) => reports,
5349 Err(e) => {
5350 log::warn!(
5351 "Lighter order report generation failed: {}",
5352 scrub_auth(&format!("{e:#}")),
5353 );
5354 partial_order_reports(&e)
5355 }
5356 };
5357 reported_orders.extend(reports.iter().map(|report| report.venue_order_id));
5358 order_reports.extend(reports);
5359 }
5360
5361 if fill_reports
5362 .iter()
5363 .any(|report| !reported_orders.contains(&report.venue_order_id))
5364 {
5365 reports_complete = false;
5366 }
5367
5368 for fill_report in &mut fill_reports {
5369 if let Some(client_order_id) = order_reports
5370 .iter()
5371 .find(|order_report| order_report.venue_order_id == fill_report.venue_order_id)
5372 .and_then(|order_report| order_report.client_order_id)
5373 {
5374 fill_report.client_order_id = Some(client_order_id);
5375 }
5376 }
5377
5378 let position_result = self.cached_position_reports(&position_cmd);
5379 let (mut position_reports, position_coverage) = match position_result {
5380 Ok((reports, complete, coverage)) => {
5381 reports_complete &= complete;
5382 (reports, coverage)
5383 }
5384 Err(e) => {
5385 reports_complete = false;
5386 log::warn!("Lighter position report generation failed: {e:#}");
5387 (Vec::new(), None)
5388 }
5389 };
5390 Self::log_report_receipt(
5391 position_reports.len(),
5392 "PositionStatusReport",
5393 position_cmd.log_receipt_level,
5394 );
5395
5396 if lookback_start.is_some() {
5397 let touched_instruments: AHashSet<InstrumentId> = order_reports
5398 .iter()
5399 .map(|report| report.instrument_id)
5400 .chain(fill_reports.iter().map(|report| report.instrument_id))
5401 .collect();
5402 let position_instruments: AHashSet<InstrumentId> = position_reports
5403 .iter()
5404 .map(|report| report.instrument_id)
5405 .collect();
5406 let cache = self.core.cache();
5407 let mut touched_instruments: Vec<InstrumentId> =
5408 touched_instruments.into_iter().collect();
5409 touched_instruments.sort_unstable();
5410
5411 for instrument_id in touched_instruments {
5412 let Some(instrument) = cache.instrument(&instrument_id) else {
5413 reports_complete = false;
5414 continue;
5415 };
5416
5417 if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
5418 continue;
5419 }
5420 let Some(market_id) = self.registry.market_index(&instrument_id) else {
5421 reports_complete = false;
5422 continue;
5423 };
5424
5425 if !position_coverage
5426 .as_ref()
5427 .is_some_and(|skipped| !skipped.contains(&market_id))
5428 {
5429 reports_complete = false;
5430 continue;
5431 }
5432
5433 if position_instruments.contains(&instrument_id) {
5434 continue;
5435 }
5436
5437 position_reports.push(PositionStatusReport::new(
5438 self.core.account_id,
5439 instrument_id,
5440 PositionSide::Flat,
5441 Quantity::zero(instrument.size_precision()),
5442 ts_init,
5443 ts_init,
5444 Some(UUID4::new()),
5445 None,
5446 None,
5447 ));
5448 }
5449 }
5450
5451 let mut mass_status = ExecutionMassStatus::new(
5452 self.core.client_id,
5453 self.core.account_id,
5454 self.core.venue,
5455 ts_init,
5456 None,
5457 );
5458 mass_status.set_report_window(lookback_start, reports_complete);
5459 mass_status.add_order_reports(order_reports);
5460 mass_status.add_fill_reports(fill_reports);
5461 mass_status.add_position_reports(position_reports);
5462
5463 log::debug!(
5464 "Generated Lighter mass status: {} orders, {} fills, {} positions",
5465 mass_status.order_reports().len(),
5466 mass_status.fill_reports().len(),
5467 mass_status.position_reports().len(),
5468 );
5469
5470 Ok(Some(mass_status))
5471 }
5472}
5473
5474struct FillSweep {
5478 reports: Vec<FillReport>,
5479 covers_window: bool,
5480}
5481
5482#[derive(Debug, thiserror::Error)]
5483#[error("incomplete Lighter order reports: {detail}")]
5484struct IncompleteOrderReports {
5485 reports: Vec<OrderStatusReport>,
5486 detail: String,
5487}
5488
5489fn incomplete_order_reports(
5490 reports: Vec<OrderStatusReport>,
5491 detail: impl Into<String>,
5492) -> anyhow::Error {
5493 anyhow::Error::new(IncompleteOrderReports {
5494 reports,
5495 detail: detail.into(),
5496 })
5497}
5498
5499fn partial_order_reports(error: &anyhow::Error) -> Vec<OrderStatusReport> {
5500 error
5501 .downcast_ref::<IncompleteOrderReports>()
5502 .map(|incomplete| incomplete.reports.clone())
5503 .unwrap_or_default()
5504}
5505
5506fn is_commission_error(error: &anyhow::Error) -> bool {
5507 error
5508 .chain()
5509 .any(|cause| cause.downcast_ref::<LighterCommissionError>().is_some())
5510}
5511
5512fn fill_reports_for_mass_status(
5513 result: anyhow::Result<FillSweep>,
5514) -> anyhow::Result<(Vec<FillReport>, bool)> {
5515 match result {
5516 Ok(sweep) => Ok((sweep.reports, sweep.covers_window)),
5517 Err(e) if is_commission_error(&e) => Err(e),
5518 Err(e) => {
5519 log::warn!(
5520 "Lighter fill report generation failed: {}",
5521 scrub_auth(&format!("{e:#}")),
5522 );
5523 Ok((Vec::new(), false))
5524 }
5525 }
5526}
5527
5528impl LighterExecutionClient {
5529 fn cached_position_reports(
5530 &self,
5531 cmd: &GeneratePositionStatusReports,
5532 ) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
5533 let (mut reports, coverage) = self.dispatch.snapshot_positions_with_coverage();
5537 let complete = match cmd.instrument_id {
5538 Some(instrument_id) => {
5539 let market_id = self.registry.market_index(&instrument_id).ok_or_else(|| {
5540 anyhow::anyhow!(
5541 "no Lighter market_index for position report instrument {instrument_id}",
5542 )
5543 })?;
5544 reports.retain(|report| report.instrument_id == instrument_id);
5545 coverage
5546 .as_ref()
5547 .is_some_and(|skipped| !skipped.contains(&market_id))
5548 }
5549 None => coverage.as_ref().is_some_and(|skipped| skipped.is_empty()),
5550 };
5551 Ok((reports, complete, coverage))
5552 }
5553
5554 async fn paginate_fill_reports(&self, cmd: &GenerateFillReports) -> anyhow::Result<FillSweep> {
5555 let Some(credential) = &self.credential else {
5556 log::warn!("Lighter generate_fill_reports: no credentials");
5557 return Ok(FillSweep {
5558 reports: Vec::new(),
5559 covers_window: true,
5560 });
5561 };
5562
5563 let market_id = match cmd.instrument_id {
5564 Some(instrument_id) => {
5565 Some(self.registry.market_index(&instrument_id).ok_or_else(|| {
5566 anyhow::anyhow!("no Lighter market_index for fill instrument {instrument_id}",)
5567 })?)
5568 }
5569 None => None,
5570 };
5571
5572 let auth = build_auth_token_for(credential)
5573 .context("failed to mint Lighter auth token for fill fetch")?;
5574
5575 let ts_init = self.clock.get_time_ns();
5576 let mut reports = Vec::new();
5577 let mut cursor: Option<String> = None;
5578 let mut seen_cursors = AHashSet::new();
5579 let mut seen_in_call = AHashSet::new();
5580 let mut pages = 0_usize;
5581 let mut oldest_served: Option<UnixNanos> = None;
5582 let mut covers_window = true;
5583
5584 loop {
5585 pages += 1;
5586 anyhow::ensure!(
5587 pages <= MAX_RECONCILIATION_PAGES,
5588 "Lighter fill reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages",
5589 );
5590 let query = Zeroizing::new(LighterTradesQuery {
5591 authorization: None,
5592 auth: Some(auth.clone()),
5593 market_id,
5594 account_index: Some(credential.account_index()),
5595 order_index: None,
5596 sort_by: LighterTradeSortBy::Timestamp,
5597 sort_dir: Some(LighterSortDirection::Desc),
5598 cursor: cursor.clone(),
5599 from_timestamp: None,
5602 ask_filter: None,
5603 role: None,
5604 trade_type: None,
5605 limit: LIGHTER_REST_PAGE_SIZE,
5606 aggregate: None,
5607 });
5608
5609 let response = match self.http_client.get_trades(&query).await {
5610 Ok(response) => response,
5611 Err(e) => {
5612 log::warn!(
5616 "Lighter get_trades failed (market_id={:?}, account_index={}, cursor={:?}): {}",
5617 query.market_id,
5618 credential.account_index(),
5619 cursor,
5620 scrub_auth(&format!("{e:#}")),
5621 );
5622 return Err(anyhow::Error::new(e).context("failed to fetch Lighter fills"));
5623 }
5624 };
5625
5626 for trade in &response.trades {
5627 let Some(instrument_id) = self.registry.instrument_id(trade.market_id) else {
5628 anyhow::bail!(
5629 "no Lighter instrument registered for fill market_index={}",
5630 trade.market_id,
5631 );
5632 };
5633 let Some(instrument) = self.core.cache().instrument(&instrument_id).cloned() else {
5634 anyhow::bail!("Lighter fill instrument {instrument_id} missing from cache");
5635 };
5636
5637 match parse_ws_fill_report(
5638 trade,
5639 credential.account_index(),
5640 &instrument,
5641 self.core.account_id,
5642 ts_init,
5643 ) {
5644 Ok(Some(report)) => {
5645 if cmd.start.is_some_and(|start| report.ts_event < start)
5646 || cmd.end.is_some_and(|end| report.ts_event > end)
5647 {
5648 continue;
5649 }
5650
5651 let report = self.dispatch.translate_fill_cloid(report);
5654
5655 if !seen_in_call.insert(report.trade_id) {
5656 log::debug!(
5657 "Lighter duplicate trade {} ignored within HTTP fill pagination",
5658 report.trade_id,
5659 );
5660 continue;
5661 }
5662
5663 self.dispatch.note_active_market(trade.market_id);
5664 reports.push(report);
5665 }
5666 Ok(None) => {}
5667 Err(e) => return Err(e).context("failed to parse Lighter fill report"),
5668 }
5669 }
5670
5671 let page_oldest = response
5672 .trades
5673 .iter()
5674 .filter_map(|trade| u64::try_from(trade.timestamp).ok())
5675 .map(|timestamp_ms| UnixNanos::from(timestamp_ms.saturating_mul(1_000_000)))
5676 .min();
5677
5678 if let Some(page_oldest) = page_oldest {
5679 oldest_served = Some(oldest_served.map_or(page_oldest, |ts| ts.min(page_oldest)));
5680 }
5681
5682 let reached_start_boundary = cmd
5683 .start
5684 .is_some_and(|start| page_oldest.is_some_and(|oldest| oldest < start));
5685
5686 if reached_start_boundary {
5687 break;
5688 }
5689
5690 match response.next_cursor {
5691 Some(next) if !next.is_empty() => {
5692 anyhow::ensure!(
5693 seen_cursors.insert(next.clone()),
5694 "Lighter fill reconciliation repeated cursor `{next}`",
5695 );
5696 cursor = Some(next);
5697 }
5698 _ => {
5699 if let (Some(start), Some(oldest)) = (cmd.start, oldest_served) {
5705 covers_window = false;
5706
5707 log::warn!(
5708 "Lighter fill reports do not cover {} to {}: trade pagination exhausted before the requested start; the venue `export` endpoint serves full history",
5709 unix_nanos_to_iso8601(start),
5710 unix_nanos_to_iso8601(oldest),
5711 );
5712 }
5713
5714 break;
5715 }
5716 }
5717 }
5718
5719 reports.retain(|report| {
5720 if matches!(
5721 self.dispatch.mark_trade_reconciled(report.trade_id),
5722 Some(TradeDedupSource::Live),
5723 ) {
5724 log::debug!(
5725 "Lighter trade {} ignored in HTTP fill reports after live delivery",
5726 report.trade_id,
5727 );
5728 false
5729 } else {
5730 true
5731 }
5732 });
5733
5734 Ok(FillSweep {
5735 reports,
5736 covers_window,
5737 })
5738 }
5739}
5740
5741fn restore_reconciled_order(
5742 core: &ExecutionClientCore,
5743 dispatch: &WsDispatchState,
5744 raw: &crate::http::models::LighterOrder,
5745 terminal: bool,
5746) {
5747 let venue_order_id = VenueOrderId::new(raw.order_id.as_str());
5748 let cached_order = {
5749 let cache = core.cache();
5750 let Some(cloid) = cache.client_order_id(&venue_order_id).copied() else {
5751 return;
5752 };
5753 let Some(order) = cache.order_owned(&cloid) else {
5754 log::warn!(
5755 "Ignoring reconciled Lighter order missing from cache: cloid={cloid}, venue_order_id={venue_order_id}",
5756 );
5757 return;
5758 };
5759 order
5760 };
5761
5762 if cached_order.instrument_id().venue != core.venue
5763 || cached_order.account_id() != Some(core.account_id)
5764 {
5765 log::warn!(
5766 "Ignoring reconciled Lighter order outside this client: cloid={}, venue_order_id={venue_order_id}",
5767 cached_order.client_order_id(),
5768 );
5769 return;
5770 }
5771
5772 if let Err(e) = dispatch.restore_reconciled_order(
5773 &cached_order,
5774 raw.client_order_index,
5775 venue_order_id,
5776 terminal,
5777 ) {
5778 log::warn!(
5779 "Ignoring conflicting Lighter reconciliation identity: cloid={}, venue_order_id={venue_order_id}, client_order_index={}, error={e}",
5780 cached_order.client_order_id(),
5781 raw.client_order_index,
5782 );
5783 }
5784}
5785
5786fn local_submit_denial_reason(
5787 order: &OrderAny,
5788 instrument: Option<&InstrumentAny>,
5789) -> Option<String> {
5790 if instrument.is_none() {
5791 return Some(
5792 OrderDeniedReason::InstrumentNotFound {
5793 instrument_id: order.instrument_id(),
5794 }
5795 .to_string(),
5796 );
5797 }
5798
5799 if !is_lighter_supported_order_type(order.order_type()) {
5800 return Some(unsupported_lighter_order_type_reason(order.order_type()));
5801 }
5802
5803 if is_lighter_limit_style_order(order.order_type()) && order.price().is_none() {
5804 return Some(
5805 OrderDeniedReason::ValidationFailed {
5806 detail: "Lighter limit-style orders require a limit price".to_string(),
5807 }
5808 .to_string(),
5809 );
5810 }
5811
5812 if order.is_quote_quantity() {
5813 return Some(
5814 OrderDeniedReason::ValidationFailed {
5815 detail:
5816 "Lighter orders do not support quote_quantity; submit base quantity instead"
5817 .to_string(),
5818 }
5819 .to_string(),
5820 );
5821 }
5822
5823 if order.display_qty().is_some() {
5824 return Some(
5825 OrderDeniedReason::ValidationFailed {
5826 detail: "Lighter orders do not support display_qty iceberg instructions"
5827 .to_string(),
5828 }
5829 .to_string(),
5830 );
5831 }
5832
5833 if is_lighter_spot_order(order, instrument) && is_lighter_conditional_order(order.order_type())
5834 {
5835 let denied = OrderDeniedReason::UnsupportedOrderType {
5836 order_type: order.order_type(),
5837 };
5838 return Some(format!(
5839 "{denied}; Lighter spot markets do not support conditional orders",
5840 ));
5841 }
5842
5843 nautilus_to_lighter_tif(
5844 order.order_type(),
5845 order.time_in_force(),
5846 order.is_post_only(),
5847 )
5848 .err()
5849 .map(|e| {
5850 let denied = OrderDeniedReason::UnsupportedTimeInForce(order.time_in_force());
5851 format!("{denied}; {e}")
5852 })
5853}
5854
5855fn unsupported_lighter_order_type_reason(order_type: OrderType) -> String {
5856 let denied = OrderDeniedReason::UnsupportedOrderType { order_type };
5857 format!(
5858 "{denied}; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
5859 )
5860}
5861
5862fn is_grouped_order(order: &OrderAny) -> bool {
5863 order.contingency_type().is_some()
5864}
5865
5866fn is_lighter_spot_order(order: &OrderAny, instrument: Option<&InstrumentAny>) -> bool {
5867 instrument.is_some_and(|instrument| matches!(instrument, InstrumentAny::CurrencyPair(_)))
5868 || product_type_from_instrument_id(&order.instrument_id()) == Some(LighterProductType::Spot)
5869}
5870
5871fn is_lighter_supported_order_type(order_type: OrderType) -> bool {
5872 matches!(
5873 order_type,
5874 OrderType::Market
5875 | OrderType::Limit
5876 | OrderType::StopMarket
5877 | OrderType::StopLimit
5878 | OrderType::MarketIfTouched
5879 | OrderType::LimitIfTouched
5880 )
5881}
5882
5883fn is_lighter_limit_style_order(order_type: OrderType) -> bool {
5884 matches!(
5885 order_type,
5886 OrderType::Limit | OrderType::StopLimit | OrderType::LimitIfTouched
5887 )
5888}
5889
5890fn is_lighter_conditional_order(order_type: OrderType) -> bool {
5891 matches!(
5892 order_type,
5893 OrderType::StopMarket
5894 | OrderType::StopLimit
5895 | OrderType::MarketIfTouched
5896 | OrderType::LimitIfTouched
5897 )
5898}
5899
5900async fn seed_active_markets_from_inactive_orders(
5901 http_client: &LighterHttpClient,
5902 dispatch: &WsDispatchState,
5903 credential: &Credential,
5904 auth: &SecretString,
5905 between_timestamps: Option<String>,
5906) -> anyhow::Result<()> {
5907 let mut cursor: Option<String> = None;
5908 let mut seen_cursors = AHashSet::new();
5909 let mut orders_seen = 0_usize;
5910 let mut pages = 0_usize;
5911
5912 loop {
5913 pages += 1;
5914 anyhow::ensure!(
5915 pages <= MAX_RECONCILIATION_PAGES,
5916 "Lighter active-market seed exceeded {MAX_RECONCILIATION_PAGES} pages",
5917 );
5918 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
5919 authorization: None,
5920 auth: Some(auth.clone()),
5921 account_index: credential.account_index(),
5922 market_id: None,
5923 ask_filter: None,
5924 between_timestamps: between_timestamps.clone(),
5925 cursor: cursor.clone(),
5926 limit: LIGHTER_REST_PAGE_SIZE,
5927 });
5928 let response = http_client
5929 .get_account_inactive_orders(&query)
5930 .await
5931 .context("failed to seed Lighter active markets from inactive orders")?;
5932
5933 for order in &response.orders {
5934 dispatch.note_active_market(order.market_index);
5935 orders_seen += 1;
5936 }
5937
5938 match response.next_cursor {
5939 Some(next) if !next.is_empty() => {
5940 anyhow::ensure!(
5941 seen_cursors.insert(next.clone()),
5942 "Lighter active-market seed repeated cursor `{next}`",
5943 );
5944 cursor = Some(next);
5945 }
5946 _ => break,
5947 }
5948 }
5949
5950 if orders_seen > 0 {
5951 log::debug!("Seeded Lighter active markets from {orders_seen} inactive order report(s)");
5952 }
5953
5954 Ok(())
5955}
5956
5957fn cancel_order_from_cancel_all(
5958 cmd: &CancelAllOrders,
5959 client_order_id: ClientOrderId,
5960 strategy_id: StrategyId,
5961) -> CancelOrder {
5962 CancelOrder {
5963 trader_id: cmd.trader_id,
5964 client_id: cmd.client_id,
5965 strategy_id,
5966 instrument_id: cmd.instrument_id,
5967 client_order_id,
5968 venue_order_id: None,
5969 command_id: cmd.command_id,
5970 ts_init: cmd.ts_init,
5971 params: cmd.params.clone(),
5972 correlation_id: cmd.correlation_id,
5973 causation_id: cmd.causation_id,
5974 }
5975}
5976
5977fn validate_order_amount(
5978 instrument: &InstrumentAny,
5979 quantity: Quantity,
5980 price_ticks: u32,
5981 price_precision: u8,
5982) -> anyhow::Result<()> {
5983 if let Some(min_quantity) = instrument.min_quantity() {
5984 anyhow::ensure!(
5985 quantity >= min_quantity,
5986 "quantity `{quantity}` below Lighter min_base_amount `{min_quantity}` for {}",
5987 instrument.id(),
5988 );
5989 }
5990
5991 if let Some(min_notional) = instrument.min_notional() {
5992 let price = decimal_from_ticks(price_ticks, price_precision);
5993 let notional = quantity.as_decimal() * price;
5994 anyhow::ensure!(
5995 notional >= min_notional.as_decimal(),
5996 "order notional `{notional}` below Lighter min_quote_amount `{}` for {}",
5997 min_notional.as_decimal(),
5998 instrument.id(),
5999 );
6000 }
6001
6002 Ok(())
6003}
6004
6005fn decimal_from_ticks(ticks: u32, decimals: u8) -> Decimal {
6006 Decimal::from(ticks) / Decimal::from(10_i64.pow(u32::from(decimals)))
6007}
6008
6009fn dispatch_lighter_order(
6013 order: &crate::http::models::LighterOrder,
6014 dispatch: &WsDispatchState,
6015 emitter: &ExecutionEventEmitter,
6016 registry: &Arc<MarketRegistry>,
6017 account_id: AccountId,
6018 trader_id: TraderId,
6019 ts_init: UnixNanos,
6020) {
6021 let instrument_id = match registry.instrument_id(order.market_index) {
6022 Some(id) => id,
6023 None => {
6024 log::debug!(
6025 "Lighter order frame dropped: no instrument for market_index={}",
6026 order.market_index,
6027 );
6028 return;
6029 }
6030 };
6031
6032 if let Some(idx) = registry.market_index(&instrument_id) {
6033 dispatch.note_active_market(idx);
6034 }
6035
6036 let instrument = match LIGHTER_INSTRUMENT_CACHE.get(&instrument_id) {
6037 Some(inst) => inst.value().clone(),
6038 None => {
6039 log::debug!("Lighter order frame dropped: instrument {instrument_id} not in cache",);
6040 return;
6041 }
6042 };
6043
6044 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
6045 let resolved_cloid =
6046 dispatch.resolve_live_order_cloid(order.client_order_id.as_str(), venue_order_id);
6047
6048 let identity = resolved_cloid.and_then(|cid| {
6049 dispatch
6050 .order_identities
6051 .get(&cid)
6052 .map(|entry| (cid, entry.value().clone()))
6053 });
6054
6055 if let Some((cloid, identity)) = identity {
6056 dispatch.venue_id_map.insert(cloid, venue_order_id);
6057
6058 let is_live_status = matches!(
6063 order.status,
6064 crate::common::enums::LighterOrderStatus::Pending
6065 | crate::common::enums::LighterOrderStatus::Open,
6066 );
6067 let pending_action = dispatch.pending_order_action(&cloid);
6068 if is_live_status && pending_action == Some(PendingOrderAction::Cancel) {
6069 log::debug!(
6070 "Deferring Lighter {:?} frame while cancel is pending for {cloid}",
6071 order.status,
6072 );
6073 return;
6074 }
6075
6076 let current_shape = match lighter_order_shape(order, &instrument, identity.order_type) {
6077 Ok(shape) => shape,
6078 Err(e) => {
6079 log::error!(
6080 "Failed to compute Lighter order shape: error={e}, voi={venue_order_id}, cloid={cloid}",
6081 );
6082 return;
6083 }
6084 };
6085 let prior_shape = dispatch.snapshot_for(&cloid);
6086 let shape_changed = prior_shape
6087 .as_ref()
6088 .is_some_and(|prev| prev != ¤t_shape);
6089 let fresh_trigger = order.status == crate::common::enums::LighterOrderStatus::Open
6090 && order.trigger_status == crate::common::enums::LighterTriggerStatus::Ready
6091 && !dispatch.triggered_was_emitted(&cloid);
6092
6093 if pending_action == Some(PendingOrderAction::Modify) && fresh_trigger && !shape_changed {
6094 log::debug!(
6095 "Deferring Lighter trigger while modify confirmation is pending for {cloid}",
6096 );
6097 return;
6098 }
6099
6100 let open_ctx = OpenFrameContext {
6101 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
6102 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
6103 shape_changed,
6104 };
6105
6106 match parse_lighter_order_event(
6107 order,
6108 &instrument,
6109 &identity,
6110 cloid,
6111 account_id,
6112 trader_id,
6113 open_ctx,
6114 ts_init,
6115 ) {
6116 Ok(event_opt) => {
6117 if is_live_status {
6127 dispatch.store_snapshot(cloid, current_shape);
6128 }
6129
6130 if let Some(event) = event_opt {
6131 dispatch_tracked_order_event(
6132 event,
6133 cloid,
6134 venue_order_id,
6135 &identity,
6136 account_id,
6137 trader_id,
6138 emitter,
6139 dispatch,
6140 ts_init,
6141 );
6142 } else if matches!(
6143 order.status,
6144 crate::common::enums::LighterOrderStatus::Filled,
6145 ) {
6146 dispatch.venue_id_map.remove(&cloid);
6147 dispatch.retire_order_identity(&cloid);
6148 }
6149 }
6150 Err(e) => {
6151 log::error!(
6152 "Failed to parse Lighter order event: error={e}, voi={venue_order_id}, cloid={cloid}",
6153 );
6154 }
6155 }
6156 } else {
6157 match parse_ws_order_status_report(order, &instrument, account_id, ts_init) {
6158 Ok(mut report) => {
6159 report = dispatch.translate_order_cloid(report);
6160 report = dispatch.preserve_pending_order_status(report);
6161
6162 if let Some(cloid) = &report.client_order_id {
6163 dispatch.venue_id_map.insert(*cloid, report.venue_order_id);
6164 }
6165
6166 if report.order_status.is_closed() {
6167 evict_terminal_mappings(&report, &dispatch.venue_id_map);
6168 }
6169
6170 log::debug!(
6171 "Lighter OrderStatusReport: voi={} status={:?} cloid={:?}",
6172 report.venue_order_id,
6173 report.order_status,
6174 report.client_order_id,
6175 );
6176 emitter.send_order_status_report(report);
6177 }
6178 Err(e) => {
6179 log::error!(
6180 "Failed to parse Lighter order status report: error={e}, order_id={}",
6181 order.order_id,
6182 );
6183 }
6184 }
6185 }
6186}
6187
6188#[expect(
6192 clippy::too_many_arguments,
6193 reason = "consumption-loop dispatch threads identity and emitter context"
6194)]
6195fn dispatch_lighter_trade(
6196 trade: &crate::http::models::LighterTrade,
6197 dispatch: &WsDispatchState,
6198 emitter: &ExecutionEventEmitter,
6199 registry: &Arc<MarketRegistry>,
6200 account_id: AccountId,
6201 trader_id: TraderId,
6202 account_index: Option<i64>,
6203 ts_init: UnixNanos,
6204) {
6205 let Some(account_index) = account_index else {
6206 log::debug!("Lighter trade frame dropped: no credential / account_index available",);
6207 return;
6208 };
6209
6210 let instrument_id = match registry.instrument_id(trade.market_id) {
6211 Some(id) => id,
6212 None => {
6213 log::debug!(
6214 "Lighter trade frame dropped: no instrument for market_id={}",
6215 trade.market_id,
6216 );
6217 return;
6218 }
6219 };
6220
6221 if let Some(idx) = registry.market_index(&instrument_id) {
6222 dispatch.note_active_market(idx);
6223 }
6224
6225 let instrument = match LIGHTER_INSTRUMENT_CACHE.get(&instrument_id) {
6226 Some(inst) => inst.value().clone(),
6227 None => {
6228 log::debug!("Lighter trade frame dropped: instrument {instrument_id} not in cache",);
6229 return;
6230 }
6231 };
6232
6233 let user_is_bidder = trade.bid_account_id == account_index;
6234 let user_is_asker = trade.ask_account_id == account_index;
6235 if !user_is_bidder && !user_is_asker {
6236 return;
6239 }
6240
6241 let trade_id = match parse_lighter_trade_id(trade) {
6244 Ok(id) => id,
6245 Err(e) => {
6246 log::error!("Lighter trade has invalid trade_id: {e}");
6247 return;
6248 }
6249 };
6250
6251 if !dispatch.mark_trade_seen(trade_id) {
6252 log::debug!("Lighter duplicate trade {trade_id} ignored (already routed)",);
6253 return;
6254 }
6255
6256 let raw_client_id = if user_is_bidder {
6257 trade
6258 .bid_client_id_str
6259 .as_deref()
6260 .map_or_else(|| trade.bid_client_id.to_string(), str::to_string)
6261 } else {
6262 trade
6263 .ask_client_id_str
6264 .as_deref()
6265 .map_or_else(|| trade.ask_client_id.to_string(), str::to_string)
6266 };
6267 let venue_order_id = if user_is_bidder {
6268 trade.bid_id_str.as_deref().map_or_else(
6269 || VenueOrderId::new(trade.bid_id.to_string()),
6270 VenueOrderId::new,
6271 )
6272 } else {
6273 trade.ask_id_str.as_deref().map_or_else(
6274 || VenueOrderId::new(trade.ask_id.to_string()),
6275 VenueOrderId::new,
6276 )
6277 };
6278 let resolved_cloid = dispatch.resolve_live_trade_cloid(raw_client_id.as_str(), venue_order_id);
6279
6280 let identity = resolved_cloid.and_then(|cid| {
6281 dispatch
6282 .order_identity(&cid)
6283 .map(|identity| (cid, identity))
6284 });
6285
6286 if let Some((cloid, identity)) = identity {
6287 ensure_accepted_emitted(
6291 cloid,
6292 venue_order_id,
6293 &identity,
6294 account_id,
6295 trader_id,
6296 emitter,
6297 dispatch,
6298 ts_init,
6299 );
6300
6301 match parse_lighter_order_filled(
6302 trade,
6303 &instrument,
6304 &identity,
6305 cloid,
6306 account_id,
6307 trader_id,
6308 account_index,
6309 ts_init,
6310 ) {
6311 Ok(Some(filled)) => {
6312 log::debug!(
6313 "Lighter OrderFilled: voi={} qty={} px={} liq={:?} cloid={cloid}",
6314 filled.venue_order_id,
6315 filled.last_qty,
6316 filled.last_px,
6317 filled.liquidity_side,
6318 );
6319 emitter.send_order_event(OrderEventAny::Filled(filled));
6320 }
6321 Ok(None) => {}
6322 Err(e) => {
6323 dispatch.unmark_trade_seen(&trade_id);
6325 log::error!("Failed to parse Lighter typed fill: error={e}, trade_id={trade_id}",);
6326 }
6327 }
6328 } else {
6329 match parse_ws_fill_report(trade, account_index, &instrument, account_id, ts_init) {
6330 Ok(Some(mut report)) => {
6331 report = dispatch.translate_fill_cloid(report);
6332 log::debug!(
6333 "Lighter FillReport: voi={} qty={} px={} liq={:?} cloid={:?}",
6334 report.venue_order_id,
6335 report.last_qty,
6336 report.last_px,
6337 report.liquidity_side,
6338 report.client_order_id,
6339 );
6340 emitter.send_fill_report(report);
6341 }
6342 Ok(None) => {}
6343 Err(e) => {
6344 dispatch.unmark_trade_seen(&trade_id);
6346 log::error!("Failed to parse Lighter fill report: error={e}, trade_id={trade_id}",);
6347 }
6348 }
6349 }
6350}
6351
6352#[expect(
6356 clippy::too_many_arguments,
6357 reason = "shared cleanup point across the typed-event variants"
6358)]
6359#[expect(
6360 clippy::needless_pass_by_value,
6361 reason = "event is destructured into typed OrderEventAny variants that consume the payload"
6362)]
6363fn dispatch_tracked_order_event(
6364 event: ParsedOrderEvent,
6365 cloid: ClientOrderId,
6366 venue_order_id: VenueOrderId,
6367 identity: &OrderIdentity,
6368 account_id: AccountId,
6369 trader_id: TraderId,
6370 emitter: &ExecutionEventEmitter,
6371 dispatch: &WsDispatchState,
6372 ts_init: UnixNanos,
6373) {
6374 let is_terminal;
6375
6376 match event {
6377 ParsedOrderEvent::Accepted(e) => {
6378 if !dispatch.claim_accepted_emission(&cloid) {
6379 log::debug!("Skipping duplicate OrderAccepted for {cloid}");
6380 return;
6381 }
6382 is_terminal = false;
6383 emitter.send_order_event(OrderEventAny::Accepted(e));
6384 }
6385 ParsedOrderEvent::Triggered(e) => {
6386 if !dispatch.mark_triggered_emitted(cloid) {
6387 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
6388 return;
6389 }
6390 ensure_accepted_emitted(
6391 cloid,
6392 venue_order_id,
6393 identity,
6394 account_id,
6395 trader_id,
6396 emitter,
6397 dispatch,
6398 ts_init,
6399 );
6400 is_terminal = false;
6401 emitter.send_order_event(OrderEventAny::Triggered(e));
6402 }
6403 ParsedOrderEvent::Updated(e) => {
6404 dispatch.clear_pending_order_action_if(&cloid, PendingOrderAction::Modify);
6409 is_terminal = false;
6410 emitter.send_order_event(OrderEventAny::Updated(e));
6411 }
6412 ParsedOrderEvent::UpdatedThenTriggered { updated, triggered } => {
6413 if !dispatch.mark_triggered_emitted(cloid) {
6414 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
6415 return;
6416 }
6417 ensure_accepted_emitted(
6418 cloid,
6419 venue_order_id,
6420 identity,
6421 account_id,
6422 trader_id,
6423 emitter,
6424 dispatch,
6425 ts_init,
6426 );
6427 dispatch.clear_pending_order_action_if(&cloid, PendingOrderAction::Modify);
6428 is_terminal = false;
6429 emitter.send_order_event(OrderEventAny::Updated(updated));
6430 emitter.send_order_event(OrderEventAny::Triggered(triggered));
6431 }
6432 ParsedOrderEvent::Canceled(e) => {
6433 ensure_accepted_emitted(
6434 cloid,
6435 venue_order_id,
6436 identity,
6437 account_id,
6438 trader_id,
6439 emitter,
6440 dispatch,
6441 ts_init,
6442 );
6443 is_terminal = true;
6444 emitter.send_order_event(OrderEventAny::Canceled(e));
6445 }
6446 ParsedOrderEvent::Expired(e) => {
6447 ensure_accepted_emitted(
6448 cloid,
6449 venue_order_id,
6450 identity,
6451 account_id,
6452 trader_id,
6453 emitter,
6454 dispatch,
6455 ts_init,
6456 );
6457 is_terminal = true;
6458 emitter.send_order_event(OrderEventAny::Expired(e));
6459 }
6460 ParsedOrderEvent::Rejected(e) => {
6461 is_terminal = true;
6462 emitter.send_order_event(OrderEventAny::Rejected(e));
6463 }
6464 }
6465
6466 if is_terminal {
6467 dispatch.venue_id_map.remove(&cloid);
6468 dispatch.retire_order_identity(&cloid);
6469 }
6470}
6471
6472#[expect(
6475 clippy::too_many_arguments,
6476 reason = "synthesized events need the full identity context to populate the event"
6477)]
6478fn ensure_accepted_emitted(
6479 cloid: ClientOrderId,
6480 venue_order_id: VenueOrderId,
6481 identity: &OrderIdentity,
6482 account_id: AccountId,
6483 trader_id: TraderId,
6484 emitter: &ExecutionEventEmitter,
6485 dispatch: &WsDispatchState,
6486 ts_init: UnixNanos,
6487) {
6488 if !dispatch.claim_accepted_emission(&cloid) {
6489 return;
6490 }
6491 let accepted = OrderAccepted::new(
6492 trader_id,
6493 identity.strategy_id,
6494 identity.instrument_id,
6495 cloid,
6496 venue_order_id,
6497 account_id,
6498 UUID4::new(),
6499 ts_init,
6500 ts_init,
6501 false,
6502 );
6503 emitter.send_order_event(OrderEventAny::Accepted(accepted));
6504}
6505
6506#[cfg(test)]
6507mod tests {
6508 use std::{
6509 cell::RefCell,
6510 rc::Rc,
6511 sync::{Arc, atomic::AtomicUsize},
6512 };
6513
6514 use axum::{
6515 Router,
6516 routing::{get, post},
6517 };
6518 use nautilus_common::{
6519 cache::Cache,
6520 clock::VirtualClock,
6521 factories::OrderFactory,
6522 messages::{ExecutionEvent, ExecutionReport as EngineExecutionReport},
6523 testing::wait_until_async,
6524 };
6525 use nautilus_model::{
6526 data::QuoteTick,
6527 enums::{ContingencyType, LiquiditySide, OrderSide, OrderStatus, TimeInForce},
6528 events::{OrderCanceled, OrderEventAny, OrderPendingCancel, OrderTriggered},
6529 identifiers::{
6530 InstrumentId, OrderListId, StrategyId, Symbol, TradeId, TraderId, VenueOrderId,
6531 },
6532 instruments::CryptoPerpetual,
6533 orders::{LimitOrder, OrderList, OrderTestBuilder, stubs::TestOrderEventStubs},
6534 types::{Currency, Money, Price},
6535 };
6536 use rstest::rstest;
6537
6538 use super::*;
6539 use crate::{
6540 common::{
6541 consts::LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
6542 enums::{LighterDeployment, LighterEnvironment, LighterProductType},
6543 },
6544 http::models::{LighterNextNonce, LighterTx},
6545 signing::tx::TX_HASH_BYTES,
6546 };
6547
6548 const TEST_PRIVATE_KEY: &str =
6549 "0b8e0f63c24d8baacd9d29ad4e9a4b73c4a8d2bb8b16dc4fa9d7c2e1d3a8b1f0e8d3a4c5b6e7f001";
6550 const TEST_ACCOUNT_INDEX: u64 = 12345;
6551 const TEST_ACCOUNT_INDEX_I64: i64 = 12345;
6552 const TEST_API_KEY_INDEX: u8 = 5;
6553 const TEST_NEXT_NONCE: i64 = 42;
6554 const TEST_MARKET_INDEX: i64 = 0;
6555 const TEST_ORDER_NONCE: i64 = 281_474_720_725_346;
6556 const TEST_SUBMISSION_NONCE: i64 = 2_042;
6557
6558 fn handle_send_tx_ack(
6559 dispatch: &WsDispatchState,
6560 account_index: Option<i64>,
6561 code: i64,
6562 tx_hash: Option<&str>,
6563 ) -> Option<PendingSendTx> {
6564 handle_send_tx_ack_for_connection(dispatch, account_index, 0, code, tx_hash)
6565 }
6566
6567 #[expect(
6568 clippy::too_many_arguments,
6569 reason = "test wrapper preserves the established rejection call shape"
6570 )]
6571 fn handle_send_tx_rejection(
6572 dispatch: &WsDispatchState,
6573 emitter: &ExecutionEventEmitter,
6574 account_index: Option<i64>,
6575 now: UnixNanos,
6576 source: SendTxRejectionSource,
6577 code: Option<i64>,
6578 message: &str,
6579 tx_hash: Option<&str>,
6580 ) -> bool {
6581 handle_send_tx_rejection_for_connection(
6582 dispatch,
6583 emitter,
6584 account_index,
6585 0,
6586 now,
6587 source,
6588 code,
6589 message,
6590 tx_hash,
6591 )
6592 }
6593
6594 fn trader_id() -> TraderId {
6595 TraderId::from("TRADER-001")
6596 }
6597
6598 fn client_id() -> ClientId {
6599 ClientId::from("LIGHTER")
6600 }
6601
6602 fn account_id() -> AccountId {
6603 AccountId::from("LIGHTER-001")
6604 }
6605
6606 fn strategy_id() -> StrategyId {
6607 StrategyId::from("S-001")
6608 }
6609
6610 fn test_credential() -> Credential {
6611 Credential::new(TEST_API_KEY_INDEX, TEST_PRIVATE_KEY, TEST_ACCOUNT_INDEX).unwrap()
6612 }
6613
6614 fn test_config() -> LighterExecutionClientConfig {
6615 test_config_with_use_gtd(true)
6616 }
6617
6618 fn test_config_with_use_gtd(use_gtd: bool) -> LighterExecutionClientConfig {
6619 LighterExecutionClientConfig {
6620 account_id: account_id(),
6621 account_index: Some(TEST_ACCOUNT_INDEX),
6622 api_key_index: Some(TEST_API_KEY_INDEX),
6623 private_key: Some(TEST_PRIVATE_KEY.into()),
6624 base_url_http: Some("http://127.0.0.1:1".to_string()),
6625 base_url_ws: Some("ws://127.0.0.1:1/stream".to_string()),
6626 proxy_url: None,
6627 environment: LighterEnvironment::Testnet,
6628 deployment: LighterDeployment::Lighter,
6629 venue: None,
6630 http_timeout_secs: 1,
6631 ws_timeout_secs: 1,
6632 market_order_slippage_bps: 50,
6633 rest_quota_per_min: None,
6634 sendtx_quota_per_min: None,
6635 transport_backend: Default::default(),
6636 use_gtd,
6637 }
6638 }
6639
6640 #[rstest]
6641 fn format_between_timestamps_uses_lighter_seconds_range() {
6642 let start = UnixNanos::from(1_700_000_000_123_456_789);
6643 let end = UnixNanos::from(1_700_003_600_987_654_321);
6644 let now = UnixNanos::from(1_700_007_200_000_000_000);
6645
6646 assert_eq!(
6647 format_between_timestamps(Some(start), Some(end), now),
6648 Some("1700000000-1700003600".to_string()),
6649 );
6650 assert_eq!(
6651 format_between_timestamps(Some(start), None, now),
6652 Some("1700000000-1700007200".to_string()),
6653 );
6654 assert_eq!(
6655 format_between_timestamps(None, Some(end), now),
6656 Some("0-1700003600".to_string()),
6657 );
6658 assert_eq!(format_between_timestamps(None, None, now), None);
6659 }
6660
6661 #[rstest]
6662 #[case::write_timeout(LighterWsError::Transport(SendError::WriteTimeout), "Ambiguous")]
6663 #[case::broken_pipe(
6664 LighterWsError::Transport(SendError::BrokenPipe("writer closed".to_string())),
6665 "Ambiguous",
6666 )]
6667 #[case::handler_result_lost(
6668 LighterWsError::SendTxOutcomeUnknown("result sender dropped".to_string()),
6669 "Ambiguous",
6670 )]
6671 #[case::network(LighterWsError::Network("disconnected".to_string()), "Ambiguous")]
6672 #[case::parse(LighterWsError::Parse("invalid ack".to_string()), "Ambiguous")]
6673 #[case::invalid_input(
6674 LighterWsError::Transport(SendError::InvalidInput("invalid payload".to_string())),
6675 "NotSent",
6676 )]
6677 #[case::closed(LighterWsError::Transport(SendError::Closed), "NotSent")]
6678 #[case::connection_changed(LighterWsError::Transport(SendError::ConnectionChanged), "NotSent")]
6679 #[case::wait_timeout(LighterWsError::Transport(SendError::Timeout), "NotSent")]
6680 #[case::authentication(
6681 LighterWsError::Authentication("invalid token".to_string()),
6682 "NotSent",
6683 )]
6684 #[case::client_unavailable(
6685 LighterWsError::Client("handler unavailable".to_string()),
6686 "NotSent",
6687 )]
6688 fn ws_command_failure_classifies_delivery_evidence(
6689 #[case] error: LighterWsError,
6690 #[case] expected: &str,
6691 ) {
6692 let failure = classify_lighter_ws_command_failure("submit_order", &error);
6693 let actual = match &failure {
6694 CommandFailure::NotSent(_) => "NotSent",
6695 CommandFailure::Ambiguous(_) => "Ambiguous",
6696 CommandFailure::VenueRejected(_) => "VenueRejected",
6697 };
6698
6699 assert_eq!(actual, expected);
6700 assert!(command_failure_reason(&failure).contains("submit_order"));
6701 }
6702
6703 #[rstest]
6704 fn venue_rejection_reason_is_clean_and_bounded() {
6705 let oversized = "x".repeat(STRATEGY_REASON_MAX_CHARS + 100);
6706 let message = format!("<html>\n rejected\0 because {oversized}</html>");
6707
6708 let failure = venue_rejection_failure(Some(20_001), &message);
6709 let reason = command_failure_reason(&failure);
6710
6711 assert!(matches!(&failure, CommandFailure::VenueRejected(_)));
6712 assert!(!reason.contains('<'));
6713 assert!(!reason.contains('\0'));
6714 assert!(!reason.contains(" "));
6715 assert_eq!(reason.chars().count(), STRATEGY_REASON_MAX_CHARS);
6716 assert!(reason.starts_with("LIGHTER_20001: rejected because "));
6717 }
6718
6719 #[rstest]
6720 fn venue_rejection_reason_uses_stable_fallback_for_empty_markup() {
6721 let failure = venue_rejection_failure(Some(20_001), "<html></html>\n");
6722
6723 assert_eq!(
6724 failure,
6725 CommandFailure::VenueRejected("LIGHTER_20001".to_string()),
6726 );
6727 }
6728
6729 #[rstest]
6730 fn venue_rejection_reason_preserves_comparison_and_removes_format_controls() {
6731 let failure = venue_rejection_failure(
6732 Some(20_001),
6733 "price must be < 100\u{202e}\u{2066}\u{200b}<strong>now</strong>",
6734 );
6735
6736 assert_eq!(
6737 failure,
6738 CommandFailure::VenueRejected("LIGHTER_20001: price must be < 100 now".to_string(),),
6739 );
6740 }
6741
6742 #[rstest]
6743 fn unsupported_order_type_reason_keeps_lighter_capabilities() {
6744 assert_eq!(
6745 unsupported_lighter_order_type_reason(OrderType::MarketToLimit),
6746 "UNSUPPORTED_ORDER_TYPE: MARKET_TO_LIMIT; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
6747 );
6748 }
6749
6750 #[rstest]
6751 fn mass_status_propagates_commission_error_and_downgrades_other_fill_errors() {
6752 let e = anyhow::Error::new(LighterCommissionError::new("invalid precision"))
6753 .context("failed to parse Lighter fill report");
6754
6755 let commission_error = fill_reports_for_mass_status(Err(e))
6756 .expect_err("commission construction must fail mass status");
6757 let ordinary_error = fill_reports_for_mass_status(Err(anyhow::anyhow!("transport failed")))
6758 .expect("ordinary source errors make mass status incomplete");
6759 let complete = fill_reports_for_mass_status(Ok(FillSweep {
6760 reports: Vec::new(),
6761 covers_window: true,
6762 }))
6763 .expect("complete fill sweep");
6764
6765 assert!(is_commission_error(&commission_error));
6766 assert!(ordinary_error.0.is_empty());
6767 assert!(!ordinary_error.1);
6768 assert!(complete.0.is_empty());
6769 assert!(complete.1);
6770 }
6771
6772 fn create_execution_client() -> (
6773 LighterExecutionClient,
6774 Rc<RefCell<Cache>>,
6775 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
6776 ) {
6777 create_execution_client_with_config(test_config())
6778 }
6779
6780 fn create_execution_client_with_config(
6781 config: LighterExecutionClientConfig,
6782 ) -> (
6783 LighterExecutionClient,
6784 Rc<RefCell<Cache>>,
6785 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
6786 ) {
6787 let cache = Rc::new(RefCell::new(Cache::default()));
6788 let venue = config.resolved_venue();
6789 let core = ExecutionClientCore::new(
6790 trader_id(),
6791 client_id(),
6792 venue,
6793 OmsType::Netting,
6794 account_id(),
6795 AccountType::Margin,
6796 None,
6797 cache.clone(),
6798 );
6799
6800 let mut client = LighterExecutionClient::new(core, config).unwrap();
6801 client.dispatch.nonce_manager.refresh(
6802 TEST_ACCOUNT_INDEX_I64,
6803 TEST_API_KEY_INDEX,
6804 TEST_NEXT_NONCE,
6805 );
6806
6807 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
6808 client.emitter.set_sender(sender);
6809
6810 (client, cache, receiver)
6811 }
6812
6813 fn register_test_instrument(
6814 client: &LighterExecutionClient,
6815 cache: &Rc<RefCell<Cache>>,
6816 ) -> InstrumentId {
6817 let instrument_id =
6818 client
6819 .registry
6820 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
6821 let instrument = InstrumentAny::CryptoPerpetual(
6822 CryptoPerpetual::builder()
6823 .instrument_id(instrument_id)
6824 .raw_symbol(Symbol::new("ETH-PERP"))
6825 .base_currency(Currency::from("ETH"))
6826 .quote_currency(Currency::from("USDC"))
6827 .settlement_currency(Currency::from("USDC"))
6828 .is_inverse(false)
6829 .price_precision(2)
6830 .size_precision(4)
6831 .price_increment(Price::from("0.01"))
6832 .size_increment(Quantity::from("0.0001"))
6833 .min_notional(Money::from("10.000000 USDC"))
6834 .ts_event(UnixNanos::default())
6835 .ts_init(UnixNanos::default())
6836 .build()
6837 .unwrap(),
6838 );
6839
6840 cache.borrow_mut().add_instrument(instrument).unwrap();
6841
6842 instrument_id
6843 }
6844
6845 fn test_order_factory() -> OrderFactory {
6846 let clock = Rc::new(RefCell::new(VirtualClock::new()));
6847 OrderFactory::new(
6848 trader_id(),
6849 strategy_id(),
6850 Some(0),
6851 Some(0),
6852 clock,
6853 false,
6854 false,
6855 )
6856 }
6857
6858 fn test_limit_order(
6859 factory: &mut OrderFactory,
6860 instrument_id: InstrumentId,
6861 client_order_id: &str,
6862 ) -> OrderAny {
6863 test_limit_order_with(
6864 factory,
6865 instrument_id,
6866 client_order_id,
6867 OrderSide::Buy,
6868 TimeInForce::Gtc,
6869 None,
6870 false,
6871 )
6872 }
6873
6874 fn test_limit_order_with(
6875 factory: &mut OrderFactory,
6876 instrument_id: InstrumentId,
6877 client_order_id: &str,
6878 side: OrderSide,
6879 tif: TimeInForce,
6880 expire_time: Option<UnixNanos>,
6881 reduce_only: bool,
6882 ) -> OrderAny {
6883 factory.limit(
6884 instrument_id,
6885 side,
6886 Quantity::from("0.1000"),
6887 Price::from("2361.31"),
6888 Some(tif),
6889 expire_time,
6890 Some(false),
6891 Some(reduce_only),
6892 None,
6893 None,
6894 None,
6895 None,
6896 None,
6897 None,
6898 None,
6899 Some(ClientOrderId::from(client_order_id)),
6900 )
6901 }
6902
6903 fn cache_order(cache: &Rc<RefCell<Cache>>, order: OrderAny) {
6904 cache
6905 .borrow_mut()
6906 .add_order(order, None, Some(client_id()), false)
6907 .unwrap();
6908 }
6909
6910 fn cache_accepted_order(
6911 cache: &Rc<RefCell<Cache>>,
6912 order: OrderAny,
6913 venue_order_id: VenueOrderId,
6914 client_id: Option<ClientId>,
6915 ) -> (InstrumentId, ClientOrderId) {
6916 let instrument_id = order.instrument_id();
6917 let client_order_id = order.client_order_id();
6918 cache
6919 .borrow_mut()
6920 .add_order(order, None, client_id, false)
6921 .unwrap();
6922
6923 let accepted = OrderEventAny::Accepted(OrderAccepted::new(
6924 trader_id(),
6925 strategy_id(),
6926 instrument_id,
6927 client_order_id,
6928 venue_order_id,
6929 account_id(),
6930 UUID4::new(),
6931 UnixNanos::default(),
6932 UnixNanos::default(),
6933 false,
6934 ));
6935 cache.borrow_mut().update_order(&accepted).unwrap();
6936
6937 (instrument_id, client_order_id)
6938 }
6939
6940 fn cache_pending_cancel_order(
6941 cache: &Rc<RefCell<Cache>>,
6942 order: OrderAny,
6943 venue_order_id: VenueOrderId,
6944 ) {
6945 let (instrument_id, client_order_id) =
6946 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
6947
6948 let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
6949 trader_id(),
6950 strategy_id(),
6951 instrument_id,
6952 client_order_id,
6953 Some(account_id()),
6954 UUID4::new(),
6955 UnixNanos::default(),
6956 UnixNanos::default(),
6957 false,
6958 Some(venue_order_id),
6959 ));
6960 cache.borrow_mut().update_order(&pending_cancel).unwrap();
6961 }
6962
6963 fn submit_order_list_command(orders: &[OrderAny], order_list_id: &str) -> SubmitOrderList {
6964 let order_list = OrderList::new(
6965 OrderListId::from(order_list_id),
6966 orders[0].instrument_id(),
6967 strategy_id(),
6968 orders.iter().map(|order| order.client_order_id()).collect(),
6969 UnixNanos::default(),
6970 );
6971 let order_inits = orders
6972 .iter()
6973 .map(|order| order.init_event().clone())
6974 .collect();
6975
6976 SubmitOrderList::new(
6977 trader_id(),
6978 Some(client_id()),
6979 strategy_id(),
6980 order_list,
6981 order_inits,
6982 None,
6983 None,
6984 None,
6985 UUID4::new(),
6986 UnixNanos::default(),
6987 None,
6988 )
6989 }
6990
6991 fn test_contingent_limit_order(
6992 instrument_id: InstrumentId,
6993 client_order_id: &str,
6994 order_list_id: &str,
6995 linked_order_id: &str,
6996 ) -> OrderAny {
6997 OrderAny::Limit(LimitOrder::new(
6998 trader_id(),
6999 strategy_id(),
7000 instrument_id,
7001 ClientOrderId::from(client_order_id),
7002 OrderSide::Buy,
7003 Quantity::from("0.1000"),
7004 Price::from("2361.31"),
7005 TimeInForce::Gtc,
7006 None,
7007 false,
7008 false,
7009 false,
7010 None,
7011 None,
7012 None,
7013 Some(ContingencyType::Oco),
7014 Some(OrderListId::from(order_list_id)),
7015 Some(vec![ClientOrderId::from(linked_order_id)]),
7016 None,
7017 None,
7018 None,
7019 None,
7020 None,
7021 UUID4::new(),
7022 UnixNanos::default(),
7023 ))
7024 }
7025
7026 async fn recv_order_event(
7027 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
7028 ) -> OrderEventAny {
7029 let event = tokio::time::timeout(Duration::from_secs(2), rx.recv())
7030 .await
7031 .expect("timed out waiting for execution event")
7032 .expect("execution event channel closed");
7033
7034 match event {
7035 ExecutionEvent::Order(event) => event,
7036 event => panic!("expected order event, was {event:?}"),
7037 }
7038 }
7039
7040 async fn assert_modify_rejected_reason(
7041 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
7042 reason_part: &str,
7043 ) {
7044 match recv_order_event(rx).await {
7045 OrderEventAny::ModifyRejected(event) => {
7046 assert!(
7047 event.reason.contains(reason_part),
7048 "expected modify rejection containing `{reason_part}`, was `{}`",
7049 event.reason,
7050 );
7051 }
7052 event => panic!("expected modify rejected event, was {event:?}"),
7053 }
7054 }
7055
7056 fn assert_nonce_reusable(dispatch: &WsDispatchState) {
7057 assert_eq!(
7058 dispatch
7059 .nonce_manager
7060 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
7061 Some(TEST_NEXT_NONCE - 1),
7062 );
7063 assert_eq!(
7064 dispatch
7065 .nonce_manager
7066 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
7067 .unwrap(),
7068 TEST_NEXT_NONCE,
7069 );
7070 }
7071
7072 #[tokio::test]
7073 async fn auth_token_rotation_retries_failed_mint() {
7074 let credential = test_credential();
7075 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
7076 let cancellation_token = CancellationToken::new();
7077 let mint_attempts = Arc::new(AtomicUsize::new(0));
7078 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
7079
7080 let outcome = tokio::time::timeout(
7081 Duration::from_secs(3),
7082 refresh_auth_token_until_rotated(
7083 &credential,
7084 &channels,
7085 &cancellation_token,
7086 AuthTokenRefreshBackoff {
7087 initial_delay: Duration::from_millis(10),
7088 max_delay: Duration::from_millis(20),
7089 window: Duration::from_secs(1),
7090 },
7091 {
7092 let mint_attempts = Arc::clone(&mint_attempts);
7093 move |_| {
7094 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
7095 if attempt == 0 {
7096 Err(anyhow::anyhow!("mint unavailable"))
7097 } else {
7098 Ok(format!("token-{attempt}").into())
7099 }
7100 }
7101 },
7102 {
7103 let subscribe_attempts = Arc::clone(&subscribe_attempts);
7104 move |_channel, _token| {
7105 let subscribe_attempts = Arc::clone(&subscribe_attempts);
7106 async move {
7107 subscribe_attempts.fetch_add(1, Ordering::AcqRel);
7108 Ok::<(), crate::websocket::error::LighterWsError>(())
7109 }
7110 }
7111 },
7112 ),
7113 )
7114 .await
7115 .expect("rotation retry must complete within the test window");
7116
7117 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
7118 assert_eq!(
7119 mint_attempts.load(Ordering::Acquire),
7120 2,
7121 "failed mint must retry before the next refresh interval",
7122 );
7123 assert_eq!(
7124 subscribe_attempts.load(Ordering::Acquire),
7125 channels.len(),
7126 "subscriptions must wait until token mint succeeds",
7127 );
7128 }
7129
7130 #[tokio::test]
7131 async fn auth_token_rotation_retries_failed_resubscribe() {
7132 let credential = test_credential();
7133 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
7134 let cancellation_token = CancellationToken::new();
7135 let mint_attempts = Arc::new(AtomicUsize::new(0));
7136 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
7137
7138 let outcome = tokio::time::timeout(
7139 Duration::from_secs(3),
7140 refresh_auth_token_until_rotated(
7141 &credential,
7142 &channels,
7143 &cancellation_token,
7144 AuthTokenRefreshBackoff {
7145 initial_delay: Duration::from_millis(10),
7146 max_delay: Duration::from_millis(20),
7147 window: Duration::from_secs(1),
7148 },
7149 {
7150 let mint_attempts = Arc::clone(&mint_attempts);
7151 move |_| {
7152 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
7153 Ok(format!("token-{attempt}").into())
7154 }
7155 },
7156 {
7157 let subscribe_attempts = Arc::clone(&subscribe_attempts);
7158 move |_channel, _token| {
7159 let subscribe_attempts = Arc::clone(&subscribe_attempts);
7160 async move {
7161 let attempt = subscribe_attempts.fetch_add(1, Ordering::AcqRel);
7162 if attempt == 0 {
7163 Err(crate::websocket::error::LighterWsError::Client(
7164 "handler unavailable".to_string(),
7165 ))
7166 } else {
7167 Ok(())
7168 }
7169 }
7170 }
7171 },
7172 ),
7173 )
7174 .await
7175 .expect("rotation retry must complete within the test window");
7176
7177 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
7178 assert_eq!(
7179 mint_attempts.load(Ordering::Acquire),
7180 2,
7181 "failed resubscribe must trigger a fresh auth-token mint",
7182 );
7183 assert_eq!(
7184 subscribe_attempts.load(Ordering::Acquire),
7185 channels.len() * 2,
7186 "failed resubscribe must retry the private account-channel set",
7187 );
7188 }
7189
7190 #[tokio::test]
7191 async fn auth_token_rotation_exhausts_retry_window() {
7192 let credential = test_credential();
7193 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
7194 let cancellation_token = CancellationToken::new();
7195 let mint_attempts = Arc::new(AtomicUsize::new(0));
7196
7197 let refresh = refresh_auth_token_until_rotated(
7198 &credential,
7199 &channels,
7200 &cancellation_token,
7201 AuthTokenRefreshBackoff {
7202 initial_delay: Duration::from_millis(100),
7203 max_delay: Duration::from_millis(100),
7204 window: Duration::from_secs(1),
7205 },
7206 {
7207 let mint_attempts = Arc::clone(&mint_attempts);
7208 move |_| {
7209 mint_attempts.fetch_add(1, Ordering::AcqRel);
7210 Err(anyhow::anyhow!("mint unavailable"))
7211 }
7212 },
7213 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
7214 );
7215 let observe_retry = wait_until_async(
7216 || async { mint_attempts.load(Ordering::Acquire) > 1 },
7217 Duration::from_secs(2),
7218 );
7219
7220 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(3), async {
7221 tokio::join!(refresh, observe_retry)
7222 })
7223 .await
7224 .expect("rotation retry exhaustion must complete within the test window");
7225
7226 assert_eq!(outcome, AuthTokenRefreshOutcome::Exhausted);
7227 assert!(
7228 mint_attempts.load(Ordering::Acquire) > 1,
7229 "persistent mint failure must retry until the window is exhausted",
7230 );
7231 }
7232
7233 #[tokio::test]
7234 async fn auth_token_rotation_cancels_during_retry_backoff() {
7235 let credential = test_credential();
7236 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
7237 let cancellation_token = CancellationToken::new();
7238 let mint_attempts = Arc::new(AtomicUsize::new(0));
7239
7240 let cancel = cancellation_token.clone();
7241 let cancel_after_first_attempt = wait_until_async(
7242 || async { mint_attempts.load(Ordering::Acquire) > 0 },
7243 Duration::from_secs(2),
7244 );
7245
7246 let refresh = refresh_auth_token_until_rotated(
7247 &credential,
7248 &channels,
7249 &cancellation_token,
7250 AuthTokenRefreshBackoff {
7251 initial_delay: Duration::from_secs(2),
7252 max_delay: Duration::from_secs(2),
7253 window: Duration::from_secs(5),
7254 },
7255 {
7256 let mint_attempts = Arc::clone(&mint_attempts);
7257 move |_| {
7258 mint_attempts.fetch_add(1, Ordering::AcqRel);
7259 Err(anyhow::anyhow!("mint unavailable"))
7260 }
7261 },
7262 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
7263 );
7264
7265 let cancel_task = async move {
7266 cancel_after_first_attempt.await;
7267 cancel.cancel();
7268 };
7269
7270 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(6), async {
7271 tokio::join!(refresh, cancel_task)
7272 })
7273 .await
7274 .expect("rotation cancellation must complete within the test window");
7275
7276 assert_eq!(outcome, AuthTokenRefreshOutcome::Cancelled);
7277 assert_eq!(
7278 mint_attempts.load(Ordering::Acquire),
7279 1,
7280 "cancellation during backoff must stop before the next retry",
7281 );
7282 }
7283
7284 #[rstest]
7285 #[case::rotated(AuthTokenRefreshOutcome::Rotated, Some(AUTH_TOKEN_REFRESH_INTERVAL))]
7286 #[case::cancelled(AuthTokenRefreshOutcome::Cancelled, None)]
7287 #[case::exhausted(
7288 AuthTokenRefreshOutcome::Exhausted,
7289 Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY)
7290 )]
7291 fn auth_token_refresh_next_delay_matches_outcome(
7292 #[case] outcome: AuthTokenRefreshOutcome,
7293 #[case] expected: Option<Duration>,
7294 ) {
7295 assert_eq!(auth_token_refresh_next_delay(outcome), expected);
7296 }
7297
7298 #[rstest]
7299 fn auth_token_rotation_channels_match_private_account_streams() {
7300 assert_eq!(
7301 auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64),
7302 [
7303 LighterWsChannel::AccountAllOrders(TEST_ACCOUNT_INDEX_I64),
7304 LighterWsChannel::AccountAllTrades(TEST_ACCOUNT_INDEX_I64),
7305 LighterWsChannel::AccountAllPositions(TEST_ACCOUNT_INDEX_I64),
7306 LighterWsChannel::AccountAllAssets(TEST_ACCOUNT_INDEX_I64),
7307 LighterWsChannel::UserStats(TEST_ACCOUNT_INDEX_I64),
7308 ],
7309 );
7310 }
7311
7312 #[rstest]
7313 #[case::below_max(
7314 Duration::from_millis(10),
7315 Duration::from_millis(100),
7316 Duration::from_millis(20)
7317 )]
7318 #[case::at_max(
7319 Duration::from_millis(100),
7320 Duration::from_millis(100),
7321 Duration::from_millis(100)
7322 )]
7323 #[case::overflow_clamps(Duration::MAX, Duration::from_secs(300), Duration::from_secs(300))]
7324 fn next_auth_token_refresh_retry_delay_doubles_and_caps(
7325 #[case] current: Duration,
7326 #[case] max: Duration,
7327 #[case] expected: Duration,
7328 ) {
7329 assert_eq!(next_auth_token_refresh_retry_delay(current, max), expected);
7330 }
7331
7332 #[rstest]
7333 #[tokio::test]
7334 async fn auth_token_refresh_is_retained_until_session_shutdown() {
7335 let (mut client, _cache, _rx) = create_execution_client();
7336
7337 client
7338 .spawn_auth_token_refresh(test_credential())
7339 .expect("spawn auth token refresh");
7340 client
7341 .nonce_recovery_inflight
7342 .store(true, Ordering::Release);
7343
7344 assert!(
7345 client
7346 .auth_refresh_handle
7347 .as_ref()
7348 .is_some_and(|handle| !handle.is_finished()),
7349 );
7350
7351 client.begin_session_shutdown();
7352 client
7353 .finish_session_shutdown()
7354 .await
7355 .expect("session shutdown");
7356
7357 assert!(client.auth_refresh_handle.is_none());
7358 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
7359 }
7360
7361 #[rstest]
7362 #[tokio::test]
7363 async fn session_shutdown_rejects_tasks_registered_during_abort() {
7364 struct SpawnOnDrop {
7365 pending_tasks: TaskSpawner,
7366 rejected: Arc<AtomicBool>,
7367 polled: Arc<AtomicBool>,
7368 }
7369
7370 impl Drop for SpawnOnDrop {
7371 fn drop(&mut self) {
7372 let polled = Arc::clone(&self.polled);
7373 let result = self.pending_tasks.spawn(async move {
7374 polled.store(true, Ordering::Release);
7375 });
7376 self.rejected.store(result.is_err(), Ordering::Release);
7377 }
7378 }
7379
7380 let (mut client, _cache, _rx) = create_execution_client();
7381 let rejected = Arc::new(AtomicBool::new(false));
7382 let polled = Arc::new(AtomicBool::new(false));
7383 let pending_spawner = client.pending_tasks.spawner().expect("task spawner");
7384 let guard = SpawnOnDrop {
7385 pending_tasks: pending_spawner.clone(),
7386 rejected: Arc::clone(&rejected),
7387 polled: Arc::clone(&polled),
7388 };
7389
7390 pending_spawner
7391 .spawn(async move {
7392 let _guard = guard;
7393 std::future::pending::<()>().await;
7394 })
7395 .expect("parent task spawn");
7396
7397 client
7398 .finish_session_shutdown()
7399 .await
7400 .expect("session shutdown");
7401
7402 assert!(client.pending_tasks.is_empty());
7403 assert!(rejected.load(Ordering::Acquire));
7404 assert!(!polled.load(Ordering::Acquire));
7405 }
7406
7407 #[rstest]
7408 #[tokio::test]
7409 async fn session_shutdown_discards_pending_sendtx_before_epoch_reuse() {
7410 struct EnqueueOnDrop {
7411 dispatch: WsDispatchState,
7412 pending: Option<PendingSendTx>,
7413 }
7414
7415 impl Drop for EnqueueOnDrop {
7416 fn drop(&mut self) {
7417 self.dispatch
7418 .enqueue_pending_sendtx(self.pending.take().expect("pending sendTx"));
7419 }
7420 }
7421
7422 let (mut client, _cache, _rx) = create_execution_client();
7423 let old_epoch = client.ws_client.connection_epoch();
7424 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
7425 batch_id: None,
7426 connection_epoch: old_epoch,
7427 kind: PendingSendTxKind::Other,
7428 submitted_at: UnixNanos::default(),
7429 nonce: TEST_NEXT_NONCE,
7430 api_key_index: TEST_API_KEY_INDEX,
7431 tx_hash: "shutdown-pending".to_string(),
7432 });
7433 let guard = EnqueueOnDrop {
7434 dispatch: client.dispatch.clone(),
7435 pending: Some(PendingSendTx {
7436 batch_id: None,
7437 connection_epoch: old_epoch,
7438 kind: PendingSendTxKind::Other,
7439 submitted_at: UnixNanos::default(),
7440 nonce: TEST_NEXT_NONCE + 1,
7441 api_key_index: TEST_API_KEY_INDEX,
7442 tx_hash: "shutdown-late".to_string(),
7443 }),
7444 };
7445
7446 client
7447 .pending_tasks
7448 .spawn(async move {
7449 let _guard = guard;
7450 std::future::pending::<()>().await;
7451 })
7452 .expect("pending sendTx producer spawn");
7453 client
7454 .ws_stream_handle
7455 .insert(get_runtime().spawn(std::future::pending()));
7456
7457 client.begin_session_shutdown();
7458 client
7459 .finish_session_shutdown()
7460 .await
7461 .expect("session shutdown");
7462
7463 assert_eq!(client.ws_client.connection_epoch(), old_epoch);
7464 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
7465 }
7466
7467 #[rstest]
7468 fn replacing_ws_client_preserves_cached_instruments() {
7469 let (mut client, cache, _rx) = create_execution_client();
7470 let instrument_id = register_test_instrument(&client, &cache);
7471 let instrument = cache
7472 .borrow()
7473 .instrument(&instrument_id)
7474 .expect("instrument")
7475 .clone();
7476 client
7477 .ws_client
7478 .cache_instrument(TEST_MARKET_INDEX, instrument.clone());
7479
7480 let _old_ws_client = client.take_ws_client();
7481
7482 let cached_instruments = client.ws_client.instruments_cache();
7483 assert_eq!(
7484 cached_instruments
7485 .get(&TEST_MARKET_INDEX)
7486 .expect("cached instrument")
7487 .value(),
7488 &instrument,
7489 );
7490 }
7491
7492 #[tokio::test]
7493 async fn tx_send_sequencer_blocks_higher_nonce_until_lower_batch_releases() {
7494 let sequencer = TxSendSequencer::new();
7495 let mut lower_a =
7496 sequencer.reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
7497 let mut lower_b = sequencer.reserve(
7498 TEST_ACCOUNT_INDEX_I64,
7499 TEST_API_KEY_INDEX,
7500 TEST_NEXT_NONCE + 1,
7501 );
7502 let mut higher = sequencer.reserve(
7503 TEST_ACCOUNT_INDEX_I64,
7504 TEST_API_KEY_INDEX,
7505 TEST_NEXT_NONCE + 2,
7506 );
7507 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
7508
7509 let higher_task = tokio::spawn(async move {
7510 higher.wait_for_turn().await;
7511 sent_tx.send(higher.nonce).unwrap();
7512 higher.release();
7513 });
7514
7515 assert!(
7516 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
7517 .await
7518 .is_err(),
7519 "higher nonce must wait while the lower batch is pending",
7520 );
7521
7522 {
7523 let lower_reservations = [&lower_a, &lower_b];
7524 tokio::time::timeout(
7525 Duration::from_millis(50),
7526 wait_for_tx_send_reservations(&lower_reservations),
7527 )
7528 .await
7529 .expect("lower batch must already have the send turn");
7530 }
7531
7532 lower_a.release();
7533 lower_b.release();
7534
7535 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
7536 .await
7537 .expect("higher nonce must proceed after lower batch releases")
7538 .expect("send channel must stay open");
7539 higher_task.await.unwrap();
7540
7541 assert_eq!(
7542 sent,
7543 TEST_NEXT_NONCE + 2,
7544 "higher nonce must send after the lower batch releases",
7545 );
7546 }
7547
7548 #[tokio::test]
7549 async fn tx_send_reservation_drop_releases_lower_nonce() {
7550 let sequencer = TxSendSequencer::new();
7551 let lower = sequencer.reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
7552 let mut higher = sequencer.reserve(
7553 TEST_ACCOUNT_INDEX_I64,
7554 TEST_API_KEY_INDEX,
7555 TEST_NEXT_NONCE + 1,
7556 );
7557 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
7558
7559 let higher_task = tokio::spawn(async move {
7560 higher.wait_for_turn().await;
7561 sent_tx.send(higher.nonce).unwrap();
7562 higher.release();
7563 });
7564
7565 assert!(
7566 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
7567 .await
7568 .is_err(),
7569 "higher nonce must wait while the lower reservation is pending",
7570 );
7571
7572 drop(lower);
7573
7574 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
7575 .await
7576 .expect("higher nonce must proceed after lower reservation drops")
7577 .expect("send channel must stay open");
7578 higher_task.await.unwrap();
7579
7580 assert_eq!(
7581 sent,
7582 TEST_NEXT_NONCE + 1,
7583 "higher nonce must send after the lower reservation drops",
7584 );
7585 }
7586
7587 #[tokio::test]
7588 async fn tx_send_sequencer_keeps_nonce_streams_independent() {
7589 let sequencer = TxSendSequencer::new();
7590 let _other_account = sequencer.reserve(
7591 TEST_ACCOUNT_INDEX_I64 + 1,
7592 TEST_API_KEY_INDEX,
7593 TEST_NEXT_NONCE,
7594 );
7595 let _other_api_key = sequencer.reserve(
7596 TEST_ACCOUNT_INDEX_I64,
7597 TEST_API_KEY_INDEX + 1,
7598 TEST_NEXT_NONCE,
7599 );
7600 let mut current = sequencer.reserve(
7601 TEST_ACCOUNT_INDEX_I64,
7602 TEST_API_KEY_INDEX,
7603 TEST_NEXT_NONCE + 10,
7604 );
7605
7606 tokio::time::timeout(Duration::from_millis(50), current.wait_for_turn())
7607 .await
7608 .expect("lower nonces for other keys must not block this key");
7609 current.release();
7610 }
7611
7612 #[rstest]
7613 fn tx_dispatch_guard_rolls_back_nonce_and_cloid_when_armed() {
7614 let dispatch = WsDispatchState::new();
7615 let credential = test_credential();
7616 dispatch
7617 .nonce_manager
7618 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
7619
7620 let cloid = ClientOrderId::from("O-GUARD-ARMED");
7621 let client_order_index = dispatch.derive_client_order_index(&cloid);
7622 dispatch.register_cloid(client_order_index, cloid).unwrap();
7623 let nonce = dispatch
7624 .nonce_manager
7625 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
7626 .unwrap();
7627
7628 {
7629 let _guard = TxDispatchGuard::new(
7630 dispatch.clone(),
7631 &credential,
7632 Some(client_order_index),
7633 nonce,
7634 );
7635 }
7636
7637 assert_nonce_reusable(&dispatch);
7638 assert!(dispatch.cloid_map.get(&client_order_index).is_none());
7639 }
7640
7641 #[rstest]
7642 fn tx_dispatch_guard_rolls_back_nonce_without_cloid_when_armed() {
7643 let dispatch = WsDispatchState::new();
7644 let credential = test_credential();
7645 dispatch
7646 .nonce_manager
7647 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
7648 let nonce = dispatch
7649 .nonce_manager
7650 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
7651 .unwrap();
7652
7653 {
7654 let _guard = TxDispatchGuard::new(dispatch.clone(), &credential, None, nonce);
7655 }
7656
7657 assert_nonce_reusable(&dispatch);
7658 assert!(dispatch.cloid_map.is_empty());
7659 }
7660
7661 #[rstest]
7662 fn tx_dispatch_guard_preserves_nonce_and_cloid_when_disarmed() {
7663 let dispatch = WsDispatchState::new();
7664 let credential = test_credential();
7665 dispatch
7666 .nonce_manager
7667 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
7668
7669 let cloid = ClientOrderId::from("O-GUARD-DISARMED");
7670 let client_order_index = dispatch.derive_client_order_index(&cloid);
7671 dispatch.register_cloid(client_order_index, cloid).unwrap();
7672 let nonce = dispatch
7673 .nonce_manager
7674 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
7675 .unwrap();
7676
7677 {
7678 let mut guard = TxDispatchGuard::new(
7679 dispatch.clone(),
7680 &credential,
7681 Some(client_order_index),
7682 nonce,
7683 );
7684 guard.disarm();
7685 }
7686
7687 assert_eq!(
7688 dispatch
7689 .nonce_manager
7690 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
7691 Some(TEST_NEXT_NONCE),
7692 );
7693 assert_eq!(
7694 dispatch
7695 .cloid_map
7696 .get(&client_order_index)
7697 .map(|entry| *entry.value()),
7698 Some(cloid),
7699 );
7700 }
7701
7702 #[tokio::test]
7703 async fn submit_order_send_failure_emits_submitted_then_rejected_and_rolls_back() {
7704 let (client, cache, mut rx) = create_execution_client();
7705 let instrument_id = register_test_instrument(&client, &cache);
7706 let mut factory = test_order_factory();
7707 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-FAIL");
7708 let client_order_index = client
7709 .dispatch
7710 .derive_client_order_index(&order.client_order_id());
7711 cache_order(&cache, order.clone());
7712
7713 let command = SubmitOrder::from_order(
7714 &order,
7715 trader_id(),
7716 Some(client_id()),
7717 None,
7718 UUID4::new(),
7719 UnixNanos::default(),
7720 );
7721 client.submit_order(command).unwrap();
7722
7723 let submitted = recv_order_event(&mut rx).await;
7724 let rejected = recv_order_event(&mut rx).await;
7725
7726 match submitted {
7727 OrderEventAny::Submitted(event) => {
7728 assert_eq!(event.client_order_id, order.client_order_id());
7729 assert_eq!(event.instrument_id, instrument_id);
7730 }
7731 event => panic!("expected submitted event, was {event:?}"),
7732 }
7733
7734 match rejected {
7735 OrderEventAny::Rejected(event) => {
7736 assert_eq!(event.client_order_id, order.client_order_id());
7737 assert_eq!(event.instrument_id, instrument_id);
7738 assert!(
7739 event
7740 .reason
7741 .contains("Lighter submit_order dispatch failed"),
7742 );
7743 assert!(event.reason.contains("handler unavailable"));
7744 }
7745 event => panic!("expected rejected event, was {event:?}"),
7746 }
7747
7748 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
7749 assert_nonce_reusable(&client.dispatch);
7750 assert_eq!(
7751 client.dispatch.pending_sendtx_len(),
7752 0,
7753 "local-send-failure must remove the pending entry by nonce",
7754 );
7755 }
7756
7757 #[tokio::test]
7758 async fn submit_order_unknown_send_outcome_retains_pending_state() {
7759 let (client, cache, mut rx) = create_execution_client();
7760 let instrument_id = register_test_instrument(&client, &cache);
7761 let mut factory = test_order_factory();
7762 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-UNKNOWN");
7763 let client_order_index = client
7764 .dispatch
7765 .derive_client_order_index(&order.client_order_id());
7766 cache_order(&cache, order.clone());
7767 client.ws_client.drop_next_send_tx_result_for_test().await;
7768
7769 client
7770 .submit_order(SubmitOrder::from_order(
7771 &order,
7772 trader_id(),
7773 Some(client_id()),
7774 None,
7775 UUID4::new(),
7776 UnixNanos::default(),
7777 ))
7778 .unwrap();
7779
7780 assert!(matches!(
7781 recv_order_event(&mut rx).await,
7782 OrderEventAny::Submitted(_)
7783 ));
7784 assert!(
7785 tokio::time::timeout(Duration::from_millis(50), rx.recv())
7786 .await
7787 .is_err(),
7788 "unknown outcome must not emit OrderRejected",
7789 );
7790 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
7791 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
7792 assert_eq!(
7793 client
7794 .dispatch
7795 .nonce_manager
7796 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
7797 Some(TEST_NEXT_NONCE),
7798 );
7799
7800 let stale = client.dispatch.drain_pending_sendtx(0);
7801 assert_eq!(stale.len(), 1);
7802 assert!(matches!(stale[0].kind, PendingSendTxKind::Create { .. }));
7803 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
7804 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
7805 assert!(
7806 client
7807 .dispatch
7808 .order_identity(&order.client_order_id())
7809 .is_some(),
7810 "reconnect draining must retain identity for reconciliation",
7811 );
7812 assert_eq!(
7813 client
7814 .dispatch
7815 .nonce_manager
7816 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
7817 Some(TEST_NEXT_NONCE),
7818 );
7819 }
7820
7821 #[tokio::test]
7822 async fn submit_order_cloid_probe_exhaustion_denies_without_overwrite() {
7823 let (client, cache, mut rx) = create_execution_client();
7824 let instrument_id = register_test_instrument(&client, &cache);
7825 let mut factory = test_order_factory();
7826 let order = test_limit_order(&mut factory, instrument_id, "O-PROBE-EXHAUSTED");
7827 let cloid = order.client_order_id();
7828 let mut index = client.dispatch.derive_client_order_index(&cloid);
7829 let mut existing = Vec::new();
7830
7831 for attempt in 0..=crate::websocket::dispatch::CLOID_INDEX_PROBE_LIMIT {
7832 let existing_cloid = ClientOrderId::from(format!("EXISTING-{attempt}"));
7833 client.dispatch.cloid_map.insert(index, existing_cloid);
7834 existing.push((index, existing_cloid));
7835 index = if index == i64::from(i32::MAX) {
7836 0
7837 } else {
7838 index + 1
7839 };
7840 }
7841 cache_order(&cache, order.clone());
7842
7843 client
7844 .submit_order(SubmitOrder::from_order(
7845 &order,
7846 trader_id(),
7847 Some(client_id()),
7848 None,
7849 UUID4::new(),
7850 UnixNanos::default(),
7851 ))
7852 .unwrap();
7853
7854 let event = recv_order_event(&mut rx).await;
7855 assert!(matches!(event, OrderEventAny::Denied(_)));
7856 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
7857 assert!(client.dispatch.order_identity(&cloid).is_none());
7858
7859 for (existing_index, existing_cloid) in existing {
7860 assert_eq!(
7861 client
7862 .dispatch
7863 .cloid_map
7864 .get(&existing_index)
7865 .map(|entry| *entry.value()),
7866 Some(existing_cloid),
7867 );
7868 }
7869 }
7870
7871 #[tokio::test]
7872 async fn submit_sell_order_send_failure_dispatches_and_rolls_back() {
7873 let (client, cache, mut rx) = create_execution_client();
7876 let instrument_id = register_test_instrument(&client, &cache);
7877 let mut factory = test_order_factory();
7878 let order = test_limit_order_with(
7879 &mut factory,
7880 instrument_id,
7881 "O-SUBMIT-FAIL-SELL",
7882 OrderSide::Sell,
7883 TimeInForce::Gtc,
7884 None,
7885 false,
7886 );
7887 let client_order_index = client
7888 .dispatch
7889 .derive_client_order_index(&order.client_order_id());
7890 cache_order(&cache, order.clone());
7891
7892 let command = SubmitOrder::from_order(
7893 &order,
7894 trader_id(),
7895 Some(client_id()),
7896 None,
7897 UUID4::new(),
7898 UnixNanos::default(),
7899 );
7900 client.submit_order(command).unwrap();
7901
7902 let _submitted = recv_order_event(&mut rx).await;
7903 let rejected = recv_order_event(&mut rx).await;
7904
7905 match rejected {
7906 OrderEventAny::Rejected(event) => {
7907 assert_eq!(event.client_order_id, order.client_order_id());
7908 assert!(
7909 event
7910 .reason
7911 .contains("Lighter submit_order dispatch failed"),
7912 );
7913 }
7914 event => panic!("expected rejected event, was {event:?}"),
7915 }
7916
7917 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
7918 assert_nonce_reusable(&client.dispatch);
7919 }
7920
7921 #[tokio::test]
7922 async fn submit_gtd_order_with_explicit_expiry_dispatches_and_rolls_back() {
7923 let (client, cache, mut rx) = create_execution_client();
7928 let instrument_id = register_test_instrument(&client, &cache);
7929 let mut factory = test_order_factory();
7930 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
7931 let order = test_limit_order_with(
7932 &mut factory,
7933 instrument_id,
7934 "O-SUBMIT-FAIL-GTD",
7935 OrderSide::Buy,
7936 TimeInForce::Gtd,
7937 Some(expiry),
7938 false,
7939 );
7940 let client_order_index = client
7941 .dispatch
7942 .derive_client_order_index(&order.client_order_id());
7943 cache_order(&cache, order.clone());
7944
7945 let command = SubmitOrder::from_order(
7946 &order,
7947 trader_id(),
7948 Some(client_id()),
7949 None,
7950 UUID4::new(),
7951 UnixNanos::default(),
7952 );
7953 client.submit_order(command).unwrap();
7954
7955 let _submitted = recv_order_event(&mut rx).await;
7956 let rejected = recv_order_event(&mut rx).await;
7957
7958 match rejected {
7959 OrderEventAny::Rejected(event) => {
7960 assert!(
7961 event
7962 .reason
7963 .contains("Lighter submit_order dispatch failed"),
7964 );
7965 }
7966 event => panic!("expected rejected event, was {event:?}"),
7967 }
7968
7969 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
7970 assert_nonce_reusable(&client.dispatch);
7971 }
7972
7973 #[tokio::test]
7974 async fn submit_gtd_order_with_short_expiry_is_denied_before_nonce_allocation() {
7975 let (client, cache, mut rx) = create_execution_client();
7976 let instrument_id = register_test_instrument(&client, &cache);
7977 let mut factory = test_order_factory();
7978 let expiry = UnixNanos::from(
7979 client
7980 .clock
7981 .get_time_ns()
7982 .as_u64()
7983 .saturating_add(60 * 1_000_000_000),
7984 );
7985 let order = test_limit_order_with(
7986 &mut factory,
7987 instrument_id,
7988 "O-SHORT-GTD",
7989 OrderSide::Buy,
7990 TimeInForce::Gtd,
7991 Some(expiry),
7992 false,
7993 );
7994 cache_order(&cache, order.clone());
7995
7996 let command = SubmitOrder::from_order(
7997 &order,
7998 trader_id(),
7999 Some(client_id()),
8000 None,
8001 UUID4::new(),
8002 UnixNanos::default(),
8003 );
8004 client.submit_order(command).unwrap();
8005
8006 let denied = recv_order_event(&mut rx).await;
8007 match denied {
8008 OrderEventAny::Denied(event) => {
8009 assert!(event.reason.contains("at least 5 minutes"));
8010 }
8011 event => panic!("expected denied event, was {event:?}"),
8012 }
8013 assert_nonce_reusable(&client.dispatch);
8014 }
8015
8016 #[tokio::test]
8017 async fn submit_gtd_order_with_short_expiry_dispatches_when_use_gtd_false() {
8018 let (client, cache, mut rx) =
8023 create_execution_client_with_config(test_config_with_use_gtd(false));
8024 let instrument_id = register_test_instrument(&client, &cache);
8025 let mut factory = test_order_factory();
8026 let expiry = UnixNanos::from(
8027 client
8028 .clock
8029 .get_time_ns()
8030 .as_u64()
8031 .saturating_add(60 * 1_000_000_000),
8032 );
8033 let order = test_limit_order_with(
8034 &mut factory,
8035 instrument_id,
8036 "O-SHORT-GTD-MANAGED",
8037 OrderSide::Buy,
8038 TimeInForce::Gtd,
8039 Some(expiry),
8040 false,
8041 );
8042 let client_order_index = client
8043 .dispatch
8044 .derive_client_order_index(&order.client_order_id());
8045 cache_order(&cache, order.clone());
8046
8047 let command = SubmitOrder::from_order(
8048 &order,
8049 trader_id(),
8050 Some(client_id()),
8051 None,
8052 UUID4::new(),
8053 UnixNanos::default(),
8054 );
8055 client.submit_order(command).unwrap();
8056
8057 let _submitted = recv_order_event(&mut rx).await;
8058 let rejected = recv_order_event(&mut rx).await;
8059
8060 match rejected {
8061 OrderEventAny::Rejected(event) => {
8062 assert!(
8063 event
8064 .reason
8065 .contains("Lighter submit_order dispatch failed"),
8066 );
8067 }
8068 event => panic!("expected rejected event, was {event:?}"),
8069 }
8070
8071 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
8072 assert_nonce_reusable(&client.dispatch);
8073 }
8074
8075 #[rstest]
8076 fn prepare_managed_gtd_plan_uses_safe_venue_expiry() {
8077 let (client, cache, _rx) =
8078 create_execution_client_with_config(test_config_with_use_gtd(false));
8079 let instrument_id = register_test_instrument(&client, &cache);
8080 let mut factory = test_order_factory();
8081 let expiry = UnixNanos::from(
8082 client
8083 .clock
8084 .get_time_ns()
8085 .as_u64()
8086 .saturating_add(60 * 1_000_000_000),
8087 );
8088 let order = test_limit_order_with(
8089 &mut factory,
8090 instrument_id,
8091 "O-SHORT-GTD-PLAN",
8092 OrderSide::Buy,
8093 TimeInForce::Gtd,
8094 Some(expiry),
8095 false,
8096 );
8097
8098 let before_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
8099 let plan = client
8100 .prepare_create_order_plan(&order, 0)
8101 .expect("managed GTD plan must validate");
8102 let after_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
8103
8104 assert!(
8105 (before_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS
8106 ..=after_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS)
8107 .contains(&plan.order_expiry)
8108 );
8109 assert_eq!(
8110 plan.time_in_force,
8111 crate::common::enums::LighterTimeInForce::GoodTillTime as u8,
8112 );
8113 }
8114
8115 #[tokio::test]
8116 async fn submit_reduce_only_order_dispatches_and_rolls_back() {
8117 let (client, cache, mut rx) = create_execution_client();
8119 let instrument_id = register_test_instrument(&client, &cache);
8120 let mut factory = test_order_factory();
8121 let order = test_limit_order_with(
8122 &mut factory,
8123 instrument_id,
8124 "O-SUBMIT-FAIL-REDUCE",
8125 OrderSide::Sell,
8126 TimeInForce::Gtc,
8127 None,
8128 true,
8129 );
8130 assert!(order.is_reduce_only());
8131 let client_order_index = client
8132 .dispatch
8133 .derive_client_order_index(&order.client_order_id());
8134 cache_order(&cache, order.clone());
8135
8136 let command = SubmitOrder::from_order(
8137 &order,
8138 trader_id(),
8139 Some(client_id()),
8140 None,
8141 UUID4::new(),
8142 UnixNanos::default(),
8143 );
8144 client.submit_order(command).unwrap();
8145
8146 let _submitted = recv_order_event(&mut rx).await;
8147 let rejected = recv_order_event(&mut rx).await;
8148
8149 match rejected {
8150 OrderEventAny::Rejected(event) => {
8151 assert!(
8152 event
8153 .reason
8154 .contains("Lighter submit_order dispatch failed"),
8155 );
8156 }
8157 event => panic!("expected rejected event, was {event:?}"),
8158 }
8159
8160 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
8161 assert_nonce_reusable(&client.dispatch);
8162 }
8163
8164 #[tokio::test]
8165 async fn submit_order_list_send_failure_emits_submitted_then_rejected_and_rolls_back() {
8166 let (client, cache, mut rx) = create_execution_client();
8167 let instrument_id = register_test_instrument(&client, &cache);
8168 let mut factory = test_order_factory();
8169 let order_a = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-A");
8170 let order_b = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-B");
8171 let index_a = client
8172 .dispatch
8173 .derive_client_order_index(&order_a.client_order_id());
8174 let index_b = client
8175 .dispatch
8176 .derive_client_order_index(&order_b.client_order_id());
8177 cache_order(&cache, order_a.clone());
8178 cache_order(&cache, order_b.clone());
8179
8180 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-FAIL");
8181 client.submit_order_list(command).unwrap();
8182
8183 for expected in [order_a.client_order_id(), order_b.client_order_id()] {
8184 match recv_order_event(&mut rx).await {
8185 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, expected),
8186 other => panic!("expected Submitted, was {other:?}"),
8187 }
8188
8189 match recv_order_event(&mut rx).await {
8190 OrderEventAny::Rejected(e) => {
8191 assert_eq!(e.client_order_id, expected);
8192 assert!(e.reason.contains("Lighter submit_order dispatch failed"),);
8193 }
8194 other => panic!("expected Rejected, was {other:?}"),
8195 }
8196 }
8197 assert!(client.dispatch.cloid_map.get(&index_a).is_none());
8198 assert!(client.dispatch.cloid_map.get(&index_b).is_none());
8199 assert_nonce_reusable(&client.dispatch);
8200 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8201 }
8202
8203 #[tokio::test]
8204 async fn submit_order_list_over_max_batch_size_denies_all_without_dispatch() {
8205 let (client, cache, mut rx) = create_execution_client();
8206 let instrument_id = register_test_instrument(&client, &cache);
8207 let mut factory = test_order_factory();
8208 let mut orders = Vec::new();
8209
8210 for i in 0..=LIGHTER_MAX_BATCH_TX {
8211 let order = test_limit_order(&mut factory, instrument_id, &format!("O-LIST-MAX-{i}"));
8212 cache_order(&cache, order.clone());
8213 orders.push(order);
8214 }
8215
8216 let command = submit_order_list_command(&orders, "OL-MAX");
8217 client.submit_order_list(command).unwrap();
8218
8219 for order in &orders {
8220 match recv_order_event(&mut rx).await {
8221 OrderEventAny::Denied(e) => {
8222 assert_eq!(e.client_order_id, order.client_order_id());
8223 assert!(
8224 e.reason
8225 .contains("order-list fanout supports at most 15 txs"),
8226 );
8227 }
8228 other => panic!("expected Denied, was {other:?}"),
8229 }
8230 }
8231
8232 assert!(
8233 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8234 .await
8235 .is_err(),
8236 "max-size denial must not emit extra events",
8237 );
8238 assert_nonce_reusable(&client.dispatch);
8239 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8240 }
8241
8242 #[tokio::test]
8243 async fn submit_order_list_denies_unsupported_order_and_dispatches_supported() {
8244 let (client, cache, mut rx) = create_execution_client();
8245 let instrument_id = register_test_instrument(&client, &cache);
8246 let mut factory = test_order_factory();
8247 let valid = test_limit_order(&mut factory, instrument_id, "O-LIST-VALID");
8248 let unsupported = factory.limit(
8249 instrument_id,
8250 OrderSide::Buy,
8251 Quantity::from("0.1000"),
8252 Price::from("2361.31"),
8253 Some(TimeInForce::Gtc),
8254 None,
8255 Some(false),
8256 Some(false),
8257 None,
8258 Some(Quantity::from("0.0500")),
8259 None,
8260 None,
8261 None,
8262 None,
8263 None,
8264 Some(ClientOrderId::from("O-LIST-ICEBERG")),
8265 );
8266 let unsupported_index = client
8267 .dispatch
8268 .derive_client_order_index(&unsupported.client_order_id());
8269 cache_order(&cache, valid.clone());
8270 cache_order(&cache, unsupported.clone());
8271
8272 let command =
8273 submit_order_list_command(&[unsupported.clone(), valid.clone()], "OL-PARTIAL");
8274 client.submit_order_list(command).unwrap();
8275
8276 match recv_order_event(&mut rx).await {
8277 OrderEventAny::Denied(e) => {
8278 assert_eq!(e.client_order_id, unsupported.client_order_id());
8279 assert!(e.reason.contains("display_qty"));
8280 }
8281 other => panic!("expected Denied, was {other:?}"),
8282 }
8283
8284 match recv_order_event(&mut rx).await {
8285 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, valid.client_order_id()),
8286 other => panic!("expected Submitted, was {other:?}"),
8287 }
8288
8289 match recv_order_event(&mut rx).await {
8290 OrderEventAny::Rejected(e) => {
8291 assert_eq!(e.client_order_id, valid.client_order_id());
8292 assert!(e.reason.contains("Lighter submit_order dispatch failed"),);
8293 }
8294 other => panic!("expected Rejected, was {other:?}"),
8295 }
8296
8297 assert!(client.dispatch.cloid_map.get(&unsupported_index).is_none());
8298 assert_nonce_reusable(&client.dispatch);
8299 }
8300
8301 #[tokio::test]
8302 async fn submit_order_list_grouped_contingency_denies_all_without_dispatch() {
8303 let (client, cache, mut rx) = create_execution_client();
8304 let instrument_id = register_test_instrument(&client, &cache);
8305 let order_a_id = "O-LIST-OCO-A";
8306 let order_b_id = "O-LIST-OCO-B";
8307 let order_a = test_contingent_limit_order(instrument_id, order_a_id, "OL-OCO", order_b_id);
8308 let order_b = test_contingent_limit_order(instrument_id, order_b_id, "OL-OCO", order_a_id);
8309 cache_order(&cache, order_a.clone());
8310 cache_order(&cache, order_b.clone());
8311
8312 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-OCO");
8313 client.submit_order_list(command).unwrap();
8314
8315 for order in [&order_a, &order_b] {
8316 match recv_order_event(&mut rx).await {
8317 OrderEventAny::Denied(e) => {
8318 assert_eq!(e.client_order_id, order.client_order_id());
8319 assert!(e.reason.contains("supports only independent"));
8320 }
8321 other => panic!("expected Denied, was {other:?}"),
8322 }
8323 }
8324
8325 assert!(
8326 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8327 .await
8328 .is_err(),
8329 "grouped list denial must not emit extra events",
8330 );
8331 assert_nonce_reusable(&client.dispatch);
8332 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8333 }
8334
8335 #[tokio::test]
8336 async fn cancel_order_send_failure_emits_cancel_rejected_and_rolls_back() {
8337 let (client, cache, mut rx) = create_execution_client();
8338 let instrument_id = register_test_instrument(&client, &cache);
8339 let mut factory = test_order_factory();
8340 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-FAIL");
8341 let client_order_id = order.client_order_id();
8342 let venue_order_id = VenueOrderId::from("123");
8343 cache_pending_cancel_order(&cache, order, venue_order_id);
8344
8345 let command = CancelOrder::new(
8346 trader_id(),
8347 Some(client_id()),
8348 strategy_id(),
8349 instrument_id,
8350 client_order_id,
8351 Some(venue_order_id),
8352 UUID4::new(),
8353 UnixNanos::default(),
8354 None,
8355 None,
8356 );
8357 client.cancel_order(command).unwrap();
8358
8359 let rejected = recv_order_event(&mut rx).await;
8360
8361 match rejected {
8362 OrderEventAny::CancelRejected(event) => {
8363 assert_eq!(event.client_order_id, client_order_id);
8364 assert_eq!(event.instrument_id, instrument_id);
8365 assert_eq!(event.venue_order_id, Some(venue_order_id));
8366 assert!(
8367 event
8368 .reason
8369 .contains("Lighter cancel_order dispatch failed"),
8370 );
8371 assert!(event.reason.contains("handler unavailable"));
8372 }
8373 event => panic!("expected cancel rejected event, was {event:?}"),
8374 }
8375
8376 assert_nonce_reusable(&client.dispatch);
8377 assert_eq!(
8378 client.dispatch.pending_sendtx_len(),
8379 0,
8380 "local-send-failure must remove the pending cancel entry",
8381 );
8382 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
8383 }
8384
8385 #[tokio::test]
8386 async fn cancel_order_unknown_send_outcome_retains_pending_state() {
8387 let (client, cache, mut rx) = create_execution_client();
8388 let instrument_id = register_test_instrument(&client, &cache);
8389 let mut factory = test_order_factory();
8390 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-UNKNOWN");
8391 let client_order_id = order.client_order_id();
8392 let venue_order_id = VenueOrderId::from("123");
8393 cache_pending_cancel_order(&cache, order, venue_order_id);
8394 client.ws_client.drop_next_send_tx_result_for_test().await;
8395
8396 client
8397 .cancel_order(CancelOrder::new(
8398 trader_id(),
8399 Some(client_id()),
8400 strategy_id(),
8401 instrument_id,
8402 client_order_id,
8403 Some(venue_order_id),
8404 UUID4::new(),
8405 UnixNanos::default(),
8406 None,
8407 None,
8408 ))
8409 .unwrap();
8410
8411 assert!(
8412 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8413 .await
8414 .is_err(),
8415 "unknown outcome must not emit OrderCancelRejected",
8416 );
8417 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
8418 assert_eq!(
8419 client.dispatch.pending_order_action(&client_order_id),
8420 Some(PendingOrderAction::Cancel),
8421 );
8422 assert_eq!(
8423 client
8424 .dispatch
8425 .nonce_manager
8426 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
8427 Some(TEST_NEXT_NONCE),
8428 );
8429 }
8430
8431 #[tokio::test]
8432 async fn cancel_order_prepare_failure_emits_cancel_rejected_without_dispatch() {
8433 let (client, cache, mut rx) = create_execution_client();
8434 let instrument_id = register_test_instrument(&client, &cache);
8435 let mut factory = test_order_factory();
8436 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NO-VOI");
8437 let client_order_id = order.client_order_id();
8438 cache_pending_cancel_order(&cache, order, VenueOrderId::from("123"));
8439
8440 let command = CancelOrder::new(
8441 trader_id(),
8442 Some(client_id()),
8443 strategy_id(),
8444 instrument_id,
8445 client_order_id,
8446 None,
8447 UUID4::new(),
8448 UnixNanos::default(),
8449 None,
8450 None,
8451 );
8452 client.cancel_order(command).unwrap();
8453
8454 let rejected = recv_order_event(&mut rx).await;
8455 match rejected {
8456 OrderEventAny::CancelRejected(event) => {
8457 assert_eq!(event.client_order_id, client_order_id);
8458 assert_eq!(event.instrument_id, instrument_id);
8459 assert_eq!(event.venue_order_id, None);
8460 assert!(event.reason.contains("Lighter cancel_order failed"));
8461 assert!(event.reason.contains("venue order_id not yet known"));
8462 }
8463 event => panic!("expected cancel rejected event, was {event:?}"),
8464 }
8465
8466 assert_nonce_reusable(&client.dispatch);
8467 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8468 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
8469 assert!(
8470 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8471 .await
8472 .is_err(),
8473 "prepare failure must emit exactly one cancel rejection",
8474 );
8475 }
8476
8477 #[tokio::test]
8478 async fn cancel_order_prepare_failure_emits_cancel_rejected_when_order_uncached() {
8479 let (client, cache, mut rx) = create_execution_client();
8480 let instrument_id = register_test_instrument(&client, &cache);
8481 let client_order_id = ClientOrderId::from("O-CANCEL-NO-CACHE");
8482 let venue_order_id = VenueOrderId::from("123");
8483
8484 let command = CancelOrder::new(
8485 trader_id(),
8486 Some(client_id()),
8487 strategy_id(),
8488 instrument_id,
8489 client_order_id,
8490 Some(venue_order_id),
8491 UUID4::new(),
8492 UnixNanos::default(),
8493 None,
8494 None,
8495 );
8496 client.cancel_order(command).unwrap();
8497
8498 let rejected = recv_order_event(&mut rx).await;
8499 match rejected {
8500 OrderEventAny::CancelRejected(event) => {
8501 assert_eq!(event.client_order_id, client_order_id);
8502 assert_eq!(event.instrument_id, instrument_id);
8503 assert_eq!(event.venue_order_id, Some(venue_order_id));
8504 assert!(event.reason.contains("Lighter cancel_order failed"));
8505 assert!(event.reason.contains("order not found in cache"));
8506 }
8507 event => panic!("expected cancel rejected event, was {event:?}"),
8508 }
8509
8510 assert_nonce_reusable(&client.dispatch);
8511 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8512 assert!(
8513 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8514 .await
8515 .is_err(),
8516 "prepare failure must emit exactly one cancel rejection",
8517 );
8518 }
8519
8520 #[rstest]
8521 #[case::buy(Some(OrderSide::Buy), false, vec![0, 2])]
8522 #[case::sell(Some(OrderSide::Sell), false, vec![1, 3])]
8523 #[case::unsided(None, false, vec![0, 1, 2, 3])]
8524 #[case::empty_buy(Some(OrderSide::Buy), true, vec![])]
8525 #[case::empty_sell(Some(OrderSide::Sell), true, vec![])]
8526 #[case::empty_unsided(None, true, vec![])]
8527 #[tokio::test]
8528 async fn cancel_all_orders_filters_orders_and_preserves_owners(
8529 #[case] order_side: Option<OrderSide>,
8530 #[case] empty: bool,
8531 #[case] expected_indices: Vec<usize>,
8532 ) {
8533 let (client, cache, mut rx) = create_execution_client();
8534 let instrument_id = register_test_instrument(&client, &cache);
8535 let other_instrument_id = InstrumentId::from("BTC-PERP.LIGHTER");
8536 let mut orders = Vec::new();
8537
8538 for (index, (instrument, owner, side, open)) in [
8539 (instrument_id, "S-001", OrderSide::Buy, true),
8540 (instrument_id, "S-001", OrderSide::Sell, true),
8541 (instrument_id, "S-002", OrderSide::Buy, true),
8542 (instrument_id, "S-002", OrderSide::Sell, true),
8543 (other_instrument_id, "S-001", OrderSide::Buy, true),
8544 (other_instrument_id, "S-002", OrderSide::Sell, true),
8545 (instrument_id, "S-001", OrderSide::Buy, false),
8546 (instrument_id, "S-002", OrderSide::Sell, false),
8547 ]
8548 .into_iter()
8549 .enumerate()
8550 {
8551 if empty
8552 && instrument == instrument_id
8553 && open
8554 && order_side.is_none_or(|filter| filter == side)
8555 {
8556 continue;
8557 }
8558
8559 let order = OrderTestBuilder::new(OrderType::Limit)
8560 .trader_id(trader_id())
8561 .strategy_id(StrategyId::from(owner))
8562 .instrument_id(instrument)
8563 .client_order_id(ClientOrderId::from(format!("O-CANCEL-ALL-{index}")))
8564 .side(side)
8565 .quantity(Quantity::from("0.1000"))
8566 .price(Price::from("2361.31"))
8567 .build();
8568 let venue_order_id = VenueOrderId::from(format!("{}", 123 + index));
8569 let accepted = TestOrderEventStubs::accepted(&order, account_id(), venue_order_id);
8570 cache_order(&cache, order.clone());
8571 cache.borrow_mut().update_order(&accepted).unwrap();
8572 client
8573 .dispatch
8574 .venue_id_map
8575 .insert(order.client_order_id(), venue_order_id);
8576
8577 let event = if open {
8578 OrderEventAny::PendingCancel(OrderPendingCancel::new(
8579 order.trader_id(),
8580 order.strategy_id(),
8581 instrument,
8582 order.client_order_id(),
8583 Some(account_id()),
8584 UUID4::new(),
8585 UnixNanos::default(),
8586 UnixNanos::default(),
8587 false,
8588 Some(venue_order_id),
8589 ))
8590 } else {
8591 TestOrderEventStubs::canceled(&order, account_id(), Some(venue_order_id))
8592 };
8593
8594 cache.borrow_mut().update_order(&event).unwrap();
8595 orders.push((order, venue_order_id));
8596 }
8597
8598 client
8599 .cancel_all_orders(CancelAllOrders::new(
8600 trader_id(),
8601 Some(client_id()),
8602 strategy_id(),
8603 instrument_id,
8604 order_side,
8605 UUID4::new(),
8606 UnixNanos::default(),
8607 None,
8608 None,
8609 ))
8610 .unwrap();
8611
8612 let mut actual = Vec::new();
8615
8616 for _ in &expected_indices {
8617 match recv_order_event(&mut rx).await {
8618 OrderEventAny::CancelRejected(event) => {
8619 assert!(
8620 event
8621 .reason
8622 .contains("Lighter cancel_order dispatch failed")
8623 );
8624 assert!(event.reason.contains("handler unavailable"));
8625 assert_eq!(event.trader_id, trader_id());
8626 assert_eq!(event.account_id, Some(account_id()));
8627 actual.push((
8628 event.client_order_id,
8629 event.strategy_id,
8630 event.instrument_id,
8631 event.venue_order_id,
8632 ));
8633 }
8634 event => panic!("expected cancel rejected event, was {event:?}"),
8635 }
8636 }
8637
8638 let mut expected: Vec<_> = expected_indices
8639 .iter()
8640 .map(|&index| {
8641 let (order, venue_order_id) = &orders[index];
8642 (
8643 order.client_order_id(),
8644 order.strategy_id(),
8645 order.instrument_id(),
8646 Some(*venue_order_id),
8647 )
8648 })
8649 .collect();
8650
8651 actual.sort();
8652 expected.sort();
8653
8654 assert_eq!(actual, expected);
8655 assert!(
8656 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8657 .await
8658 .is_err()
8659 );
8660 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8661
8662 for (order, _) in orders {
8663 assert_eq!(
8664 client
8665 .dispatch
8666 .pending_order_action(&order.client_order_id()),
8667 None
8668 );
8669 }
8670
8671 if empty {
8672 assert_nonce_reusable(&client.dispatch);
8673 }
8674 }
8675
8676 #[tokio::test]
8677 async fn cancel_all_orders_prepare_failure_suppresses_cancel_rejected_for_open_order() {
8678 let (client, cache, mut rx) = create_execution_client();
8679 let instrument_id = register_test_instrument(&client, &cache);
8680 let mut factory = test_order_factory();
8681 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-ALL-NO-VOI");
8682 cache_accepted_order(&cache, order, VenueOrderId::from("123"), Some(client_id()));
8683
8684 let command = CancelAllOrders::new(
8685 trader_id(),
8686 Some(client_id()),
8687 strategy_id(),
8688 instrument_id,
8689 Some(OrderSide::Buy),
8690 UUID4::new(),
8691 UnixNanos::default(),
8692 None,
8693 None,
8694 );
8695 client.cancel_all_orders(command).unwrap();
8696
8697 assert_nonce_reusable(&client.dispatch);
8698 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8699 assert!(
8700 tokio::time::timeout(Duration::from_millis(50), rx.recv())
8701 .await
8702 .is_err(),
8703 "cancel-all prepare failure must not emit an invalid cancel rejection",
8704 );
8705 }
8706
8707 #[tokio::test]
8708 async fn cancel_order_nonce_prepare_failure_emits_cancel_rejected() {
8709 let mut config = test_config();
8710 config.base_url_http = Some(spawn_next_nonce_server(100).await);
8711 let (client, cache, mut rx) = create_execution_client_with_config(config);
8712 let instrument_id = register_test_instrument(&client, &cache);
8713 let mut factory = test_order_factory();
8714 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NONCE-FAIL");
8715 let client_order_id = order.client_order_id();
8716 let venue_order_id = VenueOrderId::from("123");
8717 cache_pending_cancel_order(&cache, order, venue_order_id);
8718
8719 let window = i64::from(client.dispatch.nonce_manager.skip_window());
8720 for _ in 0..window {
8721 client
8722 .dispatch
8723 .nonce_manager
8724 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
8725 .unwrap();
8726 }
8727
8728 let command = CancelOrder::new(
8729 trader_id(),
8730 Some(client_id()),
8731 strategy_id(),
8732 instrument_id,
8733 client_order_id,
8734 Some(venue_order_id),
8735 UUID4::new(),
8736 UnixNanos::default(),
8737 None,
8738 None,
8739 );
8740 client.cancel_order(command).unwrap();
8741
8742 let rejected = recv_order_event(&mut rx).await;
8743 match rejected {
8744 OrderEventAny::CancelRejected(event) => {
8745 assert_eq!(event.client_order_id, client_order_id);
8746 assert_eq!(event.venue_order_id, Some(venue_order_id));
8747 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
8748 assert!(event.reason.contains("skip-window exhausted"));
8749 }
8750 event => panic!("expected cancel rejected event, was {event:?}"),
8751 }
8752
8753 wait_for_spawned_tasks(&client).await;
8754 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8755 assert_eq!(
8756 client
8757 .dispatch
8758 .nonce_manager
8759 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
8760 .unwrap(),
8761 100,
8762 );
8763 }
8764
8765 #[tokio::test]
8766 async fn batch_cancel_releases_split_reservations_before_waiting_for_ack() {
8767 let (client, cache, _rx) = create_execution_client();
8768 let instrument_id = register_test_instrument(&client, &cache);
8769 client.ws_client.drop_next_send_tx_result_for_test().await;
8770 let context = client
8771 .fanout_dispatch_context(client.credential.as_ref().unwrap())
8772 .unwrap();
8773
8774 let plan = CancelOrderPlan {
8775 client_order_id: ClientOrderId::from("O-SPLIT-CANCEL"),
8776 strategy_id: strategy_id(),
8777 instrument_id,
8778 venue_order_id: Some(VenueOrderId::from("123")),
8779 market_index: TEST_MARKET_INDEX,
8780 venue_index: 123,
8781 };
8782
8783 let first = context.sign_cancel_order(&plan).unwrap();
8784 let interleaved = context.build_tx_context().unwrap();
8785 let second = context.sign_cancel_order(&plan).unwrap();
8786 let first_nonce = first.nonce;
8787 let second_nonce = second.nonce;
8788 drop(interleaved);
8789
8790 let task = tokio::spawn(async move {
8791 context
8792 .send_cancel_batch(vec![(first, false), (second, false)])
8793 .await;
8794 });
8795
8796 let dispatched = tokio::time::timeout(Duration::from_secs(2), async {
8797 while client.dispatch.pending_sendtx_len() == 0 {
8798 tokio::task::yield_now().await;
8799 }
8800 })
8801 .await;
8802
8803 let released = tokio::time::timeout(
8804 Duration::from_millis(250),
8805 client.nonce_submission_gate.write(),
8806 )
8807 .await
8808 .is_ok();
8809 let pending = client.dispatch.pending_sendtx.lock().clone();
8810 task.abort();
8811 let _ = task.await;
8812
8813 assert!(dispatched.is_ok());
8814 assert!(
8815 released,
8816 "unacknowledged group must not hold later nonce reservations"
8817 );
8818 assert_eq!(second_nonce, first_nonce + 2);
8819 assert_eq!(pending.len(), 1);
8820 assert_eq!(pending[0].nonce, first_nonce);
8821 }
8822
8823 #[rstest]
8824 #[case::not_sent(false)]
8825 #[case::ambiguous(true)]
8826 #[tokio::test]
8827 async fn batch_cancel_orders_send_failure_preserves_delivery_semantics(
8828 #[case] ambiguous: bool,
8829 ) {
8830 let (client, cache, mut rx) = create_execution_client();
8831 let instrument_id = register_test_instrument(&client, &cache);
8832 let mut factory = test_order_factory();
8833 let cancels = ["O-BATCH-CANCEL-A", "O-BATCH-CANCEL-B"]
8834 .iter()
8835 .enumerate()
8836 .map(|(i, id)| {
8837 let order = test_limit_order(&mut factory, instrument_id, id);
8838 let client_order_id = order.client_order_id();
8839 let venue_order_id = VenueOrderId::from(format!("{}", 123 + i).as_str());
8840 cache_pending_cancel_order(&cache, order, venue_order_id);
8841
8842 CancelOrder::new(
8843 trader_id(),
8844 Some(client_id()),
8845 strategy_id(),
8846 instrument_id,
8847 client_order_id,
8848 Some(venue_order_id),
8849 UUID4::new(),
8850 UnixNanos::default(),
8851 None,
8852 None,
8853 )
8854 })
8855 .collect::<Vec<_>>();
8856
8857 let command = BatchCancelOrders::new(
8858 trader_id(),
8859 Some(client_id()),
8860 strategy_id(),
8861 instrument_id,
8862 cancels.clone(),
8863 UUID4::new(),
8864 UnixNanos::default(),
8865 None,
8866 None,
8867 );
8868
8869 if ambiguous {
8870 client.ws_client.drop_next_send_tx_result_for_test().await;
8871 }
8872
8873 client.batch_cancel_orders(command).unwrap();
8874
8875 if ambiguous {
8876 assert!(
8877 tokio::time::timeout(Duration::from_millis(100), rx.recv())
8878 .await
8879 .is_err()
8880 );
8881 let pending = client.dispatch.pending_sendtx.lock().clone();
8882 assert_eq!(pending.len(), 2);
8883 assert_eq!(pending[0].batch_id, pending[1].batch_id);
8884 assert!(
8885 pending[0]
8886 .batch_id
8887 .as_ref()
8888 .unwrap()
8889 .starts_with("cancel-batch:")
8890 );
8891 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
8892 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
8893
8894 for cancel in cancels {
8895 assert_eq!(
8896 client
8897 .dispatch
8898 .pending_order_action(&cancel.client_order_id),
8899 Some(PendingOrderAction::Cancel)
8900 );
8901 }
8902
8903 assert_eq!(
8904 client
8905 .dispatch
8906 .nonce_manager
8907 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
8908 Some(TEST_NEXT_NONCE + 1)
8909 );
8910 return;
8911 }
8912
8913 let first = recv_order_event(&mut rx).await;
8914 let second = recv_order_event(&mut rx).await;
8915 let rejected_ids = [first, second].map(|event| match event {
8916 OrderEventAny::CancelRejected(e) => {
8917 assert!(e.reason.contains("Lighter cancel_order dispatch failed"),);
8918 e.client_order_id
8919 }
8920 other => panic!("expected CancelRejected, was {other:?}"),
8921 });
8922
8923 for cancel in cancels {
8924 assert!(rejected_ids.contains(&cancel.client_order_id));
8925 assert_eq!(
8926 client
8927 .dispatch
8928 .pending_order_action(&cancel.client_order_id),
8929 None,
8930 );
8931 }
8932 assert_nonce_reusable(&client.dispatch);
8933 assert_eq!(
8934 client.dispatch.pending_sendtx_len(),
8935 0,
8936 "local-send-failure must remove batch cancel pending entries",
8937 );
8938 }
8939
8940 #[tokio::test]
8941 async fn batch_cancel_orders_over_max_batch_size_rejects_each_cancel_without_dispatch() {
8942 let (client, cache, mut rx) = create_execution_client();
8943 let instrument_id = register_test_instrument(&client, &cache);
8944 let cancels = (0..=LIGHTER_MAX_BATCH_TX)
8945 .map(|i| {
8946 CancelOrder::new(
8947 trader_id(),
8948 Some(client_id()),
8949 strategy_id(),
8950 instrument_id,
8951 ClientOrderId::from(format!("O-BATCH-CANCEL-MAX-{i}").as_str()),
8952 Some(VenueOrderId::from(format!("{}", 1_000 + i).as_str())),
8953 UUID4::new(),
8954 UnixNanos::default(),
8955 None,
8956 None,
8957 )
8958 })
8959 .collect::<Vec<_>>();
8960
8961 let command = BatchCancelOrders::new(
8962 trader_id(),
8963 Some(client_id()),
8964 strategy_id(),
8965 instrument_id,
8966 cancels.clone(),
8967 UUID4::new(),
8968 UnixNanos::default(),
8969 None,
8970 None,
8971 );
8972 client.batch_cancel_orders(command).unwrap();
8973
8974 for cancel in &cancels {
8975 match recv_order_event(&mut rx).await {
8976 OrderEventAny::CancelRejected(e) => {
8977 assert_eq!(e.client_order_id, cancel.client_order_id);
8978 assert!(e.reason.contains("batch-cancel supports at most 15 txs"),);
8979 }
8980 other => panic!("expected CancelRejected, was {other:?}"),
8981 }
8982 assert_eq!(
8983 client
8984 .dispatch
8985 .pending_order_action(&cancel.client_order_id),
8986 None,
8987 );
8988 }
8989 assert_nonce_reusable(&client.dispatch);
8990 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
8991 }
8992
8993 #[rstest]
8994 #[case::success(200, "batch-a", 0, vec!["hash-1", "hash-2"], 2, false)]
8995 #[case::partial_ack(200, "batch-a", 0, vec!["hash-2"], 1, false)]
8996 #[case::missing_hashes(200, "batch-a", 0, vec![], 0, false)]
8997 #[case::wrong_hashes(200, "batch-a", 0, vec!["unknown"], 0, false)]
8998 #[case::wrong_id(200, "unknown", 0, vec!["hash-1", "hash-2"], 0, false)]
8999 #[case::stale(200, "batch-a", 1, vec!["hash-1", "hash-2"], 0, false)]
9000 #[case::invalid_nonce(21104, "batch-a", 0, vec![], 0, true)]
9001 #[case::rejected(21727, "batch-a", 0, vec![], 0, false)]
9002 #[tokio::test]
9003 async fn batch_cancel_response_correlates_members_and_preserves_nonces(
9004 #[case] code: i64,
9005 #[case] id: &str,
9006 #[case] epoch: u64,
9007 #[case] hashes: Vec<&str>,
9008 #[case] acked_count: usize,
9009 #[case] resync_expected: bool,
9010 ) {
9011 let (client, cache, mut rx) = create_execution_client();
9012 let instrument_id = register_test_instrument(&client, &cache);
9013 for i in 0..3 {
9014 let nonce = client
9015 .dispatch
9016 .nonce_manager
9017 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
9018 .unwrap();
9019 let client_order_id = ClientOrderId::from(format!("O-BATCH-{i}"));
9020 client
9021 .dispatch
9022 .set_pending_order_action(client_order_id, PendingOrderAction::Cancel);
9023 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
9024 batch_id: Some(if i == 0 { "batch-b" } else { "batch-a" }.to_string()),
9025 connection_epoch: 0,
9026 kind: PendingSendTxKind::Cancel {
9027 strategy_id: strategy_id(),
9028 instrument_id,
9029 client_order_id,
9030 venue_order_id: Some(VenueOrderId::from(format!("{}", 100 + i))),
9031 },
9032 submitted_at: UnixNanos::default(),
9033 nonce,
9034 api_key_index: TEST_API_KEY_INDEX,
9035 tx_hash: format!("hash-{i}"),
9036 });
9037 }
9038
9039 assert!(client.dispatch.pop_pending_sendtx_if_only(0).is_none());
9040 let hashes: Vec<_> = hashes.into_iter().map(str::to_string).collect();
9041 let (acked, resync) = handle_send_tx_batch_result(
9042 &client.dispatch,
9043 &client.emitter,
9044 Some(TEST_ACCOUNT_INDEX_I64),
9045 epoch,
9046 UnixNanos::default(),
9047 id,
9048 code,
9049 "batch failure",
9050 &hashes,
9051 );
9052 assert_eq!(resync, resync_expected);
9053 assert_eq!(acked.len(), acked_count);
9054 let rejected = code != 200;
9055 assert_eq!(
9056 client.dispatch.pending_sendtx_len(),
9057 if rejected { 1 } else { 3 - acked_count }
9058 );
9059
9060 if rejected {
9061 for i in [2, 1] {
9062 match recv_order_event(&mut rx).await {
9063 OrderEventAny::CancelRejected(event) => {
9064 assert_eq!(
9065 event.client_order_id,
9066 ClientOrderId::from(format!("O-BATCH-{i}"))
9067 );
9068 assert_eq!(event.strategy_id, strategy_id());
9069 assert_eq!(event.instrument_id, instrument_id);
9070 assert_eq!(
9071 event.venue_order_id,
9072 Some(VenueOrderId::from(format!("{}", 100 + i)))
9073 );
9074 }
9075 event => panic!("expected rejection, was {event:?}"),
9076 }
9077
9078 assert_eq!(
9079 client
9080 .dispatch
9081 .pending_order_action(&ClientOrderId::from(format!("O-BATCH-{i}"))),
9082 None
9083 );
9084 }
9085
9086 assert_eq!(
9087 client
9088 .dispatch
9089 .nonce_manager
9090 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
9091 Some(TEST_NEXT_NONCE)
9092 );
9093 } else {
9094 for pending in &acked {
9095 assert!(hashes.contains(&pending.tx_hash));
9096 }
9097
9098 assert_eq!(
9099 client
9100 .dispatch
9101 .nonce_manager
9102 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
9103 Some(TEST_NEXT_NONCE + 2)
9104 );
9105 }
9106
9107 assert_eq!(
9108 client
9109 .dispatch
9110 .pending_order_action(&ClientOrderId::from("O-BATCH-0")),
9111 Some(PendingOrderAction::Cancel)
9112 );
9113 assert!(
9114 tokio::time::timeout(Duration::from_millis(20), rx.recv())
9115 .await
9116 .is_err()
9117 );
9118 let remaining = client.dispatch.drain_pending_sendtx(0);
9119 assert_eq!(remaining.len(), if rejected { 1 } else { 3 - acked_count });
9120 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
9121 }
9122
9123 #[rstest]
9124 fn cancel_order_from_cancel_all_preserves_tracing_ids() {
9125 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
9126 let client_order_id = ClientOrderId::from("O-CANCEL-ALL-CHILD");
9127 let command_id = UUID4::new();
9128 let correlation_id = UUID4::new();
9129 let causation_id = UUID4::new();
9130 let ts_init = UnixNanos::default();
9131 let mut cmd = CancelAllOrders::new(
9132 trader_id(),
9133 Some(client_id()),
9134 strategy_id(),
9135 instrument_id,
9136 Some(OrderSide::Buy),
9137 command_id,
9138 ts_init,
9139 None,
9140 Some(correlation_id),
9141 );
9142 cmd.causation_id = Some(causation_id);
9143
9144 let owner = StrategyId::from("S-002");
9145 let order_cmd = cancel_order_from_cancel_all(&cmd, client_order_id, owner);
9146
9147 assert_eq!(order_cmd.trader_id, trader_id());
9148 assert_eq!(order_cmd.client_id, Some(client_id()));
9149 assert_eq!(order_cmd.strategy_id, owner);
9150 assert_eq!(order_cmd.instrument_id, instrument_id);
9151 assert_eq!(order_cmd.client_order_id, client_order_id);
9152 assert_eq!(order_cmd.venue_order_id, None);
9153 assert_eq!(order_cmd.command_id, command_id);
9154 assert_eq!(order_cmd.ts_init, ts_init);
9155 assert_eq!(order_cmd.params, None);
9156 assert_eq!(order_cmd.correlation_id, Some(correlation_id));
9157 assert_eq!(order_cmd.causation_id, Some(causation_id));
9158 }
9159
9160 #[tokio::test]
9161 async fn modify_order_send_failure_emits_modify_rejected_and_rolls_back() {
9162 let (client, cache, mut rx) = create_execution_client();
9163 let instrument_id = register_test_instrument(&client, &cache);
9164 let mut factory = test_order_factory();
9165 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-FAIL");
9166 let client_order_id = order.client_order_id();
9167 let venue_order_id = VenueOrderId::from("123");
9168 cache_order(&cache, order);
9169
9170 let command = ModifyOrder::new(
9171 trader_id(),
9172 Some(client_id()),
9173 strategy_id(),
9174 instrument_id,
9175 client_order_id,
9176 Some(venue_order_id),
9177 Some(Quantity::from("0.2000")),
9178 Some(Price::from("2362.00")),
9179 None,
9180 UUID4::new(),
9181 UnixNanos::default(),
9182 None,
9183 None,
9184 );
9185 client.modify_order(command).unwrap();
9186
9187 let rejected = recv_order_event(&mut rx).await;
9188
9189 match rejected {
9190 OrderEventAny::ModifyRejected(event) => {
9191 assert_eq!(event.client_order_id, client_order_id);
9192 assert_eq!(event.instrument_id, instrument_id);
9193 assert_eq!(event.venue_order_id, Some(venue_order_id));
9194 assert!(
9195 event
9196 .reason
9197 .contains("Lighter modify_order dispatch failed"),
9198 );
9199 assert!(event.reason.contains("handler unavailable"));
9200 }
9201 event => panic!("expected modify rejected event, was {event:?}"),
9202 }
9203
9204 assert_nonce_reusable(&client.dispatch);
9205 assert_eq!(
9206 client.dispatch.pending_sendtx_len(),
9207 0,
9208 "local-send-failure must remove the pending modify entry",
9209 );
9210 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
9211 }
9212
9213 #[tokio::test]
9214 async fn modify_order_unknown_send_outcome_retains_pending_state() {
9215 let (client, cache, mut rx) = create_execution_client();
9216 let instrument_id = register_test_instrument(&client, &cache);
9217 let mut factory = test_order_factory();
9218 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-UNKNOWN");
9219 let client_order_id = order.client_order_id();
9220 let venue_order_id = VenueOrderId::from("123");
9221 cache_order(&cache, order);
9222 client.ws_client.drop_next_send_tx_result_for_test().await;
9223
9224 client
9225 .modify_order(ModifyOrder::new(
9226 trader_id(),
9227 Some(client_id()),
9228 strategy_id(),
9229 instrument_id,
9230 client_order_id,
9231 Some(venue_order_id),
9232 Some(Quantity::from("0.2000")),
9233 Some(Price::from("2362.00")),
9234 None,
9235 UUID4::new(),
9236 UnixNanos::default(),
9237 None,
9238 None,
9239 ))
9240 .unwrap();
9241
9242 assert!(
9243 tokio::time::timeout(Duration::from_millis(50), rx.recv())
9244 .await
9245 .is_err(),
9246 "unknown outcome must not emit OrderModifyRejected",
9247 );
9248 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
9249 assert_eq!(
9250 client.dispatch.pending_order_action(&client_order_id),
9251 Some(PendingOrderAction::Modify),
9252 );
9253 assert_eq!(
9254 client
9255 .dispatch
9256 .nonce_manager
9257 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
9258 Some(TEST_NEXT_NONCE),
9259 );
9260 }
9261
9262 #[tokio::test]
9263 async fn modify_order_prepare_failure_emits_modify_rejected_without_dispatch() {
9264 let (client, cache, mut rx) = create_execution_client();
9265 let instrument_id = register_test_instrument(&client, &cache);
9266 let client_order_id = ClientOrderId::from("O-MODIFY-NO-CACHE");
9267 let venue_order_id = VenueOrderId::from("123");
9268
9269 let command = ModifyOrder::new(
9270 trader_id(),
9271 Some(client_id()),
9272 strategy_id(),
9273 instrument_id,
9274 client_order_id,
9275 Some(venue_order_id),
9276 Some(Quantity::from("0.2000")),
9277 Some(Price::from("2362.00")),
9278 None,
9279 UUID4::new(),
9280 UnixNanos::default(),
9281 None,
9282 None,
9283 );
9284 client.modify_order(command).unwrap();
9285
9286 let rejected = recv_order_event(&mut rx).await;
9287 match rejected {
9288 OrderEventAny::ModifyRejected(event) => {
9289 assert_eq!(event.client_order_id, client_order_id);
9290 assert_eq!(event.instrument_id, instrument_id);
9291 assert_eq!(event.venue_order_id, Some(venue_order_id));
9292 assert!(event.reason.contains("Lighter modify_order failed"));
9293 assert!(event.reason.contains("order not found in cache"));
9294 }
9295 event => panic!("expected modify rejected event, was {event:?}"),
9296 }
9297
9298 assert_nonce_reusable(&client.dispatch);
9299 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
9300 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
9301 assert!(
9302 tokio::time::timeout(Duration::from_millis(50), rx.recv())
9303 .await
9304 .is_err(),
9305 "prepare failure must emit exactly one modify rejection",
9306 );
9307 }
9308
9309 #[tokio::test]
9310 async fn modify_order_prepare_failure_emits_modify_rejected_when_instrument_uncached() {
9311 let (client, cache, mut rx) = create_execution_client();
9312 let instrument_id =
9313 client
9314 .registry
9315 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
9316 let mut factory = test_order_factory();
9317 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NO-INSTRUMENT");
9318 let client_order_id = order.client_order_id();
9319 let venue_order_id = VenueOrderId::from("123");
9320 cache_order(&cache, order);
9321
9322 let command = ModifyOrder::new(
9323 trader_id(),
9324 Some(client_id()),
9325 strategy_id(),
9326 instrument_id,
9327 client_order_id,
9328 Some(venue_order_id),
9329 Some(Quantity::from("0.2000")),
9330 Some(Price::from("2362.00")),
9331 None,
9332 UUID4::new(),
9333 UnixNanos::default(),
9334 None,
9335 None,
9336 );
9337 client.modify_order(command).unwrap();
9338
9339 let rejected = recv_order_event(&mut rx).await;
9340 match rejected {
9341 OrderEventAny::ModifyRejected(event) => {
9342 assert_eq!(event.client_order_id, client_order_id);
9343 assert_eq!(event.instrument_id, instrument_id);
9344 assert_eq!(event.venue_order_id, Some(venue_order_id));
9345 assert!(event.reason.contains("Lighter modify_order failed"));
9346 assert!(event.reason.contains("instrument not found in cache"));
9347 }
9348 event => panic!("expected modify rejected event, was {event:?}"),
9349 }
9350
9351 assert_nonce_reusable(&client.dispatch);
9352 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
9353 assert!(
9354 tokio::time::timeout(Duration::from_millis(50), rx.recv())
9355 .await
9356 .is_err(),
9357 "prepare failure must emit exactly one modify rejection",
9358 );
9359 }
9360
9361 #[tokio::test]
9362 async fn modify_stop_market_derives_price_from_trigger_without_explicit_price() {
9363 let (client, cache, mut rx) = create_execution_client();
9368 let instrument_id = register_test_instrument(&client, &cache);
9369 let mut factory = test_order_factory();
9370 let order = factory.stop_market(
9371 instrument_id,
9372 OrderSide::Sell,
9373 Quantity::from("0.1000"),
9374 Price::from("2300.00"), None,
9376 Some(TimeInForce::Gtc),
9377 None,
9378 Some(false),
9379 Some(false),
9380 None,
9381 None,
9382 None,
9383 None,
9384 None,
9385 None,
9386 Some(ClientOrderId::from("O-MODIFY-STOP-MARKET")),
9387 );
9388 let client_order_id = order.client_order_id();
9389 let venue_order_id = VenueOrderId::from("123");
9390 cache_order(&cache, order);
9391
9392 let command = ModifyOrder::new(
9393 trader_id(),
9394 Some(client_id()),
9395 strategy_id(),
9396 instrument_id,
9397 client_order_id,
9398 Some(venue_order_id),
9399 None,
9400 None,
9401 Some(Price::from("2310.00")),
9402 UUID4::new(),
9403 UnixNanos::default(),
9404 None,
9405 None,
9406 );
9407 client.modify_order(command).unwrap();
9408
9409 let rejected = recv_order_event(&mut rx).await;
9410 match rejected {
9411 OrderEventAny::ModifyRejected(event) => {
9412 assert!(
9413 !event.reason.contains("requires a price"),
9414 "trigger-only stop modify must not trip the price guard, was: {}",
9415 event.reason,
9416 );
9417 assert!(
9418 event.reason.contains("dispatch failed"),
9419 "expected send-stage failure after a successful prepare, was: {}",
9420 event.reason,
9421 );
9422 }
9423 event => panic!("expected modify rejected event, was {event:?}"),
9424 }
9425 assert_nonce_reusable(&client.dispatch);
9426 }
9427
9428 #[tokio::test]
9429 async fn modify_order_rejects_sub_tick_quantity_before_nonce_allocation() {
9430 let (client, cache, mut rx) = create_execution_client();
9431 let instrument_id = register_test_instrument(&client, &cache);
9432 let mut factory = test_order_factory();
9433 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-SUB-TICK");
9434 let client_order_id = order.client_order_id();
9435 cache_order(&cache, order);
9436
9437 client
9438 .modify_order(ModifyOrder::new(
9439 trader_id(),
9440 Some(client_id()),
9441 strategy_id(),
9442 instrument_id,
9443 client_order_id,
9444 Some(VenueOrderId::from("123")),
9445 Some(Quantity::from("0.00001")),
9446 Some(Price::from("2362.00")),
9447 None,
9448 UUID4::new(),
9449 UnixNanos::default(),
9450 None,
9451 None,
9452 ))
9453 .unwrap();
9454
9455 assert_modify_rejected_reason(&mut rx, "rounds to 0 ticks").await;
9456 assert_nonce_reusable(&client.dispatch);
9457 }
9458
9459 #[tokio::test]
9460 async fn modify_order_rejects_below_min_notional_before_nonce_allocation() {
9461 let (client, cache, mut rx) = create_execution_client();
9462 let instrument_id = register_test_instrument(&client, &cache);
9463 let mut factory = test_order_factory();
9464 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-MIN-NOTIONAL");
9465 let client_order_id = order.client_order_id();
9466 cache_order(&cache, order);
9467
9468 client
9469 .modify_order(ModifyOrder::new(
9470 trader_id(),
9471 Some(client_id()),
9472 strategy_id(),
9473 instrument_id,
9474 client_order_id,
9475 Some(VenueOrderId::from("123")),
9476 Some(Quantity::from("0.0001")),
9477 Some(Price::from("2362.00")),
9478 None,
9479 UUID4::new(),
9480 UnixNanos::default(),
9481 None,
9482 None,
9483 ))
9484 .unwrap();
9485
9486 assert_modify_rejected_reason(&mut rx, "below Lighter min_quote_amount").await;
9487 assert_nonce_reusable(&client.dispatch);
9488 }
9489
9490 #[tokio::test]
9491 async fn modify_conditional_order_rejects_sub_tick_trigger_before_nonce_allocation() {
9492 let (client, cache, mut rx) = create_execution_client();
9493 let instrument_id = register_test_instrument(&client, &cache);
9494 let mut factory = test_order_factory();
9495 let order = factory.stop_market(
9496 instrument_id,
9497 OrderSide::Sell,
9498 Quantity::from("0.1000"),
9499 Price::from("2300.00"),
9500 None,
9501 Some(TimeInForce::Gtc),
9502 None,
9503 Some(false),
9504 Some(false),
9505 None,
9506 None,
9507 None,
9508 None,
9509 None,
9510 None,
9511 Some(ClientOrderId::from("O-MODIFY-SUB-TICK-TRIGGER")),
9512 );
9513 let client_order_id = order.client_order_id();
9514 cache_order(&cache, order);
9515
9516 client
9517 .modify_order(ModifyOrder::new(
9518 trader_id(),
9519 Some(client_id()),
9520 strategy_id(),
9521 instrument_id,
9522 client_order_id,
9523 Some(VenueOrderId::from("123")),
9524 None,
9525 None,
9526 Some(Price::from("0.001")),
9527 UUID4::new(),
9528 UnixNanos::default(),
9529 None,
9530 None,
9531 ))
9532 .unwrap();
9533
9534 assert_modify_rejected_reason(&mut rx, "rounds to 0 ticks").await;
9535 assert_nonce_reusable(&client.dispatch);
9536 }
9537
9538 #[tokio::test]
9539 async fn modify_order_nonce_prepare_failure_emits_modify_rejected() {
9540 let mut config = test_config();
9541 config.base_url_http = Some(spawn_next_nonce_server(101).await);
9542 let (client, cache, mut rx) = create_execution_client_with_config(config);
9543 let instrument_id = register_test_instrument(&client, &cache);
9544 let mut factory = test_order_factory();
9545 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NONCE-FAIL");
9546 let client_order_id = order.client_order_id();
9547 let venue_order_id = VenueOrderId::from("123");
9548 cache_order(&cache, order);
9549
9550 let window = i64::from(client.dispatch.nonce_manager.skip_window());
9551 for _ in 0..window {
9552 client
9553 .dispatch
9554 .nonce_manager
9555 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
9556 .unwrap();
9557 }
9558
9559 let command = ModifyOrder::new(
9560 trader_id(),
9561 Some(client_id()),
9562 strategy_id(),
9563 instrument_id,
9564 client_order_id,
9565 Some(venue_order_id),
9566 Some(Quantity::from("0.2000")),
9567 Some(Price::from("2362.00")),
9568 None,
9569 UUID4::new(),
9570 UnixNanos::default(),
9571 None,
9572 None,
9573 );
9574 client.modify_order(command).unwrap();
9575
9576 let rejected = recv_order_event(&mut rx).await;
9577 match rejected {
9578 OrderEventAny::ModifyRejected(event) => {
9579 assert_eq!(event.client_order_id, client_order_id);
9580 assert_eq!(event.venue_order_id, Some(venue_order_id));
9581 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
9582 assert!(event.reason.contains("skip-window exhausted"));
9583 }
9584 event => panic!("expected modify rejected event, was {event:?}"),
9585 }
9586
9587 wait_for_spawned_tasks(&client).await;
9588 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
9589 assert_eq!(
9590 client
9591 .dispatch
9592 .nonce_manager
9593 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
9594 .unwrap(),
9595 101,
9596 );
9597 }
9598
9599 #[tokio::test]
9600 async fn update_leverage_requires_credentials() {
9601 let (mut client, _cache, _rx) = create_execution_client();
9602 client.credential = None;
9603 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
9604
9605 let err = client
9606 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
9607 .unwrap_err();
9608
9609 assert!(
9610 err.to_string()
9611 .contains("cannot update leverage without credentials"),
9612 );
9613 }
9614
9615 #[tokio::test]
9616 async fn update_leverage_requires_registered_instrument() {
9617 let (client, _cache, _rx) = create_execution_client();
9618 let unknown = InstrumentId::from("DOGE-PERP.LIGHTER");
9619
9620 let err = client
9621 .update_leverage(unknown, 500, LighterPositionMarginMode::Isolated)
9622 .unwrap_err();
9623
9624 assert!(
9625 err.to_string()
9626 .contains("no Lighter market_index registered")
9627 );
9628 assert_nonce_reusable(&client.dispatch);
9631 }
9632
9633 #[tokio::test]
9634 async fn update_leverage_dispatches_and_rolls_back_on_send_failure() {
9635 let (client, cache, _rx) = create_execution_client();
9636 let instrument_id = register_test_instrument(&client, &cache);
9637
9638 client
9639 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
9640 .unwrap();
9641
9642 wait_for_spawned_tasks(&client).await;
9643 assert_nonce_reusable(&client.dispatch);
9644 }
9645
9646 #[tokio::test]
9647 async fn update_leverage_unknown_send_outcome_retains_pending_nonce() {
9648 let (client, cache, _rx) = create_execution_client();
9649 let instrument_id = register_test_instrument(&client, &cache);
9650 client.ws_client.drop_next_send_tx_result_for_test().await;
9651
9652 client
9653 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
9654 .unwrap();
9655
9656 wait_for_spawned_tasks(&client).await;
9657 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
9658 assert_eq!(
9659 client
9660 .dispatch
9661 .nonce_manager
9662 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
9663 Some(TEST_NEXT_NONCE),
9664 );
9665 }
9666
9667 #[tokio::test]
9668 async fn update_leverage_rejects_zero_margin_fraction() {
9669 let (client, cache, _rx) = create_execution_client();
9670 let instrument_id = register_test_instrument(&client, &cache);
9671
9672 let err = client
9673 .update_leverage(instrument_id, 0, LighterPositionMarginMode::Cross)
9674 .unwrap_err();
9675 assert!(err.to_string().contains("must be in 1..=10_000"));
9676 }
9677
9678 #[tokio::test]
9679 async fn update_leverage_rejects_above_margin_fraction_tick() {
9680 let (client, cache, _rx) = create_execution_client();
9681 let instrument_id = register_test_instrument(&client, &cache);
9682
9683 let err = client
9684 .update_leverage(instrument_id, 10_001, LighterPositionMarginMode::Cross)
9685 .unwrap_err();
9686 assert!(err.to_string().contains("must be in 1..=10_000"));
9687 }
9688
9689 #[tokio::test]
9690 async fn update_leverage_accepts_minimum_margin_fraction() {
9691 let (client, cache, _rx) = create_execution_client();
9694 let instrument_id = register_test_instrument(&client, &cache);
9695
9696 client
9697 .update_leverage(instrument_id, 1, LighterPositionMarginMode::Cross)
9698 .unwrap();
9699
9700 wait_for_spawned_tasks(&client).await;
9701 assert_nonce_reusable(&client.dispatch);
9702 }
9703
9704 #[tokio::test]
9705 async fn update_leverage_accepts_maximum_margin_fraction() {
9706 let (client, cache, _rx) = create_execution_client();
9709 let instrument_id = register_test_instrument(&client, &cache);
9710
9711 client
9712 .update_leverage(instrument_id, 10_000, LighterPositionMarginMode::Isolated)
9713 .unwrap();
9714
9715 wait_for_spawned_tasks(&client).await;
9716 assert_nonce_reusable(&client.dispatch);
9717 }
9718
9719 async fn wait_for_spawned_tasks(client: &LighterExecutionClient) {
9720 wait_until_async(
9721 || async { client.pending_tasks_all_finished() },
9722 Duration::from_secs(2),
9723 )
9724 .await;
9725 }
9726
9727 fn mark_all_streams_ready(client: &LighterExecutionClient) {
9728 let ready = &client.dispatch.account_streams_ready;
9729 ready.mark_orders();
9730 ready.mark_trades();
9731 ready.mark_positions();
9732 ready.mark_assets();
9733 ready.mark_user_stats();
9734 }
9735
9736 #[tokio::test]
9737 async fn await_account_streams_ready_times_out_when_no_frame_arrives() {
9738 let (client, _cache, _rx) = create_execution_client();
9741
9742 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
9743
9744 assert!(
9745 err.to_string().contains("Timeout")
9746 && err.to_string().contains("Lighter account streams"),
9747 "unexpected error message, was {err}",
9748 );
9749 }
9750
9751 #[tokio::test]
9752 async fn await_account_streams_ready_returns_when_all_streams_marked() {
9753 let (client, _cache, _rx) = create_execution_client();
9754 mark_all_streams_ready(&client);
9755
9756 client.await_account_streams_ready(0.05).await.unwrap();
9757 }
9758
9759 #[tokio::test]
9760 async fn await_account_streams_ready_returns_when_streams_arrive_mid_wait() {
9761 let (client, _cache, _rx) = create_execution_client();
9764 let ready = Arc::clone(&client.dispatch.account_streams_ready);
9765
9766 let wait = client.await_account_streams_ready(1.0);
9767 let seed = async move {
9768 tokio::time::sleep(Duration::from_millis(20)).await;
9769 ready.mark_orders();
9770 ready.mark_trades();
9771 ready.mark_positions();
9772 ready.mark_assets();
9773 ready.mark_user_stats();
9774 };
9775
9776 let (result, ()) = tokio::join!(wait, seed);
9777 result.unwrap();
9778 }
9779
9780 #[tokio::test]
9781 async fn await_account_streams_ready_times_out_with_partial_marks() {
9782 let (client, _cache, _rx) = create_execution_client();
9786 let ready = &client.dispatch.account_streams_ready;
9787 ready.mark_orders();
9788 ready.mark_trades();
9789 ready.mark_positions();
9790
9791 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
9792 assert!(
9793 err.to_string().contains("assets"),
9794 "pending list should call out the missing stream, was {err}",
9795 );
9796 }
9797
9798 #[tokio::test]
9799 async fn await_account_streams_ready_after_reset_requires_new_marks() {
9800 let (client, _cache, _rx) = create_execution_client();
9805 mark_all_streams_ready(&client);
9806 client.await_account_streams_ready(0.05).await.unwrap();
9807
9808 client.dispatch.account_streams_ready.reset();
9809
9810 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
9811 let msg = err.to_string();
9812 assert!(msg.contains("orders"), "pending list missing orders: {msg}");
9813 assert!(msg.contains("trades"), "pending list missing trades: {msg}");
9814 assert!(
9815 msg.contains("positions"),
9816 "pending list missing positions: {msg}",
9817 );
9818 assert!(msg.contains("assets"), "pending list missing assets: {msg}");
9819 assert!(
9820 msg.contains("user_stats"),
9821 "pending list missing user_stats: {msg}",
9822 );
9823 }
9824
9825 fn test_market_order(
9826 factory: &mut OrderFactory,
9827 instrument_id: InstrumentId,
9828 client_order_id: &str,
9829 side: OrderSide,
9830 ) -> OrderAny {
9831 factory.market(
9832 instrument_id,
9833 side,
9834 Quantity::from("0.1000"),
9835 Some(TimeInForce::Ioc),
9836 Some(false),
9837 Some(false),
9838 None,
9839 None,
9840 None,
9841 Some(ClientOrderId::from(client_order_id)),
9842 )
9843 }
9844
9845 fn add_test_quote(
9846 cache: &Rc<RefCell<Cache>>,
9847 instrument_id: InstrumentId,
9848 bid: &str,
9849 ask: &str,
9850 ) {
9851 let quote = QuoteTick::new(
9852 instrument_id,
9853 Price::from(bid),
9854 Price::from(ask),
9855 Quantity::from("1.0000"),
9856 Quantity::from("1.0000"),
9857 UnixNanos::default(),
9858 UnixNanos::default(),
9859 );
9860 cache.borrow_mut().add_quote(quote).unwrap();
9861 }
9862
9863 #[tokio::test]
9864 async fn submit_market_order_without_cached_quote_emits_denied() {
9865 let (client, cache, mut rx) = create_execution_client();
9866 let instrument_id = register_test_instrument(&client, &cache);
9867 let mut factory = test_order_factory();
9868 let order = test_market_order(
9869 &mut factory,
9870 instrument_id,
9871 "O-MARKET-NO-QUOTE",
9872 OrderSide::Buy,
9873 );
9874 cache_order(&cache, order.clone());
9875
9876 let command = SubmitOrder::from_order(
9877 &order,
9878 trader_id(),
9879 Some(client_id()),
9880 None,
9881 UUID4::new(),
9882 UnixNanos::default(),
9883 );
9884 let _ = client.submit_order(command);
9886
9887 let event = recv_order_event(&mut rx).await;
9888 match event {
9889 OrderEventAny::Denied(event) => {
9890 assert!(
9891 event.reason.contains("no cached quote"),
9892 "expected no-cached-quote in reason, was {:?}",
9893 event.reason,
9894 );
9895 }
9896 event => panic!("expected denied event, was {event:?}"),
9897 }
9898 assert_nonce_reusable(&client.dispatch);
9899 }
9900
9901 #[tokio::test]
9902 async fn submit_market_buy_with_quote_uses_ask_widened_by_slippage() {
9903 let (client, cache, mut rx) = create_execution_client();
9904 let instrument_id = register_test_instrument(&client, &cache);
9905 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
9906
9907 let mut factory = test_order_factory();
9908 let order = test_market_order(
9909 &mut factory,
9910 instrument_id,
9911 "O-MARKET-QUOTED-BUY",
9912 OrderSide::Buy,
9913 );
9914 cache_order(&cache, order.clone());
9915
9916 let command = SubmitOrder::from_order(
9917 &order,
9918 trader_id(),
9919 Some(client_id()),
9920 None,
9921 UUID4::new(),
9922 UnixNanos::default(),
9923 );
9924 let _ = client.submit_order(command);
9925
9926 let submitted = recv_order_event(&mut rx).await;
9927 assert!(
9928 matches!(submitted, OrderEventAny::Submitted(_)),
9929 "expected submitted, was {submitted:?}",
9930 );
9931 let rejected = recv_order_event(&mut rx).await;
9932 match rejected {
9933 OrderEventAny::Rejected(event) => {
9934 assert!(
9935 event
9936 .reason
9937 .contains("Lighter submit_order dispatch failed"),
9938 );
9939 }
9940 event => panic!("expected rejected event, was {event:?}"),
9941 }
9942 assert_nonce_reusable(&client.dispatch);
9943 }
9944
9945 #[tokio::test]
9946 async fn submit_order_with_sub_tick_quantity_emits_denied() {
9947 let (client, cache, mut rx) = create_execution_client();
9948 let instrument_id = register_test_instrument(&client, &cache);
9949 let mut factory = test_order_factory();
9950 let order = factory.limit(
9952 instrument_id,
9953 OrderSide::Buy,
9954 Quantity::from("0.00001"),
9955 Price::from("2361.31"),
9956 Some(TimeInForce::Gtc),
9957 None,
9958 Some(false),
9959 Some(false),
9960 None,
9961 None,
9962 None,
9963 None,
9964 None,
9965 None,
9966 None,
9967 Some(ClientOrderId::from("O-SUB-TICK-QTY")),
9968 );
9969 cache_order(&cache, order.clone());
9970
9971 let command = SubmitOrder::from_order(
9972 &order,
9973 trader_id(),
9974 Some(client_id()),
9975 None,
9976 UUID4::new(),
9977 UnixNanos::default(),
9978 );
9979 let _ = client.submit_order(command);
9980
9981 let event = recv_order_event(&mut rx).await;
9982 match event {
9983 OrderEventAny::Denied(event) => {
9984 assert!(
9985 event.reason.contains("rounds to 0 ticks"),
9986 "expected rounds-to-0 in reason, was {:?}",
9987 event.reason,
9988 );
9989 }
9990 event => panic!("expected denied event, was {event:?}"),
9991 }
9992 assert_nonce_reusable(&client.dispatch);
9993 }
9994
9995 #[tokio::test]
9996 async fn submit_order_below_min_notional_emits_denied() {
9997 let (client, cache, mut rx) = create_execution_client();
9998 let instrument_id = register_test_instrument(&client, &cache);
9999 let mut factory = test_order_factory();
10000 let order = factory.limit(
10001 instrument_id,
10002 OrderSide::Buy,
10003 Quantity::from("0.0010"),
10004 Price::from("2361.31"),
10005 Some(TimeInForce::Gtc),
10006 None,
10007 Some(false),
10008 Some(false),
10009 None,
10010 None,
10011 None,
10012 None,
10013 None,
10014 None,
10015 None,
10016 Some(ClientOrderId::from("O-BELOW-MIN-NOTIONAL")),
10017 );
10018 cache_order(&cache, order.clone());
10019
10020 let command = SubmitOrder::from_order(
10021 &order,
10022 trader_id(),
10023 Some(client_id()),
10024 None,
10025 UUID4::new(),
10026 UnixNanos::default(),
10027 );
10028 let _ = client.submit_order(command);
10029
10030 let event = recv_order_event(&mut rx).await;
10031 match event {
10032 OrderEventAny::Denied(event) => {
10033 assert!(
10034 event.reason.contains("min_quote_amount"),
10035 "expected min_quote_amount in reason, was {:?}",
10036 event.reason,
10037 );
10038 }
10039 event => panic!("expected denied event, was {event:?}"),
10040 }
10041 assert_nonce_reusable(&client.dispatch);
10042 }
10043
10044 #[tokio::test]
10045 async fn submit_stop_market_with_sub_tick_trigger_emits_denied() {
10046 let (client, cache, mut rx) = create_execution_client();
10047 let instrument_id = register_test_instrument(&client, &cache);
10048 let mut factory = test_order_factory();
10049 let order = factory.stop_market(
10051 instrument_id,
10052 OrderSide::Buy,
10053 Quantity::from("0.1000"),
10054 Price::from("0.001"),
10055 None,
10056 Some(TimeInForce::Gtc),
10057 None,
10058 Some(false),
10059 Some(false),
10060 None,
10061 None,
10062 None,
10063 None,
10064 None,
10065 None,
10066 Some(ClientOrderId::from("O-STOP-SUB-TICK")),
10067 );
10068 cache_order(&cache, order.clone());
10069
10070 let command = SubmitOrder::from_order(
10071 &order,
10072 trader_id(),
10073 Some(client_id()),
10074 None,
10075 UUID4::new(),
10076 UnixNanos::default(),
10077 );
10078 let _ = client.submit_order(command);
10079
10080 let event = recv_order_event(&mut rx).await;
10081 match event {
10082 OrderEventAny::Denied(event) => {
10083 assert!(
10084 event.reason.contains("rounds to 0 ticks"),
10085 "expected rounds-to-0 in reason, was {:?}",
10086 event.reason,
10087 );
10088 }
10089 event => panic!("expected denied event, was {event:?}"),
10090 }
10091 assert_nonce_reusable(&client.dispatch);
10092 }
10093
10094 #[tokio::test]
10095 async fn submit_stop_market_dispatches_using_trigger_widened_by_slippage() {
10096 let (client, cache, mut rx) = create_execution_client();
10097 let instrument_id = register_test_instrument(&client, &cache);
10098 let mut factory = test_order_factory();
10099 let order = factory.stop_market(
10100 instrument_id,
10101 OrderSide::Sell,
10102 Quantity::from("0.1000"),
10103 Price::from("2300.00"), None,
10105 Some(TimeInForce::Gtc),
10106 None,
10107 Some(false),
10108 Some(false),
10109 None,
10110 None,
10111 None,
10112 None,
10113 None,
10114 None,
10115 Some(ClientOrderId::from("O-STOP-MARKET")),
10116 );
10117 cache_order(&cache, order.clone());
10118
10119 let command = SubmitOrder::from_order(
10120 &order,
10121 trader_id(),
10122 Some(client_id()),
10123 None,
10124 UUID4::new(),
10125 UnixNanos::default(),
10126 );
10127 let _ = client.submit_order(command);
10128
10129 let submitted = recv_order_event(&mut rx).await;
10130 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
10131 let rejected = recv_order_event(&mut rx).await;
10132 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
10133 assert_nonce_reusable(&client.dispatch);
10134 }
10135
10136 #[tokio::test]
10137 async fn submit_market_order_respects_per_order_slippage_override() {
10138 let (client, cache, mut rx) = create_execution_client();
10141 let instrument_id = register_test_instrument(&client, &cache);
10142 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
10143
10144 let mut factory = test_order_factory();
10145 let order = test_market_order(
10146 &mut factory,
10147 instrument_id,
10148 "O-MARKET-ZERO-SLIP",
10149 OrderSide::Buy,
10150 );
10151 cache_order(&cache, order.clone());
10152
10153 let params: Params =
10154 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0})).unwrap();
10155 let mut command = SubmitOrder::from_order(
10156 &order,
10157 trader_id(),
10158 Some(client_id()),
10159 None,
10160 UUID4::new(),
10161 UnixNanos::default(),
10162 );
10163 command.params = Some(params);
10164 let _ = client.submit_order(command);
10165
10166 let submitted = recv_order_event(&mut rx).await;
10167 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
10168 let rejected = recv_order_event(&mut rx).await;
10169 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
10170 assert_nonce_reusable(&client.dispatch);
10171 }
10172
10173 #[tokio::test]
10174 async fn resolve_slippage_bps_prefers_params_over_config_default() {
10175 let (client, _cache, _rx) = create_execution_client();
10176 assert_eq!(client.resolve_slippage_bps(None), 50);
10177
10178 let override_params: Params =
10179 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 100})).unwrap();
10180 assert_eq!(client.resolve_slippage_bps(Some(&override_params)), 100);
10181
10182 let unrelated_params: Params =
10183 serde_json::from_value(serde_json::json!({"other_key": 999})).unwrap();
10184 assert_eq!(client.resolve_slippage_bps(Some(&unrelated_params)), 50);
10185 }
10186
10187 #[tokio::test]
10188 async fn submit_market_sell_with_quote_uses_bid_widened_by_slippage() {
10189 let (client, cache, mut rx) = create_execution_client();
10190 let instrument_id = register_test_instrument(&client, &cache);
10191 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
10192
10193 let mut factory = test_order_factory();
10194 let order = test_market_order(
10195 &mut factory,
10196 instrument_id,
10197 "O-MARKET-QUOTED-SELL",
10198 OrderSide::Sell,
10199 );
10200 cache_order(&cache, order.clone());
10201
10202 let command = SubmitOrder::from_order(
10203 &order,
10204 trader_id(),
10205 Some(client_id()),
10206 None,
10207 UUID4::new(),
10208 UnixNanos::default(),
10209 );
10210 let _ = client.submit_order(command);
10211
10212 let submitted = recv_order_event(&mut rx).await;
10213 assert!(
10214 matches!(submitted, OrderEventAny::Submitted(_)),
10215 "expected submitted, was {submitted:?}",
10216 );
10217 let rejected = recv_order_event(&mut rx).await;
10218 match rejected {
10219 OrderEventAny::Rejected(event) => {
10220 assert!(
10221 event
10222 .reason
10223 .contains("Lighter submit_order dispatch failed"),
10224 );
10225 }
10226 event => panic!("expected rejected event, was {event:?}"),
10227 }
10228 assert_nonce_reusable(&client.dispatch);
10229 }
10230
10231 #[rstest]
10232 fn integrator_attributes_tag_mainnet_orders() {
10233 assert_eq!(
10234 integrator_attributes(Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)),
10235 L2TxAttributes {
10236 integrator_account_index: LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
10237 ..Default::default()
10238 },
10239 );
10240 }
10241
10242 #[rstest]
10243 fn integrator_attributes_leave_testnet_orders_unattributed() {
10244 assert_eq!(integrator_attributes(None), L2TxAttributes::default());
10245 }
10246
10247 #[rstest]
10248 #[case::unavailable(None, None)]
10249 #[case::standard(Some(LighterAccountTier::Standard), None)]
10250 #[case::premium(
10251 Some(LighterAccountTier::Premium),
10252 Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
10253 )]
10254 #[case::plus(
10255 Some(LighterAccountTier::Plus),
10256 Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
10257 )]
10258 #[case::builder(Some(LighterAccountTier::Builder), None)]
10259 #[case::unknown(Some(LighterAccountTier::Unknown(7)), None)]
10260 fn integrator_account_index_follows_account_tier(
10261 #[case] account_tier: Option<LighterAccountTier>,
10262 #[case] expected: Option<u64>,
10263 ) {
10264 let mut config = test_config();
10265 config.environment = LighterEnvironment::Mainnet;
10266 let (mut client, _cache, _rx) = create_execution_client_with_config(config);
10267 client.account_tier = account_tier;
10268
10269 assert_eq!(client.integrator_account_index(), expected);
10270 }
10271
10272 #[rstest]
10273 fn robinhood_orders_keep_default_l2_attributes() {
10274 let mut config = test_config();
10275 config.deployment = LighterDeployment::Robinhood;
10276 config.environment = LighterEnvironment::Mainnet;
10277
10278 assert_eq!(
10279 integrator_attributes(deployment::integrator_account_index(
10280 config.deployment,
10281 config.environment,
10282 )),
10283 L2TxAttributes::default(),
10284 );
10285 }
10286
10287 use std::str::FromStr;
10288
10289 use nautilus_live::ExecutionEventEmitter;
10290 use rust_decimal::Decimal;
10291
10292 use crate::{
10293 common::enums::{
10294 LighterOrderKind, LighterOrderSide, LighterOrderStatus, LighterOrderTimeInForce,
10295 LighterTradeType, LighterTriggerStatus,
10296 },
10297 http::models::{LighterOrder, LighterTrade},
10298 };
10299
10300 fn dispatcher_emitter() -> (
10301 ExecutionEventEmitter,
10302 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10303 ) {
10304 let mut emitter = ExecutionEventEmitter::new(
10305 get_atomic_clock_realtime(),
10306 trader_id(),
10307 account_id(),
10308 AccountType::Margin,
10309 None,
10310 );
10311 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
10312 emitter.set_sender(sender);
10313 (emitter, receiver)
10314 }
10315
10316 struct DispatcherRig {
10320 dispatch: WsDispatchState,
10321 registry: Arc<MarketRegistry>,
10322 emitter: ExecutionEventEmitter,
10323 rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10324 instrument_id: InstrumentId,
10325 cloid: ClientOrderId,
10326 }
10327
10328 fn dispatcher_rig(cloid_suffix: &str) -> DispatcherRig {
10329 let registry = Arc::new(MarketRegistry::new());
10330 let instrument_id = registry.insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
10335 let instrument = InstrumentAny::CryptoPerpetual(
10336 CryptoPerpetual::builder()
10337 .instrument_id(instrument_id)
10338 .raw_symbol(Symbol::new("ETH-PERP"))
10339 .base_currency(Currency::from("ETH"))
10340 .quote_currency(Currency::from("USDC"))
10341 .settlement_currency(Currency::from("USDC"))
10342 .is_inverse(false)
10343 .price_precision(2)
10344 .size_precision(4)
10345 .price_increment(Price::from("0.01"))
10346 .size_increment(Quantity::from("0.0001"))
10347 .min_notional(Money::from("10.000000 USDC"))
10348 .ts_event(UnixNanos::default())
10349 .ts_init(UnixNanos::default())
10350 .build()
10351 .unwrap(),
10352 );
10353 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument);
10354 let (emitter, rx) = dispatcher_emitter();
10355 DispatcherRig {
10356 dispatch: WsDispatchState::new(),
10357 registry,
10358 emitter,
10359 rx,
10360 instrument_id,
10361 cloid: ClientOrderId::new(format!("CLOID-{cloid_suffix}")),
10362 }
10363 }
10364
10365 fn register_identity(rig: &DispatcherRig) {
10366 register_identity_for(rig, OrderType::Limit);
10367 }
10368
10369 fn register_identity_for(rig: &DispatcherRig, order_type: OrderType) {
10370 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
10371 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
10372 rig.dispatch.register_order_identity(
10373 rig.cloid,
10374 OrderIdentity::new(
10375 rig.instrument_id,
10376 strategy_id(),
10377 OrderSide::Buy,
10378 order_type,
10379 client_order_index,
10380 ),
10381 );
10382 rig.dispatch
10383 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
10384 }
10385
10386 fn register_unowned_identity(rig: &DispatcherRig) {
10387 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
10388 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
10389 rig.dispatch.register_order_identity(
10390 rig.cloid,
10391 OrderIdentity::new(
10392 rig.instrument_id,
10393 strategy_id(),
10394 OrderSide::Buy,
10395 OrderType::Limit,
10396 client_order_index,
10397 ),
10398 );
10399 }
10400
10401 fn dispatcher_test_order(rig: &DispatcherRig, status: LighterOrderStatus) -> LighterOrder {
10402 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
10403 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
10404
10405 LighterOrder {
10406 order_index: 281_476_929_510_110,
10407 client_order_index: derived,
10408 order_id: "281476929510110".to_string(),
10409 client_order_id: derived.to_string(),
10410 market_index: TEST_MARKET_INDEX,
10411 owner_account_index: TEST_ACCOUNT_INDEX_I64,
10412 initial_base_amount: Decimal::from_str("0.0050").unwrap(),
10413 price: Decimal::from_str("2352.74").unwrap(),
10414 nonce: TEST_ORDER_NONCE,
10415 remaining_base_amount: Decimal::from_str("0.0050").unwrap(),
10416 is_ask: false,
10417 base_size: 50,
10418 base_price: 235_274,
10419 filled_base_amount: Decimal::ZERO,
10420 filled_quote_amount: Decimal::ZERO,
10421 side: Some(LighterOrderSide::Buy),
10422 order_type: LighterOrderKind::Limit,
10423 time_in_force: LighterOrderTimeInForce::GoodTillTime,
10424 reduce_only: false,
10425 trigger_price: Decimal::ZERO,
10426 order_expiry: 1_780_360_584_479,
10427 status,
10428 trigger_status: LighterTriggerStatus::Na,
10429 trigger_time: 0,
10430 parent_order_index: 0,
10431 parent_order_id: "0".to_string(),
10432 to_trigger_order_id_0: "0".to_string(),
10433 to_trigger_order_id_1: "0".to_string(),
10434 to_cancel_order_id_0: "0".to_string(),
10435 integrator_fee_collector_index: "0".to_string(),
10436 integrator_taker_fee: Decimal::ZERO,
10437 integrator_maker_fee: Decimal::ZERO,
10438 block_height: 227_535_532,
10439 timestamp: 1_777_941_383_576,
10440 created_at: 1_777_941_383_576,
10441 updated_at: 1_777_941_383_900,
10442 transaction_time: 1_777_941_383_576_735,
10443 }
10444 }
10445
10446 fn dispatcher_test_trade(rig: &DispatcherRig, user_is_bidder: bool) -> LighterTrade {
10447 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
10448 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
10449 LighterTrade {
10450 trade_id: 19_209_006_902,
10451 trade_id_str: Some("19209006902".to_string()),
10452 tx_hash: "000000128b1ee814".to_string(),
10453 trade_type: LighterTradeType::Trade,
10454 market_id: TEST_MARKET_INDEX,
10455 size: Decimal::from_str("0.1336").unwrap(),
10456 price: Decimal::from_str("2352.73").unwrap(),
10457 usd_amount: Decimal::from_str("314.324728").unwrap(),
10458 ask_id: 281_476_929_510_102,
10459 ask_id_str: Some("281476929510102".to_string()),
10460 bid_id: 562_947_905_631_053,
10461 bid_id_str: Some("562947905631053".to_string()),
10462 ask_client_id: if user_is_bidder { 0 } else { derived },
10463 ask_client_id_str: Some(if user_is_bidder {
10464 "0".to_string()
10465 } else {
10466 derived.to_string()
10467 }),
10468 bid_client_id: if user_is_bidder { derived } else { 0 },
10469 bid_client_id_str: Some(if user_is_bidder {
10470 derived.to_string()
10471 } else {
10472 "0".to_string()
10473 }),
10474 ask_account_id: if user_is_bidder {
10475 91_249
10476 } else {
10477 TEST_ACCOUNT_INDEX_I64
10478 },
10479 bid_account_id: if user_is_bidder {
10480 TEST_ACCOUNT_INDEX_I64
10481 } else {
10482 91_249
10483 },
10484 is_maker_ask: false,
10485 block_height: 227_535_535,
10486 timestamp: 1_777_941_384_181,
10487 taker_fee: Some(196),
10488 taker_position_size_before: None,
10489 taker_entry_quote_before: None,
10490 taker_initial_margin_fraction_before: None,
10491 taker_position_sign_changed: None,
10492 maker_fee: Some(28),
10493 maker_position_size_before: None,
10494 maker_entry_quote_before: None,
10495 maker_initial_margin_fraction_before: None,
10496 maker_position_sign_changed: None,
10497 transaction_time: 1_777_941_384_181_586,
10498 ask_account_pnl: None,
10499 bid_account_pnl: None,
10500 }
10501 }
10502
10503 fn drain_events(
10507 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10508 ) -> Vec<ExecutionEvent> {
10509 let mut events = Vec::new();
10510 while let Ok(event) = rx.try_recv() {
10511 events.push(event);
10512 }
10513 events
10514 }
10515
10516 #[rstest]
10517 fn dispatch_lighter_order_tracked_emits_accepted_then_silent_repeat() {
10518 let mut rig = dispatcher_rig("1");
10519 register_identity(&rig);
10520 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
10521
10522 dispatch_lighter_order(
10523 &order,
10524 &rig.dispatch,
10525 &rig.emitter,
10526 &rig.registry,
10527 account_id(),
10528 trader_id(),
10529 UnixNanos::from(1),
10530 );
10531 dispatch_lighter_order(
10532 &order,
10533 &rig.dispatch,
10534 &rig.emitter,
10535 &rig.registry,
10536 account_id(),
10537 trader_id(),
10538 UnixNanos::from(2),
10539 );
10540
10541 let events = drain_events(&mut rig.rx);
10542 assert_eq!(
10543 events.len(),
10544 1,
10545 "exactly one event expected, was {events:?}",
10546 );
10547
10548 match &events[0] {
10549 ExecutionEvent::Order(OrderEventAny::Accepted(e)) => {
10550 assert_eq!(e.client_order_id, rig.cloid);
10551 assert_eq!(e.venue_order_id.to_string(), "281476929510110");
10552 }
10553 other => panic!("expected Accepted, was {other:?}"),
10554 }
10555 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
10556 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_some());
10557 }
10558
10559 #[rstest]
10560 fn dispatch_lighter_order_tracked_emits_updated_on_shape_change() {
10561 let mut rig = dispatcher_rig("2");
10562 register_identity(&rig);
10563 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
10564
10565 dispatch_lighter_order(
10566 &order,
10567 &rig.dispatch,
10568 &rig.emitter,
10569 &rig.registry,
10570 account_id(),
10571 trader_id(),
10572 UnixNanos::from(1),
10573 );
10574 assert_eq!(drain_events(&mut rig.rx).len(), 1);
10575
10576 let mut modified = order;
10577 modified.price = Decimal::from_str("2400.00").unwrap();
10578
10579 dispatch_lighter_order(
10580 &modified,
10581 &rig.dispatch,
10582 &rig.emitter,
10583 &rig.registry,
10584 account_id(),
10585 trader_id(),
10586 UnixNanos::from(2),
10587 );
10588
10589 let events = drain_events(&mut rig.rx);
10590 assert_eq!(
10591 events.len(),
10592 1,
10593 "expected one Updated event, was {events:?}",
10594 );
10595
10596 match &events[0] {
10597 ExecutionEvent::Order(OrderEventAny::Updated(e)) => {
10598 assert_eq!(e.client_order_id, rig.cloid);
10599 assert_eq!(e.price, Some(Price::from("2400.00")));
10600 }
10601 other => panic!("expected Updated, was {other:?}"),
10602 }
10603 let snapshot = rig.dispatch.snapshot_for(&rig.cloid).expect("snapshot");
10604 assert_eq!(snapshot.price, Some(Price::from("2400.00")));
10605 }
10606
10607 #[rstest]
10608 #[case(LighterOrderStatus::Pending)]
10609 #[case(LighterOrderStatus::Open)]
10610 fn dispatch_lighter_order_defers_live_frame_while_cancel_pending(
10611 #[case] status: LighterOrderStatus,
10612 ) {
10613 let mut rig = dispatcher_rig(match status {
10614 LighterOrderStatus::Pending => "24-PENDING",
10615 LighterOrderStatus::Open => "24-OPEN",
10616 other => panic!("unexpected case {other:?}"),
10617 });
10618 register_identity(&rig);
10619 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
10620
10621 dispatch_lighter_order(
10622 &order,
10623 &rig.dispatch,
10624 &rig.emitter,
10625 &rig.registry,
10626 account_id(),
10627 trader_id(),
10628 UnixNanos::from(1),
10629 );
10630 assert_eq!(drain_events(&mut rig.rx).len(), 1);
10631
10632 rig.dispatch
10633 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
10634 let mut late = order;
10635 late.status = status;
10636 late.initial_base_amount = Decimal::from_str("0.0075").unwrap();
10637 late.price = Decimal::from_str("2400.00").unwrap();
10638
10639 dispatch_lighter_order(
10640 &late,
10641 &rig.dispatch,
10642 &rig.emitter,
10643 &rig.registry,
10644 account_id(),
10645 trader_id(),
10646 UnixNanos::from(2),
10647 );
10648
10649 let events = drain_events(&mut rig.rx);
10650 assert_eq!(events.len(), 0, "late frame must be silent: {events:?}");
10651 assert_eq!(
10652 rig.dispatch.pending_order_action(&rig.cloid),
10653 Some(PendingOrderAction::Cancel),
10654 );
10655 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
10656 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
10657 assert_eq!(
10658 rig.dispatch.lookup_venue_order_id(&rig.cloid),
10659 Some(VenueOrderId::new("281476929510110")),
10660 );
10661 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
10662 assert_eq!(
10663 rig.dispatch.snapshot_for(&rig.cloid),
10664 Some(crate::websocket::dispatch::OrderShapeSnapshot {
10665 quantity: Quantity::from("0.0050"),
10666 price: Some(Price::from("2352.74")),
10667 trigger_price: None,
10668 }),
10669 );
10670 }
10671
10672 #[rstest]
10673 fn dispatch_lighter_order_emits_updated_then_triggered_for_one_frame() {
10674 let mut rig = dispatcher_rig("25");
10675 register_identity_for(&rig, OrderType::StopLimit);
10676 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
10677 order.order_type = LighterOrderKind::StopLossLimit;
10678 order.trigger_price = Decimal::from_str("2300.00").unwrap();
10679
10680 dispatch_lighter_order(
10681 &order,
10682 &rig.dispatch,
10683 &rig.emitter,
10684 &rig.registry,
10685 account_id(),
10686 trader_id(),
10687 UnixNanos::from(1),
10688 );
10689 let accepted = drain_events(&mut rig.rx);
10690 assert_eq!(accepted.len(), 1, "expected one acceptance: {accepted:?}");
10691 assert!(matches!(
10692 accepted[0],
10693 ExecutionEvent::Order(OrderEventAny::Accepted(_)),
10694 ));
10695
10696 rig.dispatch
10697 .set_pending_order_action(rig.cloid, PendingOrderAction::Modify);
10698 let mut triggered = order;
10699 triggered.status = LighterOrderStatus::Open;
10700 triggered.trigger_status = LighterTriggerStatus::Ready;
10701 triggered.initial_base_amount = Decimal::from_str("0.0075").unwrap();
10702 triggered.price = Decimal::from_str("2400.00").unwrap();
10703 triggered.trigger_price = Decimal::from_str("2390.00").unwrap();
10704
10705 dispatch_lighter_order(
10706 &triggered,
10707 &rig.dispatch,
10708 &rig.emitter,
10709 &rig.registry,
10710 account_id(),
10711 trader_id(),
10712 UnixNanos::from(2),
10713 );
10714
10715 let events = drain_events(&mut rig.rx);
10716 assert_eq!(
10717 events.len(),
10718 2,
10719 "expected Updated then Triggered: {events:?}",
10720 );
10721
10722 match &events[0] {
10723 ExecutionEvent::Order(OrderEventAny::Updated(event)) => {
10724 assert_eq!(event.trader_id, trader_id());
10725 assert_eq!(event.strategy_id, strategy_id());
10726 assert_eq!(event.instrument_id, rig.instrument_id);
10727 assert_eq!(event.client_order_id, rig.cloid);
10728 assert_eq!(
10729 event.venue_order_id,
10730 Some(VenueOrderId::new("281476929510110")),
10731 );
10732 assert_eq!(event.account_id, Some(account_id()));
10733 assert_eq!(event.quantity, Quantity::from("0.0075"));
10734 assert_eq!(event.price, Some(Price::from("2400.00")));
10735 assert_eq!(event.trigger_price, Some(Price::from("2390.00")));
10736 }
10737 other => panic!("first event should be Updated, was {other:?}"),
10738 }
10739
10740 match &events[1] {
10741 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
10742 assert_eq!(event.trader_id, trader_id());
10743 assert_eq!(event.strategy_id, strategy_id());
10744 assert_eq!(event.instrument_id, rig.instrument_id);
10745 assert_eq!(event.client_order_id, rig.cloid);
10746 assert_eq!(
10747 event.venue_order_id,
10748 Some(VenueOrderId::new("281476929510110")),
10749 );
10750 assert_eq!(event.account_id, Some(account_id()));
10751 }
10752 other => panic!("second event should be Triggered, was {other:?}"),
10753 }
10754 assert_eq!(rig.dispatch.pending_order_action(&rig.cloid), None);
10755 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
10756 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
10757 assert_eq!(
10758 rig.dispatch.snapshot_for(&rig.cloid),
10759 Some(crate::websocket::dispatch::OrderShapeSnapshot {
10760 quantity: Quantity::from("0.0075"),
10761 price: Some(Price::from("2400.00")),
10762 trigger_price: Some(Price::from("2390.00")),
10763 }),
10764 );
10765
10766 dispatch_lighter_order(
10767 &triggered,
10768 &rig.dispatch,
10769 &rig.emitter,
10770 &rig.registry,
10771 account_id(),
10772 trader_id(),
10773 UnixNanos::from(3),
10774 );
10775 assert!(
10776 drain_events(&mut rig.rx).is_empty(),
10777 "replayed frame must be silent",
10778 );
10779 }
10780
10781 #[rstest]
10782 fn dispatch_lighter_order_defers_trigger_until_modify_resolves() {
10783 let mut rig = dispatcher_rig("26");
10784 register_identity_for(&rig, OrderType::StopLimit);
10785 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
10786 order.order_type = LighterOrderKind::StopLossLimit;
10787 order.trigger_price = Decimal::from_str("2300.00").unwrap();
10788
10789 dispatch_lighter_order(
10790 &order,
10791 &rig.dispatch,
10792 &rig.emitter,
10793 &rig.registry,
10794 account_id(),
10795 trader_id(),
10796 UnixNanos::from(1),
10797 );
10798 assert_eq!(drain_events(&mut rig.rx).len(), 1);
10799
10800 rig.dispatch
10801 .set_pending_order_action(rig.cloid, PendingOrderAction::Modify);
10802 order.status = LighterOrderStatus::Open;
10803 order.trigger_status = LighterTriggerStatus::Ready;
10804 dispatch_lighter_order(
10805 &order,
10806 &rig.dispatch,
10807 &rig.emitter,
10808 &rig.registry,
10809 account_id(),
10810 trader_id(),
10811 UnixNanos::from(2),
10812 );
10813
10814 assert!(drain_events(&mut rig.rx).is_empty());
10815 assert_eq!(
10816 rig.dispatch.pending_order_action(&rig.cloid),
10817 Some(PendingOrderAction::Modify),
10818 );
10819 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
10820
10821 assert!(
10822 rig.dispatch
10823 .clear_pending_order_action_if(&rig.cloid, PendingOrderAction::Modify,)
10824 );
10825 dispatch_lighter_order(
10826 &order,
10827 &rig.dispatch,
10828 &rig.emitter,
10829 &rig.registry,
10830 account_id(),
10831 trader_id(),
10832 UnixNanos::from(3),
10833 );
10834
10835 let events = drain_events(&mut rig.rx);
10836 assert_eq!(events.len(), 1, "expected deferred trigger: {events:?}");
10837 match &events[0] {
10838 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
10839 assert_eq!(event.client_order_id, rig.cloid);
10840 assert_eq!(
10841 event.venue_order_id,
10842 Some(VenueOrderId::new("281476929510110")),
10843 );
10844 assert_eq!(event.account_id, Some(account_id()));
10845 }
10846 other => panic!("expected Triggered, was {other:?}"),
10847 }
10848 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
10849 }
10850
10851 #[rstest]
10852 fn dispatch_lighter_order_pending_emits_accepted_then_updated() {
10853 let mut rig = dispatcher_rig("23");
10854 register_identity(&rig);
10855 let order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
10856
10857 dispatch_lighter_order(
10858 &order,
10859 &rig.dispatch,
10860 &rig.emitter,
10861 &rig.registry,
10862 account_id(),
10863 trader_id(),
10864 UnixNanos::from(1),
10865 );
10866 let accepted = drain_events(&mut rig.rx);
10867
10868 let mut modified = order;
10869 modified.price = Decimal::from_str("2400.00").unwrap();
10870 dispatch_lighter_order(
10871 &modified,
10872 &rig.dispatch,
10873 &rig.emitter,
10874 &rig.registry,
10875 account_id(),
10876 trader_id(),
10877 UnixNanos::from(2),
10878 );
10879 let updated = drain_events(&mut rig.rx);
10880
10881 assert_eq!(
10882 accepted.len(),
10883 1,
10884 "expected one Accepted event: {accepted:?}"
10885 );
10886 assert!(matches!(
10887 accepted[0],
10888 ExecutionEvent::Order(OrderEventAny::Accepted(_))
10889 ));
10890 assert_eq!(updated.len(), 1, "expected one Updated event: {updated:?}");
10891 assert!(matches!(
10892 updated[0],
10893 ExecutionEvent::Order(OrderEventAny::Updated(_))
10894 ));
10895 assert_eq!(
10896 rig.dispatch
10897 .snapshot_for(&rig.cloid)
10898 .expect("snapshot")
10899 .price,
10900 Some(Price::from("2400.00")),
10901 );
10902 }
10903
10904 #[rstest]
10905 fn dispatch_lighter_order_untracked_emits_report() {
10906 let mut rig = dispatcher_rig("3");
10907 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
10909 order.client_order_id = "external-1".to_string();
10910 order.client_order_index = 0;
10911
10912 dispatch_lighter_order(
10913 &order,
10914 &rig.dispatch,
10915 &rig.emitter,
10916 &rig.registry,
10917 account_id(),
10918 trader_id(),
10919 UnixNanos::from(1),
10920 );
10921
10922 let events = drain_events(&mut rig.rx);
10923 assert_eq!(events.len(), 1);
10924 match &events[0] {
10925 ExecutionEvent::Report(report) => match report {
10926 EngineExecutionReport::Order(r) => {
10927 assert_eq!(r.venue_order_id.to_string(), "281476929510110");
10928 }
10929 other => panic!("expected order report, was {other:?}"),
10930 },
10931 other => panic!("expected report, was {other:?}"),
10932 }
10933 assert!(
10934 !rig.dispatch
10935 .accepted_was_emitted(&ClientOrderId::new("external-1"))
10936 );
10937 }
10938
10939 #[rstest]
10940 fn same_index_external_order_waits_for_create_submission_ownership() {
10941 let mut rig = dispatcher_rig("EPOCH-ORDER");
10942 register_unowned_identity(&rig);
10943 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
10944 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
10945
10946 dispatch_lighter_order(
10947 &order,
10948 &rig.dispatch,
10949 &rig.emitter,
10950 &rig.registry,
10951 account_id(),
10952 trader_id(),
10953 UnixNanos::from(1),
10954 );
10955
10956 let external = drain_events(&mut rig.rx);
10957 assert_eq!(external.len(), 1);
10958 match &external[0] {
10959 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
10960 assert_eq!(report.venue_order_id, venue_order_id);
10961 assert_eq!(
10962 report.client_order_id,
10963 Some(ClientOrderId::new(venue_order_id.as_str())),
10964 );
10965 }
10966 other => panic!("expected one external order report, was {other:?}"),
10967 }
10968 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
10969 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
10970
10971 rig.dispatch
10972 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
10973 dispatch_lighter_order(
10974 &order,
10975 &rig.dispatch,
10976 &rig.emitter,
10977 &rig.registry,
10978 account_id(),
10979 trader_id(),
10980 UnixNanos::from(2),
10981 );
10982
10983 let replayed_external = drain_events(&mut rig.rx);
10984 assert_eq!(replayed_external.len(), 1);
10985 match &replayed_external[0] {
10986 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
10987 assert_eq!(report.venue_order_id, venue_order_id);
10988 assert_eq!(
10989 report.client_order_id,
10990 Some(ClientOrderId::new(venue_order_id.as_str())),
10991 );
10992 }
10993 other => panic!("expected replayed external order report, was {other:?}"),
10994 }
10995 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
10996 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
10997
10998 let mut local_order = order.clone();
10999 local_order.order_id = "281476929510111".to_string();
11000 let local_venue_order_id = VenueOrderId::new(local_order.order_id.as_str());
11001 dispatch_lighter_order(
11002 &local_order,
11003 &rig.dispatch,
11004 &rig.emitter,
11005 &rig.registry,
11006 account_id(),
11007 trader_id(),
11008 UnixNanos::from(3),
11009 );
11010
11011 let local = drain_events(&mut rig.rx);
11012 assert_eq!(local.len(), 1);
11013 match &local[0] {
11014 ExecutionEvent::Order(OrderEventAny::Accepted(event)) => {
11015 assert_eq!(event.client_order_id, rig.cloid);
11016 assert_eq!(event.venue_order_id, local_venue_order_id);
11017 assert_eq!(event.account_id, account_id());
11018 }
11019 other => panic!("expected one local OrderAccepted, was {other:?}"),
11020 }
11021 assert_eq!(
11022 rig.dispatch
11023 .venue_id_map
11024 .get(&rig.cloid)
11025 .map(|entry| *entry),
11026 Some(local_venue_order_id),
11027 );
11028 assert_eq!(
11029 rig.dispatch
11030 .resolve_live_order_cloid(order.client_order_id.as_str(), local_venue_order_id,),
11031 Some(rig.cloid),
11032 "a bound identity remains stable across replay epochs",
11033 );
11034 }
11035
11036 #[rstest]
11037 fn dispatch_lighter_trade_tracked_synthesizes_accepted_before_filled() {
11038 let mut rig = dispatcher_rig("4");
11042 register_identity(&rig);
11043
11044 let trade = dispatcher_test_trade(&rig, true);
11045
11046 dispatch_lighter_trade(
11047 &trade,
11048 &rig.dispatch,
11049 &rig.emitter,
11050 &rig.registry,
11051 account_id(),
11052 trader_id(),
11053 Some(TEST_ACCOUNT_INDEX_I64),
11054 UnixNanos::from(1),
11055 );
11056
11057 let events = drain_events(&mut rig.rx);
11058 assert_eq!(
11059 events.len(),
11060 2,
11061 "expected Accepted then Filled, was {events:?}",
11062 );
11063
11064 match &events[0] {
11065 ExecutionEvent::Order(OrderEventAny::Accepted(_)) => {}
11066 other => panic!("first event should be Accepted, was {other:?}"),
11067 }
11068
11069 match &events[1] {
11070 ExecutionEvent::Order(OrderEventAny::Filled(e)) => {
11071 assert_eq!(e.client_order_id, rig.cloid);
11072 assert_eq!(e.last_qty, Quantity::from("0.1336"));
11073 assert_eq!(e.last_px, Price::from("2352.73"));
11074 }
11075 other => panic!("second event should be Filled, was {other:?}"),
11076 }
11077 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
11078 }
11079
11080 #[rstest]
11081 fn dispatch_lighter_trade_dedupes_repeated_trade_ids() {
11082 let mut rig = dispatcher_rig("5");
11083 register_identity(&rig);
11084 let trade = dispatcher_test_trade(&rig, true);
11085
11086 for _ in 0..3 {
11087 dispatch_lighter_trade(
11088 &trade,
11089 &rig.dispatch,
11090 &rig.emitter,
11091 &rig.registry,
11092 account_id(),
11093 trader_id(),
11094 Some(TEST_ACCOUNT_INDEX_I64),
11095 UnixNanos::from(1),
11096 );
11097 }
11098
11099 let events = drain_events(&mut rig.rx);
11100 assert_eq!(
11102 events.len(),
11103 2,
11104 "expected dedup after first dispatch, was {events:?}"
11105 );
11106 }
11107
11108 #[rstest]
11109 fn dispatch_lighter_trade_parse_failure_rolls_back_dedup() {
11110 let mut rig = dispatcher_rig("21");
11112 register_identity(&rig);
11113 let mut trade = dispatcher_test_trade(&rig, true);
11114 trade.timestamp = -1;
11115 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
11116
11117 dispatch_lighter_trade(
11118 &trade,
11119 &rig.dispatch,
11120 &rig.emitter,
11121 &rig.registry,
11122 account_id(),
11123 trader_id(),
11124 Some(TEST_ACCOUNT_INDEX_I64),
11125 UnixNanos::from(1),
11126 );
11127
11128 let events = drain_events(&mut rig.rx);
11129 assert!(
11130 !events
11131 .iter()
11132 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
11133 "malformed trade must not emit a fill, was {events:?}",
11134 );
11135 assert!(
11136 rig.dispatch.mark_trade_seen(trade_id),
11137 "parse-failed trade must be re-markable (dedup rolled back)",
11138 );
11139 }
11140
11141 #[rstest]
11142 fn dispatch_lighter_trade_untracked_parse_failure_rolls_back_dedup() {
11143 let mut rig = dispatcher_rig("22");
11145 let mut trade = dispatcher_test_trade(&rig, true);
11146 trade.timestamp = -1;
11147 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
11148
11149 dispatch_lighter_trade(
11150 &trade,
11151 &rig.dispatch,
11152 &rig.emitter,
11153 &rig.registry,
11154 account_id(),
11155 trader_id(),
11156 Some(TEST_ACCOUNT_INDEX_I64),
11157 UnixNanos::from(1),
11158 );
11159
11160 let events = drain_events(&mut rig.rx);
11161 assert!(
11162 events.is_empty(),
11163 "untracked malformed trade must emit nothing, was {events:?}",
11164 );
11165 assert!(
11166 rig.dispatch.mark_trade_seen(trade_id),
11167 "untracked parse-failed trade must be re-markable (dedup rolled back)",
11168 );
11169 }
11170
11171 #[rstest]
11172 fn dispatch_tracked_order_event_terminal_cancel_retires_identity_and_removes_snapshot() {
11173 let mut rig = dispatcher_rig("6");
11174 register_identity(&rig);
11175 rig.dispatch.mark_accepted_emitted(rig.cloid);
11176 rig.dispatch.store_snapshot(
11177 rig.cloid,
11178 crate::websocket::dispatch::OrderShapeSnapshot {
11179 quantity: Quantity::from("0.0050"),
11180 price: Some(Price::from("2352.74")),
11181 trigger_price: None,
11182 },
11183 );
11184 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
11185 order.filled_base_amount = Decimal::ZERO;
11186 let mut trade = dispatcher_test_trade(&rig, true);
11187 trade.bid_id = order.order_index;
11188 trade.bid_id_str = Some(order.order_id.clone());
11189 dispatch_lighter_order(
11190 &order,
11191 &rig.dispatch,
11192 &rig.emitter,
11193 &rig.registry,
11194 account_id(),
11195 trader_id(),
11196 UnixNanos::from(1),
11197 );
11198
11199 let events = drain_events(&mut rig.rx);
11200 let canceled = events
11201 .iter()
11202 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Canceled(_))))
11203 .expect("expected a Canceled event");
11204 if let ExecutionEvent::Order(OrderEventAny::Canceled(e)) = canceled {
11205 assert_eq!(e.client_order_id, rig.cloid);
11206 }
11207 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
11208 assert!(rig.dispatch.order_identity(&rig.cloid).is_some());
11209 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_none());
11210 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
11211
11212 dispatch_lighter_trade(
11213 &trade,
11214 &rig.dispatch,
11215 &rig.emitter,
11216 &rig.registry,
11217 account_id(),
11218 trader_id(),
11219 Some(TEST_ACCOUNT_INDEX_I64),
11220 UnixNanos::from(2),
11221 );
11222 let trailing = drain_events(&mut rig.rx);
11223 assert_eq!(
11224 trailing.len(),
11225 1,
11226 "expected one trailing fill: {trailing:?}"
11227 );
11228
11229 match &trailing[0] {
11230 ExecutionEvent::Order(OrderEventAny::Filled(event)) => {
11231 assert_eq!(event.client_order_id, rig.cloid);
11232 }
11233 other => panic!("expected typed trailing OrderFilled, was {other:?}"),
11234 }
11235 }
11236
11237 #[rstest]
11238 fn dispatch_tracked_cancel_after_report_seed_skips_synthesized_accept() {
11239 let mut rig = dispatcher_rig("10");
11240 register_identity(&rig);
11241
11242 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11243 let instrument = LIGHTER_INSTRUMENT_CACHE
11244 .get(&rig.instrument_id)
11245 .expect("instrument cached");
11246 let report = parse_ws_order_status_report(
11247 &report_order,
11248 instrument.value(),
11249 account_id(),
11250 UnixNanos::from(1),
11251 )
11252 .map(|report| report.with_client_order_id(rig.cloid))
11253 .expect("report parses");
11254
11255 assert_eq!(report.order_status, OrderStatus::Accepted);
11256 rig.dispatch.seed_accepted_from_report(&report);
11257
11258 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
11259 cancel_order.filled_base_amount = Decimal::ZERO;
11260 dispatch_lighter_order(
11261 &cancel_order,
11262 &rig.dispatch,
11263 &rig.emitter,
11264 &rig.registry,
11265 account_id(),
11266 trader_id(),
11267 UnixNanos::from(2),
11268 );
11269
11270 let events = drain_events(&mut rig.rx);
11271 assert_eq!(
11272 events.len(),
11273 1,
11274 "report-seeded cancel should emit only Canceled, was {events:?}",
11275 );
11276 assert!(
11277 !events
11278 .iter()
11279 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_)))),
11280 "typed cancel must not synthesize a second Accepted",
11281 );
11282
11283 match &events[0] {
11284 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
11285 assert_eq!(e.client_order_id, rig.cloid);
11286 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
11287 }
11288 other => panic!("expected Canceled, was {other:?}"),
11289 }
11290 }
11291
11292 #[rstest]
11293 fn dispatch_tracked_cancel_after_submitted_report_seed_skips_synthesized_accept() {
11294 let mut rig = dispatcher_rig("11");
11295 register_identity(&rig);
11296
11297 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11298 let instrument = LIGHTER_INSTRUMENT_CACHE
11299 .get(&rig.instrument_id)
11300 .expect("instrument cached");
11301 let mut report = parse_ws_order_status_report(
11302 &report_order,
11303 instrument.value(),
11304 account_id(),
11305 UnixNanos::from(1),
11306 )
11307 .map(|report| report.with_client_order_id(rig.cloid))
11308 .expect("report parses");
11309 report.order_status = OrderStatus::Submitted;
11310
11311 rig.dispatch.seed_accepted_from_report(&report);
11312 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
11313
11314 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
11315 cancel_order.filled_base_amount = Decimal::ZERO;
11316
11317 dispatch_lighter_order(
11318 &cancel_order,
11319 &rig.dispatch,
11320 &rig.emitter,
11321 &rig.registry,
11322 account_id(),
11323 trader_id(),
11324 UnixNanos::from(2),
11325 );
11326
11327 let events = drain_events(&mut rig.rx);
11328 assert_eq!(
11329 events.len(),
11330 1,
11331 "Cancel after Submitted report should emit only Canceled, was {events:?}",
11332 );
11333
11334 match &events[0] {
11335 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
11336 assert_eq!(e.client_order_id, rig.cloid);
11337 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
11338 }
11339 other => panic!("expected Canceled, was {other:?}"),
11340 }
11341 }
11342
11343 #[rstest]
11344 fn dispatch_tracked_order_event_accept_dedup_is_idempotent() {
11345 let mut rig = dispatcher_rig("7");
11346 register_identity(&rig);
11347 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11348
11349 dispatch_lighter_order(
11352 &order,
11353 &rig.dispatch,
11354 &rig.emitter,
11355 &rig.registry,
11356 account_id(),
11357 trader_id(),
11358 UnixNanos::from(1),
11359 );
11360 dispatch_lighter_order(
11361 &order,
11362 &rig.dispatch,
11363 &rig.emitter,
11364 &rig.registry,
11365 account_id(),
11366 trader_id(),
11367 UnixNanos::from(2),
11368 );
11369
11370 let events = drain_events(&mut rig.rx);
11371 let accepted_count = events
11372 .iter()
11373 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_))))
11374 .count();
11375 assert_eq!(accepted_count, 1, "Accepted must be emitted exactly once");
11376 }
11377
11378 #[rstest]
11379 fn dispatch_lighter_order_drops_when_instrument_uncached() {
11380 let registry = Arc::new(MarketRegistry::new());
11382 let (emitter, mut rx) = dispatcher_emitter();
11383 let dispatch = WsDispatchState::new();
11384 let cloid = ClientOrderId::new("CLOID-MISSING");
11385 dispatch.register_order_identity(
11386 cloid,
11387 OrderIdentity::new(
11388 InstrumentId::from("MISSING-PERP.LIGHTER"),
11389 strategy_id(),
11390 OrderSide::Buy,
11391 OrderType::Limit,
11392 1,
11393 ),
11394 );
11395 let mut order = LighterOrder {
11396 order_index: 1,
11397 client_order_index: 1,
11398 order_id: "1".to_string(),
11399 client_order_id: "1".to_string(),
11400 market_index: 999, owner_account_index: TEST_ACCOUNT_INDEX_I64,
11402 initial_base_amount: Decimal::ZERO,
11403 price: Decimal::ZERO,
11404 nonce: 0,
11405 remaining_base_amount: Decimal::ZERO,
11406 is_ask: false,
11407 base_size: 0,
11408 base_price: 0,
11409 filled_base_amount: Decimal::ZERO,
11410 filled_quote_amount: Decimal::ZERO,
11411 side: Some(LighterOrderSide::Buy),
11412 order_type: LighterOrderKind::Limit,
11413 time_in_force: LighterOrderTimeInForce::GoodTillTime,
11414 reduce_only: false,
11415 trigger_price: Decimal::ZERO,
11416 order_expiry: 0,
11417 status: LighterOrderStatus::Open,
11418 trigger_status: LighterTriggerStatus::Na,
11419 trigger_time: 0,
11420 parent_order_index: 0,
11421 parent_order_id: "0".to_string(),
11422 to_trigger_order_id_0: "0".to_string(),
11423 to_trigger_order_id_1: "0".to_string(),
11424 to_cancel_order_id_0: "0".to_string(),
11425 integrator_fee_collector_index: "0".to_string(),
11426 integrator_taker_fee: Decimal::ZERO,
11427 integrator_maker_fee: Decimal::ZERO,
11428 block_height: 0,
11429 timestamp: 0,
11430 created_at: 0,
11431 updated_at: 0,
11432 transaction_time: 0,
11433 };
11434 order.client_order_id = "1".to_string();
11435
11436 dispatch_lighter_order(
11437 &order,
11438 &dispatch,
11439 &emitter,
11440 ®istry,
11441 account_id(),
11442 trader_id(),
11443 UnixNanos::from(1),
11444 );
11445
11446 let events = drain_events(&mut rx);
11447 assert!(
11448 events.is_empty(),
11449 "no event for uncached instrument, was {events:?}"
11450 );
11451 assert!(dispatch.order_identities.contains_key(&cloid));
11452 }
11453
11454 #[rstest]
11455 fn dispatch_lighter_trade_filters_non_account_trades_defensively() {
11456 let mut rig = dispatcher_rig("8");
11457 let mut trade = dispatcher_test_trade(&rig, true);
11462 trade.bid_account_id = 91_249;
11463 trade.ask_account_id = 91_250;
11464
11465 dispatch_lighter_trade(
11466 &trade,
11467 &rig.dispatch,
11468 &rig.emitter,
11469 &rig.registry,
11470 account_id(),
11471 trader_id(),
11472 Some(TEST_ACCOUNT_INDEX_I64),
11473 UnixNanos::from(1),
11474 );
11475
11476 let events = drain_events(&mut rig.rx);
11477 assert!(
11478 events.is_empty(),
11479 "foreign trade must produce no event, was {events:?}"
11480 );
11481 assert!(
11482 !rig.dispatch
11483 .seen_trade_ids
11484 .contains(&TradeId::new("19209006902"),)
11485 );
11486 }
11487
11488 #[rstest]
11489 fn register_cloid_in_submit_path_uses_probed_index_on_collision() {
11490 let dispatch = WsDispatchState::new();
11495 let cloid = ClientOrderId::new("PROBE-CLOID");
11496 let derived = dispatch.derive_client_order_index(&cloid);
11497
11498 let intruder = ClientOrderId::new("INTRUDER");
11499 dispatch.cloid_map.insert(derived, intruder);
11500
11501 let chosen = dispatch.register_cloid(derived, cloid).unwrap();
11502
11503 assert_ne!(chosen, derived);
11504 assert_eq!(
11505 dispatch.cloid_map.get(&derived).map(|e| *e.value()),
11506 Some(intruder),
11507 );
11508 assert_eq!(
11509 dispatch.cloid_map.get(&chosen).map(|e| *e.value()),
11510 Some(cloid),
11511 );
11512 }
11513
11514 #[rstest]
11515 fn reconciliation_restores_active_collision_probed_identity() {
11516 let (client, cache, _rx) = create_execution_client();
11517 let instrument_id = register_test_instrument(&client, &cache);
11518 let mut factory = test_order_factory();
11519 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-ACTIVE");
11520 let cloid = order.client_order_id();
11521 let venue_order_id = VenueOrderId::from("281476929510110");
11522 cache_accepted_order(&cache, order, venue_order_id, None);
11523 let (base_index, client_order_index) = forced_probed_index(cloid);
11524 let raw =
11525 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
11526
11527 restore_reconciled_order(&client.core, &client.dispatch, &raw, false);
11528 let report =
11529 parse_http_order_to_report(&raw, &client.registry, account_id(), UnixNanos::from(1))
11530 .unwrap();
11531 let translated = client.dispatch.translate_order_cloid(report);
11532 let replacement = ClientOrderId::from("O-RECON-ACTIVE-REPLACEMENT");
11533 let replacement_index = client
11534 .dispatch
11535 .register_cloid(client_order_index, replacement)
11536 .unwrap();
11537 let wrong_venue_order_id = VenueOrderId::from("281476929510111");
11538 let wrong_raw = reconciliation_raw_order(
11539 client_order_index,
11540 wrong_venue_order_id,
11541 LighterOrderStatus::Open,
11542 );
11543 let wrong_report = parse_http_order_to_report(
11544 &wrong_raw,
11545 &client.registry,
11546 account_id(),
11547 UnixNanos::from(2),
11548 )
11549 .unwrap();
11550 let wrong_translated = client.dispatch.translate_order_cloid(wrong_report);
11551
11552 assert_ne!(client_order_index, base_index);
11553 assert_eq!(
11554 client.dispatch.client_order_index(&cloid),
11555 Some(client_order_index),
11556 );
11557 assert_eq!(
11558 client
11559 .dispatch
11560 .resolve_client_order_index(client_order_index),
11561 Some(cloid),
11562 );
11563 assert_eq!(
11564 client.dispatch.lookup_venue_order_id(&cloid),
11565 Some(venue_order_id),
11566 );
11567 assert!(client.dispatch.order_identities.contains_key(&cloid));
11568 assert!(client.dispatch.accepted_was_emitted(&cloid));
11569 assert_eq!(translated.client_order_id, Some(cloid));
11570 assert_eq!(translated.venue_order_id, venue_order_id);
11571 assert_ne!(replacement_index, client_order_index);
11572 assert_eq!(
11573 client
11574 .dispatch
11575 .resolve_client_order_index(replacement_index),
11576 Some(replacement),
11577 );
11578 assert_eq!(
11579 wrong_translated.client_order_id,
11580 Some(ClientOrderId::new(wrong_venue_order_id.as_str())),
11581 );
11582 assert_eq!(wrong_translated.venue_order_id, wrong_venue_order_id);
11583 }
11584
11585 #[rstest]
11586 fn reconciliation_retires_stale_active_cache_from_terminal_report() {
11587 let (client, cache, _rx) = create_execution_client();
11588 let instrument_id = register_test_instrument(&client, &cache);
11589 let mut factory = test_order_factory();
11590 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-FILLED");
11591 let cloid = order.client_order_id();
11592 let venue_order_id = VenueOrderId::from("281476929510119");
11593 cache_accepted_order(&cache, order, venue_order_id, None);
11594 let (_, client_order_index) = forced_probed_index(cloid);
11595 let raw = reconciliation_raw_order(
11596 client_order_index,
11597 venue_order_id,
11598 LighterOrderStatus::Filled,
11599 );
11600
11601 restore_reconciled_order(&client.core, &client.dispatch, &raw, true);
11602 let fill = client
11603 .dispatch
11604 .translate_fill_cloid(reconciliation_fill_report(
11605 instrument_id,
11606 client_order_index,
11607 venue_order_id,
11608 "19209006919",
11609 ));
11610
11611 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
11612 assert!(!client.dispatch.order_identities.contains_key(&cloid));
11613 assert_eq!(
11614 client.dispatch.client_order_index(&cloid),
11615 Some(client_order_index),
11616 );
11617 assert_eq!(fill.client_order_id, Some(cloid));
11618 assert_eq!(fill.venue_order_id, venue_order_id);
11619 }
11620
11621 #[rstest]
11622 fn reconciliation_restores_reused_retired_index_for_fill_translation() {
11623 let (client, cache, _rx) = create_execution_client();
11624 let instrument_id = register_test_instrument(&client, &cache);
11625 let mut factory = test_order_factory();
11626 let first_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-A");
11627 let second_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-B");
11628 let first_cloid = first_order.client_order_id();
11629 let second_cloid = second_order.client_order_id();
11630 let first_venue_order_id = VenueOrderId::from("281476929510120");
11631 let second_venue_order_id = VenueOrderId::from("281476929510121");
11632 cache_canceled_order(&cache, first_order, first_venue_order_id);
11633 cache_canceled_order(&cache, second_order, second_venue_order_id);
11634 let (_, client_order_index) = forced_probed_index(first_cloid);
11635 let first_raw = reconciliation_raw_order(
11636 client_order_index,
11637 first_venue_order_id,
11638 LighterOrderStatus::Canceled,
11639 );
11640 let second_raw = reconciliation_raw_order(
11641 client_order_index,
11642 second_venue_order_id,
11643 LighterOrderStatus::Canceled,
11644 );
11645
11646 restore_reconciled_order(&client.core, &client.dispatch, &first_raw, true);
11647 restore_reconciled_order(&client.core, &client.dispatch, &second_raw, true);
11648 let first_fill = client
11649 .dispatch
11650 .translate_fill_cloid(reconciliation_fill_report(
11651 instrument_id,
11652 client_order_index,
11653 first_venue_order_id,
11654 "19209006920",
11655 ));
11656 let second_fill = client
11657 .dispatch
11658 .translate_fill_cloid(reconciliation_fill_report(
11659 instrument_id,
11660 client_order_index,
11661 second_venue_order_id,
11662 "19209006921",
11663 ));
11664 let replacement = ClientOrderId::from("O-RECON-RETIRED-REPLACEMENT");
11665 let replacement_index = client
11666 .dispatch
11667 .register_cloid(client_order_index, replacement)
11668 .unwrap();
11669
11670 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
11671 assert!(!client.dispatch.order_identities.contains_key(&first_cloid));
11672 assert!(!client.dispatch.order_identities.contains_key(&second_cloid));
11673 assert_eq!(
11674 client.dispatch.client_order_index(&first_cloid),
11675 Some(client_order_index),
11676 );
11677 assert_eq!(
11678 client.dispatch.client_order_index(&second_cloid),
11679 Some(client_order_index),
11680 );
11681 assert_eq!(
11682 client
11683 .dispatch
11684 .resolve_client_order_index(client_order_index),
11685 None,
11686 );
11687 assert_eq!(first_fill.client_order_id, Some(first_cloid));
11688 assert_eq!(first_fill.venue_order_id, first_venue_order_id);
11689 assert_eq!(second_fill.client_order_id, Some(second_cloid));
11690 assert_eq!(second_fill.venue_order_id, second_venue_order_id);
11691 assert_ne!(replacement_index, client_order_index);
11692 assert_eq!(
11693 client
11694 .dispatch
11695 .resolve_client_order_index(replacement_index),
11696 Some(replacement),
11697 );
11698 }
11699
11700 #[rstest]
11701 fn reconciliation_restores_triggered_marker() {
11702 let (client, cache, _rx) = create_execution_client();
11703 let instrument_id = register_test_instrument(&client, &cache);
11704 let mut factory = test_order_factory();
11705 let order = factory.stop_limit(
11706 instrument_id,
11707 OrderSide::Sell,
11708 Quantity::from("0.1000"),
11709 Price::from("2290.00"),
11710 Price::from("2300.00"),
11711 None,
11712 Some(TimeInForce::Gtc),
11713 None,
11714 Some(false),
11715 Some(false),
11716 Some(false),
11717 None,
11718 None,
11719 None,
11720 None,
11721 None,
11722 None,
11723 Some(ClientOrderId::from("O-RECON-TRIGGERED")),
11724 );
11725 let cloid = order.client_order_id();
11726 let venue_order_id = VenueOrderId::from("281476929510124");
11727 cache_accepted_order(&cache, order, venue_order_id, None);
11728 let triggered = OrderEventAny::Triggered(OrderTriggered::new(
11729 trader_id(),
11730 strategy_id(),
11731 instrument_id,
11732 cloid,
11733 UUID4::new(),
11734 UnixNanos::from(1),
11735 UnixNanos::from(2),
11736 false,
11737 Some(venue_order_id),
11738 Some(account_id()),
11739 ));
11740 cache.borrow_mut().update_order(&triggered).unwrap();
11741 let (_, client_order_index) = forced_probed_index(cloid);
11742 let raw =
11743 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
11744
11745 restore_reconciled_order(&client.core, &client.dispatch, &raw, false);
11746
11747 assert!(client.dispatch.order_identities.contains_key(&cloid));
11748 assert!(client.dispatch.accepted_was_emitted(&cloid));
11749 assert!(client.dispatch.triggered_was_emitted(&cloid));
11750 }
11751
11752 #[rstest]
11753 fn reconciliation_routes_reused_index_by_exact_venue_order_id() {
11754 let (client, cache, _rx) = create_execution_client();
11755 let instrument_id = register_test_instrument(&client, &cache);
11756 let mut factory = test_order_factory();
11757 let active_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-ACTIVE");
11758 let retired_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-RETIRED");
11759 let active_cloid = active_order.client_order_id();
11760 let retired_cloid = retired_order.client_order_id();
11761 let active_venue_order_id = VenueOrderId::from("281476929510125");
11762 let retired_venue_order_id = VenueOrderId::from("281476929510126");
11763 cache_accepted_order(
11764 &cache,
11765 active_order,
11766 active_venue_order_id,
11767 Some(client_id()),
11768 );
11769 cache_canceled_order(&cache, retired_order, retired_venue_order_id);
11770 let (_, client_order_index) = forced_probed_index(active_cloid);
11771 let active_raw = reconciliation_raw_order(
11772 client_order_index,
11773 active_venue_order_id,
11774 LighterOrderStatus::Open,
11775 );
11776 let retired_raw = reconciliation_raw_order(
11777 client_order_index,
11778 retired_venue_order_id,
11779 LighterOrderStatus::Canceled,
11780 );
11781 let raw_client_id = client_order_index.to_string();
11782
11783 restore_reconciled_order(&client.core, &client.dispatch, &active_raw, false);
11784 restore_reconciled_order(&client.core, &client.dispatch, &retired_raw, true);
11785
11786 assert_eq!(
11787 client
11788 .dispatch
11789 .resolve_live_order_cloid(&raw_client_id, active_venue_order_id,),
11790 Some(active_cloid),
11791 );
11792 assert_eq!(
11793 client
11794 .dispatch
11795 .resolve_live_order_cloid(&raw_client_id, retired_venue_order_id,),
11796 Some(retired_cloid),
11797 );
11798 assert_eq!(
11799 client
11800 .dispatch
11801 .resolve_live_trade_cloid(&raw_client_id, active_venue_order_id),
11802 Some(active_cloid),
11803 );
11804 assert_eq!(
11805 client
11806 .dispatch
11807 .resolve_live_trade_cloid(&raw_client_id, retired_venue_order_id),
11808 Some(retired_cloid),
11809 );
11810 }
11811
11812 #[rstest]
11813 fn reconciliation_does_not_restore_from_numeric_client_index_alone() {
11814 let (client, cache, _rx) = create_execution_client();
11815 let instrument_id = register_test_instrument(&client, &cache);
11816 let mut factory = test_order_factory();
11817 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-NUMERIC-ONLY");
11818 let cloid = order.client_order_id();
11819 let client_order_index = client.dispatch.derive_client_order_index(&cloid);
11820 let venue_order_id = VenueOrderId::from("281476929510130");
11821 cache_order(&cache, order);
11822 let raw =
11823 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
11824
11825 restore_reconciled_order(&client.core, &client.dispatch, &raw, false);
11826 let report =
11827 parse_http_order_to_report(&raw, &client.registry, account_id(), UnixNanos::from(1))
11828 .unwrap();
11829 let translated = client.dispatch.translate_order_cloid(report);
11830
11831 assert_eq!(client.dispatch.client_order_index(&cloid), None);
11832 assert_eq!(
11833 client
11834 .dispatch
11835 .resolve_client_order_index(client_order_index),
11836 None,
11837 );
11838 assert!(!client.dispatch.order_identities.contains_key(&cloid));
11839 assert_eq!(
11840 translated.client_order_id,
11841 Some(ClientOrderId::new(venue_order_id.as_str())),
11842 );
11843 assert_eq!(translated.venue_order_id, venue_order_id);
11844 }
11845
11846 fn forced_probed_index(cloid: ClientOrderId) -> (i64, i64) {
11847 let source = WsDispatchState::new();
11848 let base_index = source.derive_client_order_index(&cloid);
11849 source
11850 .cloid_map
11851 .insert(base_index, ClientOrderId::from("O-FORCED-COLLISION"));
11852 let client_order_index = source.register_cloid(base_index, cloid).unwrap();
11853 (base_index, client_order_index)
11854 }
11855
11856 fn reconciliation_raw_order(
11857 client_order_index: i64,
11858 venue_order_id: VenueOrderId,
11859 status: LighterOrderStatus,
11860 ) -> LighterOrder {
11861 let rig = dispatcher_rig("RECONCILIATION-RAW");
11862 let mut raw = dispatcher_test_order(&rig, status);
11863 raw.order_index = venue_order_id.as_str().parse().unwrap();
11864 raw.order_id = venue_order_id.to_string();
11865 raw.client_order_index = client_order_index;
11866 raw.client_order_id = client_order_index.to_string();
11867 raw
11868 }
11869
11870 fn cache_canceled_order(
11871 cache: &Rc<RefCell<Cache>>,
11872 order: OrderAny,
11873 venue_order_id: VenueOrderId,
11874 ) {
11875 let (instrument_id, client_order_id) =
11876 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
11877 let canceled = OrderEventAny::Canceled(OrderCanceled::new(
11878 trader_id(),
11879 strategy_id(),
11880 instrument_id,
11881 client_order_id,
11882 UUID4::new(),
11883 UnixNanos::from(1),
11884 UnixNanos::from(2),
11885 false,
11886 Some(venue_order_id),
11887 Some(account_id()),
11888 None,
11889 ));
11890 cache.borrow_mut().update_order(&canceled).unwrap();
11891 }
11892
11893 fn reconciliation_fill_report(
11894 instrument_id: InstrumentId,
11895 client_order_index: i64,
11896 venue_order_id: VenueOrderId,
11897 trade_id: &str,
11898 ) -> FillReport {
11899 FillReport::new(
11900 account_id(),
11901 instrument_id,
11902 venue_order_id,
11903 TradeId::new(trade_id),
11904 OrderSide::Buy,
11905 Quantity::from("0.1336"),
11906 Price::from("2352.73"),
11907 Money::from("0.000196 USDC"),
11908 LiquiditySide::Taker,
11909 Some(ClientOrderId::new(client_order_index.to_string())),
11910 None,
11911 UnixNanos::from(1),
11912 UnixNanos::from(2),
11913 Some(UUID4::new()),
11914 )
11915 }
11916
11917 #[rstest]
11918 fn dispatch_lighter_order_seeds_snapshot_after_synthesized_accept() {
11919 let mut rig = dispatcher_rig("9");
11924 register_identity(&rig);
11925
11926 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11927 let mut trade = dispatcher_test_trade(&rig, true);
11928 trade.bid_id = order.order_index;
11929 trade.bid_id_str = Some(order.order_id.clone());
11930 dispatch_lighter_trade(
11931 &trade,
11932 &rig.dispatch,
11933 &rig.emitter,
11934 &rig.registry,
11935 account_id(),
11936 trader_id(),
11937 Some(TEST_ACCOUNT_INDEX_I64),
11938 UnixNanos::from(1),
11939 );
11940 assert_eq!(drain_events(&mut rig.rx).len(), 2);
11941 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
11942 assert!(
11943 rig.dispatch.snapshot_for(&rig.cloid).is_none(),
11944 "synthesized Accept has no snapshot until the Open frame seeds one",
11945 );
11946
11947 dispatch_lighter_order(
11951 &order,
11952 &rig.dispatch,
11953 &rig.emitter,
11954 &rig.registry,
11955 account_id(),
11956 trader_id(),
11957 UnixNanos::from(2),
11958 );
11959 assert!(
11960 rig.dispatch.snapshot_for(&rig.cloid).is_some(),
11961 "Open frame after synthesized accept must seed the snapshot",
11962 );
11963
11964 let mut modified = order;
11966 modified.price = Decimal::from_str("2400.00").unwrap();
11967 dispatch_lighter_order(
11968 &modified,
11969 &rig.dispatch,
11970 &rig.emitter,
11971 &rig.registry,
11972 account_id(),
11973 trader_id(),
11974 UnixNanos::from(3),
11975 );
11976 let events = drain_events(&mut rig.rx);
11977 let updated = events
11978 .iter()
11979 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Updated(_))));
11980 assert!(
11981 updated.is_some(),
11982 "real modify must produce Updated, events={events:?}"
11983 );
11984 }
11985
11986 fn enqueue_create(client: &LighterExecutionClient, order: &OrderAny, nonce: i64) -> i64 {
11987 let client_order_index = client
11988 .dispatch
11989 .derive_client_order_index(&order.client_order_id());
11990 client
11991 .dispatch
11992 .register_cloid(client_order_index, order.client_order_id())
11993 .unwrap();
11994 client.dispatch.register_order_identity(
11995 order.client_order_id(),
11996 OrderIdentity::new(
11997 order.instrument_id(),
11998 order.strategy_id(),
11999 order.order_side(),
12000 order.order_type(),
12001 client_order_index,
12002 ),
12003 );
12004 let now = UnixNanos::from(1_000_000_000);
12005 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12006 batch_id: None,
12007 connection_epoch: 0,
12008 kind: PendingSendTxKind::Create {
12009 order: Box::new(order.clone()),
12010 client_order_index,
12011 },
12012 submitted_at: now,
12013 nonce,
12014 api_key_index: TEST_API_KEY_INDEX,
12015 tx_hash: format!("hash{nonce:02x}"),
12016 });
12017 client.dispatch.nonce_manager.refresh(
12018 TEST_ACCOUNT_INDEX_I64,
12019 TEST_API_KEY_INDEX,
12020 nonce + 1,
12021 );
12022 let _ = client
12023 .dispatch
12024 .nonce_manager
12025 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX);
12026 client_order_index
12027 }
12028
12029 fn enqueue_other(client: &LighterExecutionClient, nonce: i64) {
12030 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12031 batch_id: None,
12032 connection_epoch: 0,
12033 kind: PendingSendTxKind::Other,
12034 submitted_at: UnixNanos::from(1_000_000_000),
12035 nonce,
12036 api_key_index: TEST_API_KEY_INDEX,
12037 tx_hash: format!("hash{nonce:02x}"),
12038 });
12039 }
12040
12041 fn enqueue_cancel(
12042 client: &LighterExecutionClient,
12043 instrument_id: InstrumentId,
12044 client_order_id: ClientOrderId,
12045 venue_order_id: VenueOrderId,
12046 nonce: i64,
12047 ) {
12048 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12049 batch_id: None,
12050 connection_epoch: 0,
12051 kind: PendingSendTxKind::Cancel {
12052 strategy_id: strategy_id(),
12053 instrument_id,
12054 client_order_id,
12055 venue_order_id: Some(venue_order_id),
12056 },
12057 submitted_at: UnixNanos::from(1_000_000_000),
12058 nonce,
12059 api_key_index: TEST_API_KEY_INDEX,
12060 tx_hash: format!("hash{nonce:02x}"),
12061 });
12062 }
12063
12064 fn enqueue_modify(
12065 client: &LighterExecutionClient,
12066 instrument_id: InstrumentId,
12067 client_order_id: ClientOrderId,
12068 venue_order_id: VenueOrderId,
12069 nonce: i64,
12070 ) {
12071 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12072 batch_id: None,
12073 connection_epoch: 0,
12074 kind: PendingSendTxKind::Modify {
12075 strategy_id: strategy_id(),
12076 instrument_id,
12077 client_order_id,
12078 venue_order_id: Some(venue_order_id),
12079 },
12080 submitted_at: UnixNanos::from(1_000_000_000),
12081 nonce,
12082 api_key_index: TEST_API_KEY_INDEX,
12083 tx_hash: format!("hash{nonce:02x}"),
12084 });
12085 }
12086
12087 #[tokio::test]
12088 async fn handle_send_tx_ack_removes_hash_matched_entry() {
12089 let (client, cache, mut rx) = create_execution_client();
12090 let instrument_id = register_test_instrument(&client, &cache);
12091 let mut factory = test_order_factory();
12092 let order_a = test_limit_order(&mut factory, instrument_id, "ACK-A");
12093 let order_b = test_limit_order(&mut factory, instrument_id, "ACK-B");
12094 enqueue_create(&client, &order_a, 10);
12095 enqueue_create(&client, &order_b, 11);
12096
12097 let acked = handle_send_tx_ack(
12099 &client.dispatch,
12100 Some(TEST_ACCOUNT_INDEX_I64),
12101 200,
12102 Some("hash0b"),
12103 );
12104
12105 let acked = acked.expect("create ack");
12106 assert!(matches!(acked.kind, PendingSendTxKind::Create { .. }));
12107 assert!(matches!(
12108 AckedOrderProbe::from_pending(&acked),
12109 Some(AckedOrderProbe::Create {
12110 nonce: 11,
12111 connection_epoch: 0,
12112 ..
12113 }),
12114 ));
12115 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "only B pops");
12116 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
12117 match head.kind {
12118 PendingSendTxKind::Create { order, .. } => {
12119 assert_eq!(order.client_order_id(), order_a.client_order_id());
12120 }
12121 _ => panic!("expected Create kind"),
12122 }
12123 assert!(
12124 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12125 .await
12126 .is_err(),
12127 "ack must not emit an event",
12128 );
12129 }
12130
12131 #[rstest]
12132 fn acknowledged_create_tx_validation_requires_exact_identity() {
12133 let tx = LighterTx {
12134 code: 200,
12135 message: None,
12136 hash: "ABCDEF".to_string(),
12137 tx_type: LighterTxType::CreateOrder as u8,
12138 info: serde_json::json!({"ClientOrderIndex": 42}).to_string(),
12139 event_info: serde_json::json!({"ae": ""}).to_string(),
12140 status: LighterTxStatus::Failed,
12141 account_index: TEST_ACCOUNT_INDEX_I64,
12142 nonce: 10,
12143 api_key_index: TEST_API_KEY_INDEX,
12144 };
12145 assert!(
12146 validate_acked_create_tx(
12147 &tx,
12148 TEST_ACCOUNT_INDEX_I64,
12149 TEST_API_KEY_INDEX,
12150 42,
12151 10,
12152 "0xabcdef",
12153 )
12154 .is_ok(),
12155 );
12156
12157 let mismatches = [
12158 LighterTx {
12159 hash: "different".to_string(),
12160 ..tx.clone()
12161 },
12162 LighterTx {
12163 tx_type: LighterTxType::CancelOrder as u8,
12164 ..tx.clone()
12165 },
12166 LighterTx {
12167 account_index: TEST_ACCOUNT_INDEX_I64 + 1,
12168 ..tx.clone()
12169 },
12170 LighterTx {
12171 nonce: 11,
12172 ..tx.clone()
12173 },
12174 LighterTx {
12175 api_key_index: TEST_API_KEY_INDEX + 1,
12176 ..tx.clone()
12177 },
12178 ];
12179
12180 for mismatch in mismatches {
12181 let error = validate_acked_create_tx(
12182 &mismatch,
12183 TEST_ACCOUNT_INDEX_I64,
12184 TEST_API_KEY_INDEX,
12185 42,
12186 10,
12187 "abcdef",
12188 )
12189 .expect_err("mismatched transaction identity must fail");
12190 assert_eq!(
12191 error.to_string(),
12192 "Lighter transaction lookup did not match acknowledged create identity",
12193 );
12194 }
12195
12196 let wrong_index = LighterTx {
12197 info: serde_json::json!({"ClientOrderIndex": 43}).to_string(),
12198 ..tx
12199 };
12200 let error = validate_acked_create_tx(
12201 &wrong_index,
12202 TEST_ACCOUNT_INDEX_I64,
12203 TEST_API_KEY_INDEX,
12204 42,
12205 10,
12206 "abcdef",
12207 )
12208 .expect_err("mismatched client order index must fail");
12209 assert_eq!(
12210 error.to_string(),
12211 "Lighter transaction lookup returned client_order_index 43 for acknowledged create 42",
12212 );
12213 }
12214
12215 #[tokio::test]
12216 async fn send_tx_responses_only_attribute_within_their_connection_epoch() {
12217 let (client, _cache, mut rx) = create_execution_client();
12218 for (connection_epoch, nonce) in [(4, 20), (5, 21)] {
12219 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12220 batch_id: None,
12221 connection_epoch,
12222 kind: PendingSendTxKind::Other,
12223 submitted_at: UnixNanos::from(1_000_000_000 + nonce as u64),
12224 nonce,
12225 api_key_index: TEST_API_KEY_INDEX,
12226 tx_hash: "same-hash".to_string(),
12227 });
12228 }
12229
12230 let acked =
12231 handle_send_tx_ack_for_connection(&client.dispatch, None, 5, 200, Some("same-hash"))
12232 .expect("replacement epoch response should match");
12233 assert_eq!(acked.connection_epoch, 5);
12234 assert_eq!(acked.nonce, 21);
12235 {
12236 let queue = client.dispatch.pending_sendtx.lock();
12237 assert_eq!(queue.len(), 1);
12238 assert_eq!(queue[0].connection_epoch, 4);
12239 assert_eq!(queue[0].nonce, 20);
12240 }
12241
12242 assert!(!handle_send_tx_rejection_for_connection(
12243 &client.dispatch,
12244 &client.emitter,
12245 None,
12246 4,
12247 UnixNanos::from(1_000_000_100),
12248 SendTxRejectionSource::Ack,
12249 Some(21702),
12250 "invalid price",
12251 Some("same-hash"),
12252 ));
12253 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12254 assert!(
12255 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12256 .await
12257 .is_err(),
12258 "non-order responses must not emit order events",
12259 );
12260 }
12261
12262 #[tokio::test]
12263 async fn handle_send_tx_ack_unmatched_hash_pops_nothing() {
12264 let (client, cache, mut rx) = create_execution_client();
12265 let instrument_id = register_test_instrument(&client, &cache);
12266 let mut factory = test_order_factory();
12267 let order = test_limit_order(&mut factory, instrument_id, "ACK-UNMATCHED");
12268 enqueue_create(&client, &order, 10);
12269
12270 let acked = handle_send_tx_ack(
12271 &client.dispatch,
12272 Some(TEST_ACCOUNT_INDEX_I64),
12273 200,
12274 Some("0xabc"),
12275 );
12276
12277 assert!(acked.is_none());
12278 assert_eq!(
12279 client.dispatch.pending_sendtx_len(),
12280 1,
12281 "an echoed hash with no matching entry must not pop the head",
12282 );
12283 assert!(
12284 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12285 .await
12286 .is_err(),
12287 "unmatched ack must not emit an event",
12288 );
12289 }
12290
12291 #[tokio::test]
12292 async fn handle_send_tx_ack_hashless_with_multiple_pending_pops_nothing() {
12293 let (client, _cache, _rx) = create_execution_client();
12294 enqueue_other(&client, 10);
12295 enqueue_other(&client, 11);
12296
12297 let acked = handle_send_tx_ack(&client.dispatch, Some(TEST_ACCOUNT_INDEX_I64), 200, None);
12298
12299 assert!(acked.is_none());
12300 assert_eq!(
12301 client.dispatch.pending_sendtx_len(),
12302 2,
12303 "a hashless ack is ambiguous when multiple transactions are pending",
12304 );
12305 }
12306
12307 #[tokio::test]
12308 async fn prepared_create_tx_hash_round_trips_through_ack() {
12309 let (client, cache, _rx) = create_execution_client();
12312 let instrument_id = register_test_instrument(&client, &cache);
12313 let mut factory = test_order_factory();
12314 let order = test_limit_order(&mut factory, instrument_id, "ACK-REAL-HASH");
12315
12316 let credential = test_credential();
12317 let plan = client
12318 .prepare_create_order_plan(&order, 0)
12319 .expect("prepare must validate");
12320 let prepared = client
12321 .fanout_dispatch_context(&credential)
12322 .expect("task admission")
12323 .sign_create_order(plan)
12324 .expect("prepare must sign");
12325
12326 assert_eq!(prepared.tx_hash.len(), TX_HASH_BYTES * 2);
12327 assert!(
12328 prepared
12329 .tx_hash
12330 .chars()
12331 .all(|c| matches!(c, '0'..='9' | 'a'..='f')),
12332 "tx_hash must be lowercase hex, was `{}`",
12333 prepared.tx_hash,
12334 );
12335
12336 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12337 batch_id: None,
12338 connection_epoch: 0,
12339 kind: PendingSendTxKind::Create {
12340 order: Box::new(order),
12341 client_order_index: prepared.client_order_index,
12342 },
12343 submitted_at: UnixNanos::from(1_000_000_000),
12344 nonce: prepared.nonce,
12345 api_key_index: prepared.api_key_index,
12346 tx_hash: prepared.tx_hash.clone(),
12347 });
12348
12349 let acked = handle_send_tx_ack(
12350 &client.dispatch,
12351 Some(TEST_ACCOUNT_INDEX_I64),
12352 200,
12353 Some(&prepared.tx_hash),
12354 );
12355
12356 assert!(acked.is_some());
12357 assert_eq!(
12358 client.dispatch.pending_sendtx_len(),
12359 0,
12360 "venue echo of the signed hash must match the enqueued entry",
12361 );
12362 }
12363
12364 #[tokio::test]
12365 async fn handle_send_tx_ack_returns_cancel_for_noop_probe() {
12366 let (client, cache, mut rx) = create_execution_client();
12367 let instrument_id = register_test_instrument(&client, &cache);
12368 let client_order_id = ClientOrderId::from("ACK-CANCEL-NOOP");
12369 let venue_order_id = VenueOrderId::from("123");
12370 enqueue_cancel(&client, instrument_id, client_order_id, venue_order_id, 12);
12371
12372 let acked = handle_send_tx_ack(
12373 &client.dispatch,
12374 Some(TEST_ACCOUNT_INDEX_I64),
12375 200,
12376 Some("hash0c"),
12377 )
12378 .expect("acked cancel pending entry");
12379 let probe = AckedOrderProbe::from_pending(&acked).expect("cancel should schedule probe");
12380
12381 match probe {
12382 AckedOrderProbe::Cancel {
12383 instrument_id: actual_instrument_id,
12384 client_order_id: actual_client_order_id,
12385 venue_order_id: actual_venue_order_id,
12386 } => {
12387 assert_eq!(actual_instrument_id, instrument_id);
12388 assert_eq!(actual_client_order_id, client_order_id);
12389 assert_eq!(actual_venue_order_id, Some(venue_order_id));
12390 }
12391 other => panic!("expected cancel probe, was {other:?}"),
12392 }
12393 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12394 assert!(
12395 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12396 .await
12397 .is_err(),
12398 "ack itself must not emit before the no-op probe runs",
12399 );
12400 }
12401
12402 #[rstest]
12403 #[tokio::test]
12404 async fn acknowledged_order_probe_is_retained_until_session_shutdown() {
12405 let (mut client, cache, _rx) = create_execution_client();
12406 let instrument_id = register_test_instrument(&client, &cache);
12407 let pending = PendingSendTx {
12408 batch_id: None,
12409 connection_epoch: 0,
12410 kind: PendingSendTxKind::Cancel {
12411 strategy_id: strategy_id(),
12412 instrument_id,
12413 client_order_id: ClientOrderId::from("ACK-RETAINED"),
12414 venue_order_id: Some(VenueOrderId::from("123")),
12415 },
12416 submitted_at: UnixNanos::from(1_000_000_000),
12417 nonce: 12,
12418 api_key_index: TEST_API_KEY_INDEX,
12419 tx_hash: "hash0c".to_string(),
12420 };
12421 let context = AckedOrderProbeContext {
12422 http_client: client.http_client.clone(),
12423 registry: Arc::clone(&client.registry),
12424 credential: test_credential(),
12425 dispatch: client.dispatch.clone(),
12426 account_id: client.core.account_id,
12427 clock: client.clock,
12428 emitter: client.emitter.clone(),
12429 connection_epoch: client.ws_client.connection_epoch_atomic(),
12430 cancellation_token: client.cancellation_token.clone(),
12431 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
12432 };
12433
12434 spawn_acked_order_probe(&pending, context);
12435
12436 assert_eq!(client.pending_tasks.len(), 1);
12437
12438 client.begin_session_shutdown();
12439 client
12440 .finish_session_shutdown()
12441 .await
12442 .expect("session shutdown");
12443
12444 assert!(client.pending_tasks.is_empty());
12445 }
12446
12447 #[tokio::test]
12448 async fn handle_send_tx_ack_returns_modify_for_noop_probe() {
12449 let (client, cache, mut rx) = create_execution_client();
12450 let instrument_id = register_test_instrument(&client, &cache);
12451 let client_order_id = ClientOrderId::from("ACK-MODIFY-NOOP");
12452 let venue_order_id = VenueOrderId::from("456");
12453 enqueue_modify(&client, instrument_id, client_order_id, venue_order_id, 13);
12454
12455 let acked = handle_send_tx_ack(
12456 &client.dispatch,
12457 Some(TEST_ACCOUNT_INDEX_I64),
12458 200,
12459 Some("hash0d"),
12460 )
12461 .expect("acked modify pending entry");
12462 let probe = AckedOrderProbe::from_pending(&acked).expect("modify should schedule probe");
12463
12464 match probe {
12465 AckedOrderProbe::Modify {
12466 instrument_id: actual_instrument_id,
12467 client_order_id: actual_client_order_id,
12468 venue_order_id: actual_venue_order_id,
12469 } => {
12470 assert_eq!(actual_instrument_id, instrument_id);
12471 assert_eq!(actual_client_order_id, client_order_id);
12472 assert_eq!(actual_venue_order_id, Some(venue_order_id));
12473 }
12474 other => panic!("expected modify probe, was {other:?}"),
12475 }
12476 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12477 assert!(
12478 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12479 .await
12480 .is_err(),
12481 "ack itself must not emit before the no-op probe runs",
12482 );
12483 }
12484
12485 #[tokio::test]
12486 async fn ack_noop_probe_missing_cancel_leaves_outcome_unresolved() {
12487 let (client, cache, mut rx) = create_execution_client();
12488 let instrument_id = register_test_instrument(&client, &cache);
12489 let client_order_id = ClientOrderId::from("ACK-CANCEL-MISSING");
12490 let venue_order_id = VenueOrderId::from("123");
12491 let probe = AckedOrderProbe::Cancel {
12492 instrument_id,
12493 client_order_id,
12494 venue_order_id: Some(venue_order_id),
12495 };
12496
12497 warn_if_acked_order_missing(&probe, false);
12498 assert!(
12499 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12500 .await
12501 .is_err(),
12502 "one missing lookup must not become CancelRejected",
12503 );
12504 }
12505
12506 #[tokio::test]
12507 async fn ack_noop_probe_missing_modify_leaves_outcome_unresolved() {
12508 let (client, cache, mut rx) = create_execution_client();
12509 let instrument_id = register_test_instrument(&client, &cache);
12510 let client_order_id = ClientOrderId::from("ACK-MODIFY-MISSING");
12511 let venue_order_id = VenueOrderId::from("456");
12512 let probe = AckedOrderProbe::Modify {
12513 instrument_id,
12514 client_order_id,
12515 venue_order_id: Some(venue_order_id),
12516 };
12517
12518 warn_if_acked_order_missing(&probe, false);
12519 assert!(
12520 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12521 .await
12522 .is_err(),
12523 "one missing lookup must not become ModifyRejected",
12524 );
12525 }
12526
12527 #[tokio::test]
12528 async fn ack_noop_probe_found_order_skips_rejection() {
12529 let (client, cache, mut rx) = create_execution_client();
12530 let instrument_id = register_test_instrument(&client, &cache);
12531 let probe = AckedOrderProbe::Cancel {
12532 instrument_id,
12533 client_order_id: ClientOrderId::from("ACK-CANCEL-FOUND"),
12534 venue_order_id: Some(VenueOrderId::from("123")),
12535 };
12536
12537 warn_if_acked_order_missing(&probe, true);
12538 assert!(
12539 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12540 .await
12541 .is_err(),
12542 "found acked order must not emit a no-op rejection",
12543 );
12544 }
12545
12546 #[tokio::test]
12547 async fn send_tx_acks_recover_skip_window_past_window_size() {
12548 let (client, _cache, _rx) = create_execution_client();
12549 let window = i64::from(client.dispatch.nonce_manager.skip_window());
12550
12551 for _ in 0..window {
12552 let nonce = client
12553 .dispatch
12554 .nonce_manager
12555 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12556 .unwrap();
12557 enqueue_other(&client, nonce);
12558 }
12559 assert!(
12560 client
12561 .dispatch
12562 .nonce_manager
12563 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12564 .is_err(),
12565 "window must trip after {window} unacked txs",
12566 );
12567
12568 for i in 0..window {
12571 let acked = handle_send_tx_ack(
12572 &client.dispatch,
12573 Some(TEST_ACCOUNT_INDEX_I64),
12574 200,
12575 Some(&format!("hash{:02x}", TEST_NEXT_NONCE + i)),
12576 );
12577 assert!(matches!(
12578 acked.map(|pending| pending.kind),
12579 Some(PendingSendTxKind::Other),
12580 ));
12581 let nonce = client
12582 .dispatch
12583 .nonce_manager
12584 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12585 .unwrap();
12586 assert_eq!(nonce, TEST_NEXT_NONCE + window + i);
12587 enqueue_other(&client, nonce);
12588 }
12589 }
12590
12591 #[tokio::test]
12592 async fn handle_send_tx_rejection_rolls_back_latest_nonce() {
12593 let (client, cache, mut rx) = create_execution_client();
12594 let instrument_id = register_test_instrument(&client, &cache);
12595 let mut factory = test_order_factory();
12596 let order = test_limit_order(&mut factory, instrument_id, "REJECT-LATEST");
12597
12598 let nonce = client
12599 .dispatch
12600 .nonce_manager
12601 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12602 .unwrap();
12603 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
12604 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12605 batch_id: None,
12606 connection_epoch: 0,
12607 kind: PendingSendTxKind::Create {
12608 order: Box::new(order.clone()),
12609 client_order_index,
12610 },
12611 submitted_at: UnixNanos::from(1_000_000_000),
12612 nonce,
12613 api_key_index: TEST_API_KEY_INDEX,
12614 tx_hash: format!("hash{nonce:02x}"),
12615 });
12616
12617 handle_send_tx_rejection(
12618 &client.dispatch,
12619 &client.emitter,
12620 Some(TEST_ACCOUNT_INDEX_I64),
12621 UnixNanos::from(1_000_000_000),
12622 SendTxRejectionSource::Ack,
12623 Some(21702),
12624 "invalid price",
12625 None,
12626 );
12627
12628 let event = recv_order_event(&mut rx).await;
12629 assert!(matches!(event, OrderEventAny::Rejected(_)));
12630 assert_nonce_reusable(&client.dispatch);
12631 }
12632
12633 #[tokio::test]
12634 async fn handle_send_tx_rejection_hashless_with_multiple_pending_is_unattributed() {
12635 let (client, cache, mut rx) = create_execution_client();
12636 let instrument_id = register_test_instrument(&client, &cache);
12637 let mut factory = test_order_factory();
12638 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-A");
12639 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-B");
12640 enqueue_create(&client, &order_a, 10);
12641 enqueue_create(&client, &order_b, 11);
12642
12643 handle_send_tx_rejection(
12644 &client.dispatch,
12645 &client.emitter,
12646 Some(TEST_ACCOUNT_INDEX_I64),
12647 UnixNanos::from(1_000_000_000),
12648 SendTxRejectionSource::Ack,
12649 Some(21702),
12650 "invalid price",
12651 None,
12652 );
12653
12654 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
12655 assert!(
12656 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12657 .await
12658 .is_err(),
12659 "an ambiguous hashless rejection must not reject either order",
12660 );
12661 }
12662
12663 #[tokio::test]
12664 async fn handle_send_tx_rejection_with_newer_issuance_skips_rollback() {
12665 let (client, cache, mut rx) = create_execution_client();
12666 let instrument_id = register_test_instrument(&client, &cache);
12667 let mut factory = test_order_factory();
12668 let order = test_limit_order(&mut factory, instrument_id, "REJECT-STALE");
12669
12670 let rejected_nonce = client
12671 .dispatch
12672 .nonce_manager
12673 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12674 .unwrap();
12675 let newer_nonce = client
12676 .dispatch
12677 .nonce_manager
12678 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12679 .unwrap();
12680 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
12681 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12682 batch_id: None,
12683 connection_epoch: 0,
12684 kind: PendingSendTxKind::Create {
12685 order: Box::new(order.clone()),
12686 client_order_index,
12687 },
12688 submitted_at: UnixNanos::from(1_000_000_000),
12689 nonce: rejected_nonce,
12690 api_key_index: TEST_API_KEY_INDEX,
12691 tx_hash: format!("hash{rejected_nonce:02x}"),
12692 });
12693
12694 handle_send_tx_rejection(
12695 &client.dispatch,
12696 &client.emitter,
12697 Some(TEST_ACCOUNT_INDEX_I64),
12698 UnixNanos::from(1_000_000_000),
12699 SendTxRejectionSource::Ack,
12700 Some(21702),
12701 "invalid price",
12702 None,
12703 );
12704
12705 let event = recv_order_event(&mut rx).await;
12706 assert!(matches!(event, OrderEventAny::Rejected(_)));
12707 assert_eq!(
12710 client
12711 .dispatch
12712 .nonce_manager
12713 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12714 Some(newer_nonce),
12715 );
12716 assert_eq!(
12717 client
12718 .dispatch
12719 .nonce_manager
12720 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12721 .unwrap(),
12722 newer_nonce + 1,
12723 );
12724 }
12725
12726 async fn spawn_next_nonce_server(nonce: i64) -> String {
12727 let body = serde_json::to_string(&LighterNextNonce {
12728 code: 200,
12729 message: None,
12730 nonce,
12731 })
12732 .unwrap();
12733 let app = Router::new().route(
12734 "/api/v1/nextNonce",
12735 get(move || {
12736 let body = body.clone();
12737 async move { body }
12738 }),
12739 );
12740 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
12741 let addr = listener.local_addr().unwrap();
12742 tokio::spawn(async move {
12743 axum::serve(listener, app).await.unwrap();
12744 });
12745
12746 format!("http://{addr}")
12747 }
12748
12749 async fn spawn_counting_next_nonce_server(nonce: i64) -> (String, Arc<AtomicUsize>) {
12750 let requests = Arc::new(AtomicUsize::new(0));
12751 let requests_for_route = Arc::clone(&requests);
12752 let body = serde_json::to_string(&LighterNextNonce {
12753 code: 200,
12754 message: None,
12755 nonce,
12756 })
12757 .unwrap();
12758 let app = Router::new().route(
12759 "/api/v1/nextNonce",
12760 get(move || {
12761 requests_for_route.fetch_add(1, Ordering::SeqCst);
12762 let body = body.clone();
12763 async move { body }
12764 }),
12765 );
12766 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
12767 let addr = listener.local_addr().unwrap();
12768 tokio::spawn(async move {
12769 axum::serve(listener, app).await.unwrap();
12770 });
12771
12772 (format!("http://{addr}"), requests)
12773 }
12774
12775 #[tokio::test]
12776 async fn hard_nonce_refresh_waits_for_submission_and_replaces_its_baseline() {
12777 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
12778 let mut config = test_config();
12779 config.base_url_http = Some(base_url_http);
12780 let (client, _cache, _rx) = create_execution_client_with_config(config);
12781 let credential = test_credential();
12782 client
12783 .dispatch
12784 .nonce_manager
12785 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
12786
12787 let reserved = client.build_tx_context(&credential).unwrap();
12788 assert_eq!(reserved.context.nonce, 40);
12789 let mut refresh = Box::pin(client.refresh_nonce());
12790 assert!(
12791 tokio::time::timeout(Duration::from_millis(50), refresh.as_mut())
12792 .await
12793 .is_err(),
12794 "hard refresh must wait while a submission retains its nonce",
12795 );
12796 assert_eq!(requests.load(Ordering::SeqCst), 0);
12797 assert_eq!(
12798 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
12799 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
12800 );
12801 assert_eq!(
12802 client
12803 .dispatch
12804 .nonce_manager
12805 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12806 Some(40),
12807 );
12808
12809 drop(reserved);
12810 refresh.await.unwrap();
12811
12812 assert_eq!(requests.load(Ordering::SeqCst), 1);
12813 assert_eq!(
12814 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
12815 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
12816 );
12817 client
12818 .nonce_ready_connection_epoch
12819 .store(client.ws_client.connection_epoch(), Ordering::Release);
12820 let next = client.build_tx_context(&credential).unwrap();
12821 assert_eq!(next.context.nonce, 100);
12822 assert_eq!(
12823 client
12824 .dispatch
12825 .nonce_manager
12826 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12827 Some(100),
12828 );
12829 }
12830
12831 #[tokio::test]
12832 async fn nonce_refresh_retry_reopens_submission_with_replacement_baseline() {
12833 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
12834 let mut config = test_config();
12835 config.base_url_http = Some(base_url_http);
12836 let (client, _cache, _rx) = create_execution_client_with_config(config);
12837 let credential = test_credential();
12838 let connection_epoch = client.ws_client.connection_epoch();
12839 client
12840 .dispatch
12841 .nonce_manager
12842 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
12843 client
12844 .nonce_ready_connection_epoch
12845 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
12846
12847 NonceRefreshRetry {
12848 http_client: client.http_client.clone(),
12849 dispatch: client.dispatch.clone(),
12850 credential: Some(credential.clone()),
12851 submission_gate: Arc::clone(&client.nonce_submission_gate),
12852 ready_connection_epoch: Arc::clone(&client.nonce_ready_connection_epoch),
12853 ws_client: client.ws_client.clone(),
12854 cancellation_token: client.cancellation_token.clone(),
12855 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
12856 }
12857 .spawn(connection_epoch);
12858
12859 assert_eq!(client.pending_tasks.len(), 1);
12860
12861 let err = client
12862 .build_tx_context(&credential)
12863 .expect_err("submission must stay closed until retry succeeds");
12864 assert_eq!(
12865 err.to_string(),
12866 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}"),
12867 );
12868 wait_until_async(
12869 || async {
12870 client.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
12871 },
12872 Duration::from_secs(3),
12873 )
12874 .await;
12875
12876 let reserved = client.build_tx_context(&credential).unwrap();
12877 assert_eq!(requests.load(Ordering::SeqCst), 1);
12878 assert_eq!(reserved.context.nonce, 100);
12879 assert_eq!(
12880 client
12881 .dispatch
12882 .nonce_manager
12883 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12884 Some(99),
12885 );
12886 assert_eq!(
12887 client
12888 .dispatch
12889 .nonce_manager
12890 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12891 Some(100),
12892 );
12893 assert!(client.pending_tasks.all_finished());
12894 }
12895
12896 async fn spawn_integrator_approval_rejection_server() -> String {
12897 let app = Router::new()
12898 .route(
12899 "/api/v1/getMakerOnlyApiKeys",
12900 get(|| async { r#"{"code":200,"api_key_indexes":[]}"# }),
12901 )
12902 .route(
12903 "/api/v1/sendTx",
12904 post(|| async { r#"{"code":21702,"message":"invalid approval"}"# }),
12905 );
12906 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
12907 let addr = listener.local_addr().unwrap();
12908 tokio::spawn(async move {
12909 axum::serve(listener, app).await.unwrap();
12910 });
12911
12912 format!("http://{addr}")
12913 }
12914
12915 #[tokio::test]
12916 async fn integrator_approval_api_rejection_releases_reservation_and_nonce() {
12917 let mut config = test_config();
12918 config.environment = LighterEnvironment::Mainnet;
12919 config.base_url_http = Some(spawn_integrator_approval_rejection_server().await);
12920 let (mut client, _cache, _rx) = create_execution_client_with_config(config);
12921 client.account_tier = Some(LighterAccountTier::Premium);
12922
12923 let err = client.submit_integrator_auto_approval().await.unwrap_err();
12924
12925 assert!(format!("{err:#}").contains("invalid approval"));
12926 assert_nonce_reusable(&client.dispatch);
12927 }
12928
12929 #[tokio::test]
12930 async fn skip_window_exhaustion_resyncs_baseline_from_venue() {
12931 let venue_next_nonce = 100;
12932 let mut config = test_config();
12933 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
12934 let (client, _cache, _rx) = create_execution_client_with_config(config);
12935 let credential = test_credential();
12936
12937 let window = i64::from(client.dispatch.nonce_manager.skip_window());
12938 for _ in 0..window {
12939 client
12940 .dispatch
12941 .nonce_manager
12942 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12943 .unwrap();
12944 }
12945
12946 let err = client
12947 .build_tx_context(&credential)
12948 .expect_err("window must trip");
12949 assert!(
12950 err.to_string().contains("skip-window exhausted"),
12951 "unexpected error, was {err}",
12952 );
12953
12954 wait_for_spawned_tasks(&client).await;
12955
12956 assert_eq!(
12957 client
12958 .dispatch
12959 .nonce_manager
12960 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12961 Some(venue_next_nonce - 1),
12962 "venue resync must advance the baseline",
12963 );
12964 assert!(
12965 !client.nonce_recovery_inflight.load(Ordering::Acquire),
12966 "recovery latch must release for the next exhaustion",
12967 );
12968 let context = client.build_tx_context(&credential).unwrap().context;
12969 assert_eq!(
12970 context.nonce, venue_next_nonce,
12971 "allocation must resume at the venue nonce",
12972 );
12973 }
12974
12975 #[tokio::test]
12976 async fn skip_window_recovery_fetch_failure_releases_latch() {
12977 let app = Router::new();
12979 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
12980 let addr = listener.local_addr().unwrap();
12981 tokio::spawn(async move {
12982 axum::serve(listener, app).await.unwrap();
12983 });
12984 let mut config = test_config();
12985 config.base_url_http = Some(format!("http://{addr}"));
12986 let (client, _cache, _rx) = create_execution_client_with_config(config);
12987 let credential = test_credential();
12988
12989 let window = i64::from(client.dispatch.nonce_manager.skip_window());
12990 for _ in 0..window {
12991 client
12992 .dispatch
12993 .nonce_manager
12994 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12995 .unwrap();
12996 }
12997 client
12998 .build_tx_context(&credential)
12999 .expect_err("window must trip");
13000
13001 wait_for_spawned_tasks(&client).await;
13002
13003 assert_eq!(
13004 client
13005 .dispatch
13006 .nonce_manager
13007 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13008 Some(TEST_NEXT_NONCE - 1),
13009 "failed venue fetch must leave the baseline unchanged",
13010 );
13011 assert!(
13012 !client.nonce_recovery_inflight.load(Ordering::Acquire),
13013 "failed venue fetch must release the recovery latch",
13014 );
13015 }
13016
13017 #[tokio::test]
13018 async fn skip_window_recovery_dedupes_concurrent_fetches() {
13019 let hits = Arc::new(AtomicUsize::new(0));
13020 let server_hits = Arc::clone(&hits);
13021 let body = serde_json::to_string(&LighterNextNonce {
13022 code: 200,
13023 message: None,
13024 nonce: 100,
13025 })
13026 .unwrap();
13027 let app = Router::new().route(
13028 "/api/v1/nextNonce",
13029 get(move || {
13030 server_hits.fetch_add(1, Ordering::AcqRel);
13031 let body = body.clone();
13032 async move { body }
13033 }),
13034 );
13035 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
13036 let addr = listener.local_addr().unwrap();
13037 tokio::spawn(async move {
13038 axum::serve(listener, app).await.unwrap();
13039 });
13040
13041 let mut config = test_config();
13042 config.base_url_http = Some(format!("http://{addr}"));
13043 let (client, _cache, _rx) = create_execution_client_with_config(config);
13044 let credential = test_credential();
13045
13046 let window = i64::from(client.dispatch.nonce_manager.skip_window());
13047 for _ in 0..window {
13048 client
13049 .dispatch
13050 .nonce_manager
13051 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13052 .unwrap();
13053 }
13054
13055 client
13058 .build_tx_context(&credential)
13059 .expect_err("window must trip");
13060 client
13061 .build_tx_context(&credential)
13062 .expect_err("window must still be exhausted");
13063
13064 wait_for_spawned_tasks(&client).await;
13065
13066 assert_eq!(
13067 hits.load(Ordering::Acquire),
13068 1,
13069 "burst exhaustion must trigger a single venue fetch",
13070 );
13071 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
13072 }
13073
13074 #[tokio::test]
13075 async fn refresh_nonce_releases_recovery_latch() {
13076 let venue_next_nonce = 77;
13077 let mut config = test_config();
13078 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
13079 let (client, _cache, _rx) = create_execution_client_with_config(config);
13080
13081 client
13083 .nonce_recovery_inflight
13084 .store(true, Ordering::Release);
13085
13086 client.refresh_nonce().await.unwrap();
13087
13088 assert!(
13089 !client.nonce_recovery_inflight.load(Ordering::Acquire),
13090 "connect-time refresh must release a stuck recovery latch",
13091 );
13092 assert_eq!(
13093 client
13094 .dispatch
13095 .nonce_manager
13096 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13097 .unwrap(),
13098 venue_next_nonce,
13099 "refresh must hard-reset allocation to the venue nonce",
13100 );
13101 }
13102
13103 #[tokio::test]
13104 async fn handle_send_tx_rejection_ack_create_emits_order_rejected() {
13105 let (client, cache, mut rx) = create_execution_client();
13106 let instrument_id = register_test_instrument(&client, &cache);
13107 let mut factory = test_order_factory();
13108 let order = test_limit_order(&mut factory, instrument_id, "REJECT-CREATE");
13109 let client_order_index = enqueue_create(&client, &order, 42);
13110
13111 handle_send_tx_rejection(
13112 &client.dispatch,
13113 &client.emitter,
13114 Some(TEST_ACCOUNT_INDEX_I64),
13115 UnixNanos::from(1_000_000_000),
13116 SendTxRejectionSource::Ack,
13117 Some(21702),
13118 "invalid price",
13119 None,
13120 );
13121
13122 let event = recv_order_event(&mut rx).await;
13123 match event {
13124 OrderEventAny::Rejected(e) => {
13125 assert_eq!(e.client_order_id, order.client_order_id());
13126 assert_eq!(e.reason, "LIGHTER_21702: invalid price");
13127 assert!(!e.due_post_only);
13128 }
13129 other => panic!("expected Rejected, was {other:?}"),
13130 }
13131 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13132 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
13133 }
13134
13135 #[tokio::test]
13136 async fn handle_send_tx_rejection_ack_create_sets_due_post_only() {
13137 let (client, cache, mut rx) = create_execution_client();
13138 let instrument_id = register_test_instrument(&client, &cache);
13139 let mut factory = test_order_factory();
13140 let order = test_limit_order(&mut factory, instrument_id, "REJECT-POST-ONLY");
13141 enqueue_create(&client, &order, 43);
13142
13143 handle_send_tx_rejection(
13144 &client.dispatch,
13145 &client.emitter,
13146 Some(TEST_ACCOUNT_INDEX_I64),
13147 UnixNanos::from(1_000_000_000),
13148 SendTxRejectionSource::Ack,
13149 Some(21700),
13150 "post-only order would execute",
13151 None,
13152 );
13153
13154 let event = recv_order_event(&mut rx).await;
13155 match event {
13156 OrderEventAny::Rejected(e) => {
13157 assert_eq!(e.client_order_id, order.client_order_id());
13158 assert!(e.due_post_only);
13159 }
13160 other => panic!("expected Rejected, was {other:?}"),
13161 }
13162 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13163 }
13164
13165 #[tokio::test]
13166 async fn handle_send_tx_rejection_cancel_emits_cancel_rejected() {
13167 let (client, cache, mut rx) = create_execution_client();
13168 let instrument_id = register_test_instrument(&client, &cache);
13169 let client_order_id = ClientOrderId::from("REJECT-CANCEL");
13170 let venue_order_id = VenueOrderId::from("123");
13171 let nonce = client
13172 .dispatch
13173 .nonce_manager
13174 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13175 .unwrap();
13176
13177 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
13178 batch_id: None,
13179 connection_epoch: 0,
13180 kind: PendingSendTxKind::Cancel {
13181 strategy_id: strategy_id(),
13182 instrument_id,
13183 client_order_id,
13184 venue_order_id: Some(venue_order_id),
13185 },
13186 submitted_at: UnixNanos::from(1_000_000_000),
13187 nonce,
13188 api_key_index: TEST_API_KEY_INDEX,
13189 tx_hash: format!("hash{nonce:02x}"),
13190 });
13191
13192 handle_send_tx_rejection(
13193 &client.dispatch,
13194 &client.emitter,
13195 Some(TEST_ACCOUNT_INDEX_I64),
13196 UnixNanos::from(1_000_000_000),
13197 SendTxRejectionSource::Ack,
13198 Some(21727),
13199 "order is not cancelable",
13200 None,
13201 );
13202
13203 let event = recv_order_event(&mut rx).await;
13204 match event {
13205 OrderEventAny::CancelRejected(e) => {
13206 assert_eq!(e.client_order_id, client_order_id);
13207 assert_eq!(e.instrument_id, instrument_id);
13208 assert_eq!(e.venue_order_id, Some(venue_order_id));
13209 assert_eq!(e.reason, "LIGHTER_21727: order is not cancelable",);
13210 }
13211 other => panic!("expected CancelRejected, was {other:?}"),
13212 }
13213 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13214 assert_nonce_reusable(&client.dispatch);
13215 }
13216
13217 #[tokio::test]
13218 async fn handle_send_tx_rejection_modify_emits_modify_rejected() {
13219 let (client, cache, mut rx) = create_execution_client();
13220 let instrument_id = register_test_instrument(&client, &cache);
13221 let client_order_id = ClientOrderId::from("REJECT-MODIFY");
13222 let venue_order_id = VenueOrderId::from("456");
13223 let nonce = client
13224 .dispatch
13225 .nonce_manager
13226 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13227 .unwrap();
13228
13229 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
13230 batch_id: None,
13231 connection_epoch: 0,
13232 kind: PendingSendTxKind::Modify {
13233 strategy_id: strategy_id(),
13234 instrument_id,
13235 client_order_id,
13236 venue_order_id: Some(venue_order_id),
13237 },
13238 submitted_at: UnixNanos::from(1_000_000_000),
13239 nonce,
13240 api_key_index: TEST_API_KEY_INDEX,
13241 tx_hash: format!("hash{nonce:02x}"),
13242 });
13243
13244 handle_send_tx_rejection(
13245 &client.dispatch,
13246 &client.emitter,
13247 Some(TEST_ACCOUNT_INDEX_I64),
13248 UnixNanos::from(1_000_000_000),
13249 SendTxRejectionSource::Ack,
13250 Some(21702),
13251 "modify rejected by venue",
13252 None,
13253 );
13254
13255 let event = recv_order_event(&mut rx).await;
13256 match event {
13257 OrderEventAny::ModifyRejected(e) => {
13258 assert_eq!(e.client_order_id, client_order_id);
13259 assert_eq!(e.instrument_id, instrument_id);
13260 assert_eq!(e.venue_order_id, Some(venue_order_id));
13261 assert_eq!(e.reason, "LIGHTER_21702: modify rejected by venue",);
13262 }
13263 other => panic!("expected ModifyRejected, was {other:?}"),
13264 }
13265 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13266 assert_nonce_reusable(&client.dispatch);
13267 }
13268
13269 #[tokio::test]
13270 async fn handle_send_tx_rejection_hash_match_attributes_past_the_head() {
13271 let (client, cache, mut rx) = create_execution_client();
13272 let instrument_id = register_test_instrument(&client, &cache);
13273 let mut factory = test_order_factory();
13274 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-A");
13275 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-B");
13276 enqueue_create(&client, &order_a, 10);
13277 enqueue_create(&client, &order_b, 11);
13278
13279 handle_send_tx_rejection(
13281 &client.dispatch,
13282 &client.emitter,
13283 Some(TEST_ACCOUNT_INDEX_I64),
13284 UnixNanos::from(1_000_000_000),
13285 SendTxRejectionSource::Ack,
13286 Some(21702),
13287 "invalid price",
13288 Some("hash0b"),
13289 );
13290
13291 let event = recv_order_event(&mut rx).await;
13292 match event {
13293 OrderEventAny::Rejected(e) => {
13294 assert_eq!(e.client_order_id, order_b.client_order_id());
13295 }
13296 other => panic!("expected Rejected, was {other:?}"),
13297 }
13298 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "A must survive");
13299 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
13300 match head.kind {
13301 PendingSendTxKind::Create { order, .. } => {
13302 assert_eq!(order.client_order_id(), order_a.client_order_id());
13303 }
13304 _ => panic!("expected Create kind"),
13305 }
13306 }
13307
13308 #[tokio::test]
13309 async fn handle_send_tx_rejection_unmatched_hash_pops_nothing() {
13310 let (client, cache, mut rx) = create_execution_client();
13311 let instrument_id = register_test_instrument(&client, &cache);
13312 let mut factory = test_order_factory();
13313 let order = test_limit_order(&mut factory, instrument_id, "REJECT-UNMATCHED");
13314 enqueue_create(&client, &order, 10);
13315
13316 handle_send_tx_rejection(
13317 &client.dispatch,
13318 &client.emitter,
13319 Some(TEST_ACCOUNT_INDEX_I64),
13320 UnixNanos::from(1_000_000_000),
13321 SendTxRejectionSource::Ack,
13322 Some(21702),
13323 "invalid price",
13324 Some("0xbeef"),
13325 );
13326
13327 assert_eq!(
13328 client.dispatch.pending_sendtx_len(),
13329 1,
13330 "an echoed hash with no matching entry must not pop the head",
13331 );
13332 assert!(
13333 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13334 .await
13335 .is_err(),
13336 "unmatched rejection must not emit an event",
13337 );
13338 }
13339
13340 #[tokio::test]
13341 async fn handle_send_tx_rejection_bare_error_within_window_attributes() {
13342 let (client, cache, mut rx) = create_execution_client();
13343 let instrument_id = register_test_instrument(&client, &cache);
13344 let mut factory = test_order_factory();
13345 let order = test_limit_order(&mut factory, instrument_id, "BARE-IN");
13346 enqueue_create(&client, &order, 50);
13347
13348 let within_window = UnixNanos::from(1_000_000_000 + 500 * 1_000_000);
13349 handle_send_tx_rejection(
13350 &client.dispatch,
13351 &client.emitter,
13352 Some(TEST_ACCOUNT_INDEX_I64),
13353 within_window,
13354 SendTxRejectionSource::BareError,
13355 Some(21149),
13356 "integrator is not approved",
13357 None,
13358 );
13359
13360 let event = recv_order_event(&mut rx).await;
13361 assert!(matches!(event, OrderEventAny::Rejected(_)));
13362 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13363 }
13364
13365 #[tokio::test]
13366 async fn handle_send_tx_rejection_bare_error_outside_window_skips() {
13367 let (client, cache, mut rx) = create_execution_client();
13368 let instrument_id = register_test_instrument(&client, &cache);
13369 let mut factory = test_order_factory();
13370 let order = test_limit_order(&mut factory, instrument_id, "BARE-OUT");
13371 enqueue_create(&client, &order, 60);
13372
13373 let outside_window = UnixNanos::from(1_000_000_000 + 2_000 * 1_000_000);
13374 handle_send_tx_rejection(
13375 &client.dispatch,
13376 &client.emitter,
13377 Some(TEST_ACCOUNT_INDEX_I64),
13378 outside_window,
13379 SendTxRejectionSource::BareError,
13380 Some(99),
13381 "late error",
13382 None,
13383 );
13384
13385 assert_eq!(
13386 client.dispatch.pending_sendtx_len(),
13387 1,
13388 "head must remain queued past the 1s attribution window",
13389 );
13390 assert!(
13391 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13392 .await
13393 .is_err(),
13394 "no event must be emitted outside the window",
13395 );
13396 }
13397
13398 #[tokio::test]
13399 async fn handle_send_tx_rejection_bare_error_with_multiple_pending_skips() {
13400 let (client, cache, mut rx) = create_execution_client();
13401 let instrument_id = register_test_instrument(&client, &cache);
13402 let mut factory = test_order_factory();
13403 let order_a = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-A");
13404 let order_b = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-B");
13405 enqueue_create(&client, &order_a, 70);
13406 enqueue_create(&client, &order_b, 71);
13407
13408 handle_send_tx_rejection(
13409 &client.dispatch,
13410 &client.emitter,
13411 Some(TEST_ACCOUNT_INDEX_I64),
13412 UnixNanos::from(1_000_000_000 + 500 * 1_000_000),
13413 SendTxRejectionSource::BareError,
13414 Some(21149),
13415 "integrator is not approved",
13416 None,
13417 );
13418
13419 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
13420 assert!(
13421 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13422 .await
13423 .is_err(),
13424 "a bare error cannot be attributed with multiple transactions pending",
13425 );
13426 }
13427
13428 #[tokio::test]
13429 async fn handle_send_tx_rejection_other_kind_rolls_back_latest_nonce() {
13430 let (client, _cache, _rx) = create_execution_client();
13431 let nonce = client
13432 .dispatch
13433 .nonce_manager
13434 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13435 .unwrap();
13436 enqueue_other(&client, nonce);
13437
13438 let needs_resync = handle_send_tx_rejection(
13439 &client.dispatch,
13440 &client.emitter,
13441 Some(TEST_ACCOUNT_INDEX_I64),
13442 UnixNanos::from(1_000_000_000),
13443 SendTxRejectionSource::Ack,
13444 Some(23000),
13445 "Too Many Requests",
13446 None,
13447 );
13448
13449 assert!(!needs_resync, "rate-limit rejection must not force resync");
13450 assert_nonce_reusable(&client.dispatch);
13451 }
13452
13453 #[tokio::test]
13454 async fn handle_send_tx_rejection_other_kind_skips_rollback_with_newer_issuance() {
13455 let (client, _cache, _rx) = create_execution_client();
13456 let rejected_nonce = client
13457 .dispatch
13458 .nonce_manager
13459 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13460 .unwrap();
13461 let newer_nonce = client
13462 .dispatch
13463 .nonce_manager
13464 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
13465 .unwrap();
13466 enqueue_other(&client, rejected_nonce);
13467
13468 handle_send_tx_rejection(
13469 &client.dispatch,
13470 &client.emitter,
13471 Some(TEST_ACCOUNT_INDEX_I64),
13472 UnixNanos::from(1_000_000_000),
13473 SendTxRejectionSource::Ack,
13474 Some(23000),
13475 "Too Many Requests",
13476 None,
13477 );
13478
13479 assert_eq!(
13480 client
13481 .dispatch
13482 .nonce_manager
13483 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13484 Some(newer_nonce),
13485 "non-latest rejection must leave last_issued alone",
13486 );
13487 }
13488
13489 #[tokio::test]
13490 async fn handle_send_tx_rejection_invalid_nonce_signals_resync() {
13491 let (client, _cache, _rx) = create_execution_client();
13492
13493 enqueue_other(&client, 70);
13494 let attributed = handle_send_tx_rejection(
13495 &client.dispatch,
13496 &client.emitter,
13497 Some(TEST_ACCOUNT_INDEX_I64),
13498 UnixNanos::from(1_000_000_000),
13499 SendTxRejectionSource::Ack,
13500 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
13501 "invalid nonce",
13502 None,
13503 );
13504 assert!(attributed, "attributed invalid nonce must signal resync");
13505
13506 let unattributed = handle_send_tx_rejection(
13507 &client.dispatch,
13508 &client.emitter,
13509 Some(TEST_ACCOUNT_INDEX_I64),
13510 UnixNanos::from(1_000_000_000),
13511 SendTxRejectionSource::Ack,
13512 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
13513 "invalid nonce",
13514 None,
13515 );
13516 assert!(
13517 unattributed,
13518 "unattributed invalid nonce must still signal resync",
13519 );
13520
13521 enqueue_other(&client, 71);
13522 let other_code = handle_send_tx_rejection(
13523 &client.dispatch,
13524 &client.emitter,
13525 Some(TEST_ACCOUNT_INDEX_I64),
13526 UnixNanos::from(1_000_000_000),
13527 SendTxRejectionSource::Ack,
13528 Some(21702),
13529 "invalid price",
13530 None,
13531 );
13532 assert!(!other_code, "other rejection codes must not force resync");
13533 }
13534
13535 #[tokio::test]
13536 async fn handle_send_tx_rejection_other_kind_logs_and_skips_emit() {
13537 let (client, _cache, mut rx) = create_execution_client();
13538 enqueue_other(&client, 70);
13539
13540 handle_send_tx_rejection(
13541 &client.dispatch,
13542 &client.emitter,
13543 Some(TEST_ACCOUNT_INDEX_I64),
13544 UnixNanos::from(1_000_000_000),
13545 SendTxRejectionSource::Ack,
13546 Some(21727),
13547 "invalid client order index",
13548 None,
13549 );
13550
13551 assert_eq!(client.dispatch.pending_sendtx_len(), 0, "Other head pops");
13552 assert!(
13553 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13554 .await
13555 .is_err(),
13556 "Other-kind rejection must not emit OrderRejected",
13557 );
13558 }
13559
13560 #[tokio::test]
13561 async fn handle_send_tx_rejection_empty_queue_logs_warn() {
13562 let (client, _cache, mut rx) = create_execution_client();
13563
13564 handle_send_tx_rejection(
13565 &client.dispatch,
13566 &client.emitter,
13567 Some(TEST_ACCOUNT_INDEX_I64),
13568 UnixNanos::from(1_000_000_000),
13569 SendTxRejectionSource::Ack,
13570 Some(1),
13571 "no pending",
13572 None,
13573 );
13574
13575 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13576 assert!(
13577 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13578 .await
13579 .is_err(),
13580 );
13581 }
13582
13583 #[rstest]
13584 #[case::standard_no_override(LighterAccountTier::Standard, None, 60, None)]
13585 #[case::standard_zero_is_default(LighterAccountTier::Standard, Some(0), 60, None)]
13586 #[case::standard_override_above_tier(
13587 LighterAccountTier::Standard,
13588 Some(24_000),
13589 24_000,
13590 Some(TierCrossCheck::AboveTier { documented: 60 })
13591 )]
13592 #[case::premium_raise_hint(
13593 LighterAccountTier::Premium,
13594 None,
13595 60,
13596 Some(TierCrossCheck::RaiseHint { documented: 24_000 })
13597 )]
13598 #[case::premium_configured_no_advisory(LighterAccountTier::Premium, Some(24_000), 24_000, None)]
13599 #[case::unknown_no_advisory(LighterAccountTier::Unknown(7), None, 60, None)]
13600 fn test_tier_quota_report(
13601 #[case] tier: LighterAccountTier,
13602 #[case] configured: Option<u32>,
13603 #[case] expected_active: u32,
13604 #[case] expected_cross_check: Option<TierCrossCheck>,
13605 ) {
13606 assert_eq!(
13607 tier_quota_report(tier, configured, 60),
13608 (expected_active, expected_cross_check),
13609 );
13610 }
13611}