1use std::{
19 collections::VecDeque,
20 fmt::Debug,
21 future::Future,
22 pin::Pin,
23 sync::{
24 Arc,
25 atomic::{AtomicBool, Ordering},
26 },
27 time::Duration,
28};
29
30use ahash::AHashMap;
31use futures_util::{StreamExt, stream::FuturesUnordered};
32use nautilus_core::{
33 AtomicTime,
34 nanos::UnixNanos,
35 string::secret::{REDACTED, SecretString},
36 time::get_atomic_clock_realtime,
37};
38#[cfg(test)]
39use nautilus_live::book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS;
40use nautilus_live::book::{
41 BookSequenceOutcome,
42 recovery::{BookRecoveryOutcome, BookRecoveryState},
43 snapshot::{PendingSnapshot, SnapshotGate},
44};
45use nautilus_model::{identifiers::AccountId, instruments::InstrumentAny, types::Currency};
46use nautilus_network::{
47 RECONNECTED,
48 error::SendError,
49 retry::{RetryManager, create_websocket_retry_manager},
50 websocket::{SubscriptionState, WebSocketClient},
51};
52use tokio_tungstenite::tungstenite::Message;
53use tokio_util::sync::CancellationToken;
54use ustr::Ustr;
55use zeroize::Zeroize;
56
57use super::{
58 account_state::LighterAccountStateReconciler,
59 error::LighterWsError,
60 messages::{
61 AccountStream, CANCEL_BATCH_ID_PREFIX, ExecutionReport, LighterAsset, LighterPosition,
62 LighterUserStats, LighterWsCandle, LighterWsChannel, LighterWsChannelKind, LighterWsFrame,
63 LighterWsOrderBook, LighterWsRequest, NautilusWsMessage, SendTxRejectionSource,
64 },
65 parse::{
66 parse_ws_bar, parse_ws_funding_rate_update, parse_ws_index_price_update,
67 parse_ws_mark_price_update, parse_ws_position_status_report, parse_ws_quote_tick,
68 parse_ws_spot_index_price_update, parse_ws_trade_tick,
69 },
70};
71use crate::{
72 book::{
73 recovery::{self, BookWorkResult, BookWrite},
74 sync::BookSyncTracker,
75 },
76 common::{
77 consts::{
78 LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED, LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED,
79 LIGHTER_ERROR_CODE_TX_RANGE, LIGHTER_ERROR_CODE_WS_RATE_LIMITED,
80 LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED, LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL,
81 SUBSCRIBE_INFLIGHT_MAX, SUBSCRIBE_RETRY_BASE_BACKOFF, SUBSCRIBE_RETRY_MAX,
82 },
83 enums::LighterCandleResolution,
84 rate_limit::LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY,
85 },
86 http::models::{LighterOrder, LighterTrade},
87};
88
89const CTRL_TYPE_CONNECTED: &str = "connected";
93const CTRL_TYPE_SUBSCRIBED: &str = "subscribed";
94const CTRL_TYPE_UNSUBSCRIBED: &str = "unsubscribed";
95const CTRL_TYPE_PING: &str = "ping";
96const CTRL_TYPE_PONG: &str = "pong";
97const CTRL_TYPE_ERROR: &str = "error";
98const CTRL_TYPE_SEND_TX: &str = "jsonapi/sendtx";
99
100#[derive(serde::Deserialize)]
101struct LighterWsFrameHeader<'a> {
102 #[serde(rename = "type", borrow)]
103 kind: &'a str,
104 #[serde(borrow)]
105 channel: Option<&'a str>,
106}
107
108#[expect(
110 clippy::large_enum_variant,
111 reason = "commands are ephemeral and immediately consumed"
112)]
113pub enum HandlerCommand {
114 SetClient(WebSocketClient),
117 Disconnect,
119 Subscribe {
122 channel: LighterWsChannel,
123 auth: Option<SecretString>,
124 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
125 },
126 Unsubscribe { channel: LighterWsChannel },
128 RecoverBook {
130 market_index: i64,
131 cancel: CancellationToken,
132 gate: SnapshotGate,
133 completion: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
134 },
135 InitializeInstruments(Vec<(i64, InstrumentAny)>),
137 UpdateInstrument {
139 market_index: i64,
140 instrument: InstrumentAny,
141 },
142 SetBookDeltasSub { market_index: i64, subscribed: bool },
145 SetDepthSub { market_index: i64, subscribed: bool },
148 SetExecutionContext {
153 account_id: AccountId,
154 account_index: i64,
155 },
156 SendTx {
162 tx_type: u8,
163 tx_info: Box<serde_json::value::RawValue>,
164 connection_epoch: u64,
165 response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
166 },
167 SendTxBatch {
168 data: super::messages::LighterWsSendTxBatch,
169 connection_epoch: u64,
170 response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
171 },
172}
173
174impl Debug for HandlerCommand {
175 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
180 match self {
181 Self::SetClient(_) => f.write_str("SetClient(<WebSocketClient>)"),
182 Self::Disconnect => f.write_str("Disconnect"),
183 Self::Subscribe { channel, auth, .. } => f
184 .debug_struct(stringify!(Subscribe))
185 .field("channel", channel)
186 .field("auth", auth)
187 .finish(),
188 Self::Unsubscribe { channel } => f
189 .debug_struct(stringify!(Unsubscribe))
190 .field("channel", channel)
191 .finish(),
192 Self::RecoverBook { market_index, .. } => f
193 .debug_struct(stringify!(RecoverBook))
194 .field("market_index", market_index)
195 .finish(),
196 Self::InitializeInstruments(instruments) => f
197 .debug_tuple(stringify!(InitializeInstruments))
198 .field(&instruments.len())
199 .finish(),
200 Self::UpdateInstrument { market_index, .. } => f
201 .debug_struct(stringify!(UpdateInstrument))
202 .field("market_index", market_index)
203 .finish(),
204 Self::SetBookDeltasSub {
205 market_index,
206 subscribed,
207 } => f
208 .debug_struct(stringify!(SetBookDeltasSub))
209 .field("market_index", market_index)
210 .field("subscribed", subscribed)
211 .finish(),
212 Self::SetDepthSub {
213 market_index,
214 subscribed,
215 } => f
216 .debug_struct(stringify!(SetDepthSub))
217 .field("market_index", market_index)
218 .field("subscribed", subscribed)
219 .finish(),
220 Self::SetExecutionContext {
221 account_id,
222 account_index,
223 } => f
224 .debug_struct(stringify!(SetExecutionContext))
225 .field("account_id", account_id)
226 .field("account_index", account_index)
227 .finish(),
228 Self::SendTxBatch { .. } => f.write_str("SendTxBatch(<redacted>)"),
229 Self::SendTx { tx_type, .. } => f
230 .debug_struct(stringify!(SendTx))
231 .field("tx_type", tx_type)
232 .field("tx_info", &REDACTED)
233 .finish(),
234 }
235 }
236}
237
238pub(super) struct FeedHandler {
245 clock: &'static AtomicTime,
246 signal: Arc<AtomicBool>,
247 inner: Option<Arc<WebSocketClient>>,
248 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
249 cmd_tx: Option<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>,
250 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
251 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
252 subscriptions: SubscriptionState,
253 retry_manager: RetryManager<LighterWsError>,
254 pending_messages: VecDeque<NautilusWsMessage>,
255 pending_subs: VecDeque<(Ustr, u64)>,
256 inflight_subs: AHashMap<Ustr, u64>,
257 subscription_attempts: AHashMap<Ustr, SubscriptionAttempt>,
258 subscription_retries: FuturesUnordered<SubscriptionRetry>,
259 ignored_completions: AHashMap<Ustr, CompletionKind>,
260 next_subscription_generation: u64,
261 instruments: AHashMap<i64, InstrumentAny>,
262 book: BookSyncTracker,
263 book_snapshot_timeout: Duration,
264 last_candles: AHashMap<(i64, LighterCandleResolution), LighterWsCandle>,
265 exec_account: Option<(AccountId, i64)>,
266 account_state_reconciler: LighterAccountStateReconciler,
267}
268
269type SubscriptionRetry = Pin<Box<dyn Future<Output = (Ustr, u64)> + Send + Sync + 'static>>;
270
271struct SubscriptionAttempt {
272 channel: LighterWsChannel,
273 auth: Option<SecretString>,
274 pending_auth: Option<SecretString>,
275 generation: u64,
276 retries: u8,
277 response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
278 pending_response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
279}
280
281impl SubscriptionAttempt {
282 fn fold_pending_auth(&mut self) {
283 if let Some(auth) = self.pending_auth.take() {
284 self.auth = Some(auth);
285 self.response_txs.append(&mut self.pending_response_txs);
286 } else {
287 debug_assert!(self.pending_response_txs.is_empty());
288 }
289 }
290}
291
292#[derive(Clone, Copy, Debug, PartialEq, Eq)]
300enum CompletionKind {
301 ControlAck,
302 Typed,
303 AlreadySubscribed,
304}
305
306impl FeedHandler {
307 #[cfg(test)]
308 pub(super) fn new(
309 signal: Arc<AtomicBool>,
310 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
311 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
312 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
313 subscriptions: SubscriptionState,
314 ) -> Self {
315 Self::new_with_settlement_currency(
316 signal,
317 cmd_rx,
318 raw_rx,
319 out_tx,
320 subscriptions,
321 Currency::get_or_create_crypto("USDC"),
322 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
323 )
324 }
325
326 pub(super) fn new_with_settlement_currency(
327 signal: Arc<AtomicBool>,
328 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
329 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
330 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
331 subscriptions: SubscriptionState,
332 settlement_currency: Currency,
333 book_snapshot_timeout: Duration,
334 ) -> Self {
335 Self {
336 clock: get_atomic_clock_realtime(),
337 signal,
338 inner: None,
339 cmd_rx,
340 cmd_tx: None,
341 raw_rx,
342 out_tx,
343 subscriptions,
344 retry_manager: create_websocket_retry_manager(),
345 pending_messages: VecDeque::new(),
346 pending_subs: VecDeque::new(),
347 inflight_subs: AHashMap::new(),
348 subscription_attempts: AHashMap::new(),
349 subscription_retries: FuturesUnordered::new(),
350 ignored_completions: AHashMap::new(),
351 next_subscription_generation: 1,
352 instruments: AHashMap::new(),
353 book: BookSyncTracker::default(),
354 book_snapshot_timeout,
355 last_candles: AHashMap::new(),
356 exec_account: None,
357 account_state_reconciler: LighterAccountStateReconciler::new_with_settlement_currency(
358 settlement_currency,
359 ),
360 }
361 }
362
363 pub(super) fn send(&self, msg: NautilusWsMessage) -> Result<(), String> {
364 self.out_tx
365 .send(msg)
366 .map_err(|e| format!("Failed to send message: {e}"))
367 }
368
369 pub(super) fn set_command_sender(
370 &mut self,
371 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
372 ) {
373 self.cmd_tx = Some(cmd_tx);
374 }
375
376 pub(super) fn is_stopped(&self) -> bool {
377 self.signal.load(Ordering::Relaxed)
378 }
379
380 async fn send_with_retry(&self, payload: String) -> Result<(), LighterWsError> {
381 self.send_secret_with_retry(SecretString::from(payload))
382 .await
383 }
384
385 async fn send_secret_with_retry(&self, payload: SecretString) -> Result<(), LighterWsError> {
386 if let Some(client) = &self.inner {
387 self.retry_manager
388 .invocation(
389 "websocket_send",
390 || {
391 let payload = payload.clone();
392 async move {
393 client
394 .send_text(
395 payload.expose_secret().to_owned(),
396 Some(LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY.as_slice()),
397 )
398 .await
399 .map_err(LighterWsError::Transport)
400 }
401 },
402 should_retry_lighter_ws_error,
403 |e| create_lighter_ws_timeout_error(e.to_string()),
404 )
405 .execute()
406 .await
407 } else {
408 Err(LighterWsError::Client(
409 "no active WebSocket client".to_string(),
410 ))
411 }
412 }
413
414 async fn send_once(
417 &self,
418 payload: String,
419 connection_epoch: u64,
420 ) -> Result<(), LighterWsError> {
421 if let Some(client) = &self.inner {
422 match client
423 .send_text_on_connection(payload, None, connection_epoch)
424 .await
425 {
426 Err(SendError::BrokenPipe(message)) => {
427 Err(LighterWsError::SendTxOutcomeUnknown(message))
428 }
429 result => result.map_err(LighterWsError::Transport),
430 }
431 } else {
432 Err(LighterWsError::Client(
433 "no active WebSocket client".to_string(),
434 ))
435 }
436 }
437
438 async fn dispatch_subscribe(
439 &self,
440 channel: LighterWsChannel,
441 auth: Option<SecretString>,
442 ) -> Result<(), String> {
443 let topic = channel.topic_key();
444
445 let authed = auth.is_some();
446 let mut request = match auth {
447 Some(token) => LighterWsRequest::subscribe_auth(
448 channel.subscription_channel(),
449 token.expose_secret(),
450 ),
451 None => LighterWsRequest::subscribe(channel.subscription_channel()),
452 };
453
454 let payload = serde_json::to_string(&request);
455 request.zeroize();
456
457 match payload {
458 Ok(payload) => {
459 log::debug!("Sending Lighter subscribe: topic={topic} authed={authed}");
462 let result = if authed {
463 self.send_secret_with_retry(SecretString::from(payload))
464 .await
465 } else {
466 self.send_with_retry(payload).await
467 };
468
469 if let Err(e) = result {
470 log::error!("Error subscribing to {topic}: {e}");
471 Err(e.to_string())
472 } else {
473 Ok(())
474 }
475 }
476 Err(e) => {
477 log::error!("Error serializing subscription for {topic}: {e}");
478 Err(format!("failed to serialize subscription for {topic}: {e}"))
479 }
480 }
481 }
482
483 async fn dispatch_send_tx(
484 &self,
485 tx_type: u8,
486 tx_info: Box<serde_json::value::RawValue>,
487 connection_epoch: u64,
488 ) -> Result<(), LighterWsError> {
489 let request = LighterWsRequest::SendTx {
490 data: super::messages::LighterWsSendTx { tx_type, tx_info },
491 };
492
493 match serde_json::to_string(&request) {
494 Ok(payload) => {
495 log::debug!(
496 "Sending Lighter sendTx: tx_type={tx_type} ({} bytes)",
497 payload.len(),
498 );
499
500 match self.send_once(payload, connection_epoch).await {
501 Ok(()) => Ok(()),
502 Err(e) => {
503 log::error!("Error dispatching Lighter sendTx (tx_type={tx_type}): {e}");
504 Err(e)
505 }
506 }
507 }
508 Err(e) => {
509 log::error!("Error serializing Lighter sendTx (tx_type={tx_type}): {e}");
510 Err(LighterWsError::Client(format!(
511 "failed to serialize Lighter sendTx: {e}"
512 )))
513 }
514 }
515 }
516
517 async fn dispatch_unsubscribe(&self, channel: LighterWsChannel) {
518 let topic = channel.topic_key();
519 self.subscriptions.mark_unsubscribe(&topic);
520
521 let request = LighterWsRequest::unsubscribe(channel.subscription_channel());
522 match serde_json::to_string(&request) {
523 Ok(payload) => {
524 log::debug!("Sending Lighter unsubscribe ({} bytes)", payload.len());
525 if let Err(e) = self.send_with_retry(payload).await {
526 log::error!("Error unsubscribing from {topic}: {e}");
527 }
528 }
529 Err(e) => {
530 log::error!("Error serializing unsubscription for {topic}: {e}");
531 }
532 }
533 }
534
535 pub(super) async fn next(&mut self) -> Option<NautilusWsMessage> {
536 if let Some(msg) = self.pending_messages.pop_front() {
537 return Some(msg);
538 }
539
540 loop {
541 self.pump_pending_subscribes().await;
543
544 tokio::select! {
545 biased;
546 Some(work) = self.book.work.next(), if !self.book.work.is_empty() => {
547 match work {
548 BookWorkResult::Sent { market_index, generation, cancel, write, result } => {
549 self.complete_book_send(market_index, generation, cancel, write, result);
550 }
551 BookWorkResult::Initial { market_index, cancel } => {
552 if !cancel.is_cancelled() {
553 self.start_book_recovery(market_index);
554 }
555 }
556 BookWorkResult::Recovery { market_index, recovery, result } => {
557 if let Err(e) = result
558 && !recovery.cancellation.is_cancelled()
559 && let Some(state) = self.book.recovery.get_mut(&market_index)
560 && state.fail(Some(&recovery))
561 {
562 self.book.clear_cached_order_book(market_index);
563 self.book.writes.remove(&market_index);
564 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
565 self.cancel_subscription_attempt(&topic, &e.to_string());
566 log::error!("Lighter book recovery failed for market_index={market_index}; output suppressed until reconnect or resubscribe: {e}");
567 }
568 }
569 }
570 }
571 Some(cmd) = self.cmd_rx.recv() => {
572 match cmd {
573 HandlerCommand::SetClient(client) => {
574 log::debug!("Setting WebSocket client in Lighter handler");
575 self.inner = Some(Arc::new(client));
576 }
577 HandlerCommand::Disconnect => {
578 log::debug!("Lighter handler received disconnect");
579 if let Some(ref client) = self.inner {
580 client.disconnect().await;
581 }
582 self.signal.store(true, Ordering::SeqCst);
583 return None;
584 }
585 HandlerCommand::Subscribe {
586 channel,
587 auth,
588 response_tx,
589 } => {
590 self.queue_subscribe(channel, auth, response_tx);
591 }
592 HandlerCommand::Unsubscribe { channel } => {
593 let topic = channel.topic_key();
596 self.cancel_subscription_attempt(
597 &topic,
598 "subscription cancelled before venue acknowledgement",
599 );
600
601 if let LighterWsChannel::OrderBook(market_index) = &channel {
602 self.book.cancel(*market_index);
603 self.book.clear_cached_order_book(*market_index);
604 }
605 self.dispatch_unsubscribe(channel).await;
606 }
607 HandlerCommand::RecoverBook { market_index, cancel, gate, completion } => {
608 self.queue_book_replacement(market_index, BookWrite { cancel, gate, completion });
609 }
610 HandlerCommand::InitializeInstruments(instruments) => {
611 self.instruments.clear();
612 for (market_index, inst) in instruments {
613 self.instruments.insert(market_index, inst);
614 }
615 }
616 HandlerCommand::UpdateInstrument { market_index, instrument } => {
617 self.instruments.insert(market_index, instrument);
618 }
619 HandlerCommand::SetBookDeltasSub { market_index, subscribed } => {
620 if subscribed {
621 let inserted = self.book.delta_subs.insert(market_index);
622 if inserted
623 && let Some(first) = self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_deltas_snapshot(market_index, instrument, self.clock.get_time_ns()))
624 {
625 return Some(first);
626 }
627 } else {
628 self.book.delta_subs.remove(&market_index);
629 }
630 }
631 HandlerCommand::SetDepthSub { market_index, subscribed } => {
632 if subscribed {
633 let inserted = self.book.depth_subs.insert(market_index);
634 if inserted
635 && let Some(first) =
636 self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_depth_snapshot(market_index, instrument, self.clock.get_time_ns()))
637 {
638 return Some(first);
639 }
640 } else {
641 self.book.depth_subs.remove(&market_index);
642 }
643 }
644 HandlerCommand::SetExecutionContext { account_id, account_index } => {
645 self.exec_account = Some((account_id, account_index));
646 }
647 HandlerCommand::SendTxBatch { data, connection_epoch, response_tx } => {
648 let result = match serde_json::to_string(&LighterWsRequest::SendTxBatch { data }) {
649 Ok(payload) => self.send_once(payload, connection_epoch).await,
650 Err(e) => Err(LighterWsError::Client(format!("failed to serialize Lighter sendTxBatch: {e}"))),
651 };
652 let _ = response_tx.send(result);
653 }
654 HandlerCommand::SendTx {
655 tx_type,
656 tx_info,
657 connection_epoch,
658 response_tx,
659 } => {
660 let result = self
661 .dispatch_send_tx(tx_type, tx_info, connection_epoch)
662 .await;
663
664 if response_tx.send(result).is_err() {
665 log::debug!("Lighter sendTx result receiver dropped");
666 }
667 }
668 }
669 }
670 Some((topic, generation)) = self.subscription_retries.next(),
671 if !self.subscription_retries.is_empty() =>
672 {
673 self.queue_subscription_retry(topic, generation);
674 }
675 Some((connection_epoch, raw_msg)) = self.raw_rx.recv() => {
676 match raw_msg {
677 Message::Text(text) => {
678 if text == RECONNECTED {
679 log::debug!("Received Lighter WebSocket RECONNECTED sentinel");
680 self.book.reset_on_reconnect();
681 self.last_candles.clear();
683 self.reset_subscription_attempts_after_reconnect();
687 self.account_state_reconciler.reset();
688 return Some(NautilusWsMessage::Reconnected {
689 connection_epoch,
690 });
691 }
692
693 let ts_init = self.clock.get_time_ns();
694 let subscribed_topic = typed_subscribe_topic(&text);
695
696 if let Ok(frame) = serde_json::from_str::<LighterWsFrame>(&text) {
697 if let Some(topic) = subscribed_topic
698 && !self.complete_typed_subscription(topic, connection_epoch)
699 {
700 continue;
701 }
702 let messages = self
703 .handle_frame(frame, ts_init)
704 .into_iter()
705 .map(|msg| msg.with_connection_epoch(connection_epoch))
706 .collect();
707
708 if let Some(first) = self.dispatch_results(messages) {
709 return Some(first);
710 }
711 } else if let Ok(value) =
712 serde_json::from_str::<serde_json::Value>(&text)
713 {
714 if let Some(topic) = subscribed_topic {
719 self.complete_typed_subscription(topic, connection_epoch);
720 }
721
722 let (matched, msg) = self.handle_control_value(&value);
723 if let Some(first) = msg {
724 return Some(first.with_connection_epoch(connection_epoch));
725 }
726
727 if !matched {
728 log::warn!("Lighter WS unparsed frame: {value}");
733 return Some(NautilusWsMessage::Raw(value));
734 }
735 } else {
736 log::warn!("Lighter WS non-JSON text: {text}");
737 }
738 }
739 Message::Ping(data) => {
740 if let Some(ref client) = self.inner
741 && let Err(e) = client.send_pong(data.to_vec()).await {
742 log::error!("Error sending Lighter pong: {e}");
743 }
744 }
745 Message::Close(frame) => {
746 log::debug!("Received Lighter WebSocket close frame: {frame:?}");
747 return None;
748 }
749 _ => {}
750 }
751 }
752 else => {
753 log::debug!("Lighter handler shutting down: stream ended or command channel closed");
754 return None;
755 }
756 }
757 }
758 }
759
760 fn dispatch_results(
761 &mut self,
762 mut messages: Vec<NautilusWsMessage>,
763 ) -> Option<NautilusWsMessage> {
764 if messages.is_empty() {
765 return None;
766 }
767 let first = messages.remove(0);
768 for extra in messages {
769 self.pending_messages.push_back(extra);
770 }
771 Some(first)
772 }
773
774 async fn pump_pending_subscribes(&mut self) {
777 while self.inflight_subs.len() < SUBSCRIBE_INFLIGHT_MAX {
778 let Some((topic, generation)) = self.pending_subs.pop_front() else {
779 break;
780 };
781
782 let Some(attempt) = self.subscription_attempts.get(&topic) else {
783 continue;
784 };
785
786 if attempt.generation != generation {
787 continue;
788 }
789
790 let channel = attempt.channel.clone();
791 let auth = attempt.auth.clone();
792 self.inflight_subs.insert(topic, generation);
793 if let LighterWsChannel::OrderBook(market_index) = channel {
794 self.send_book_subscribe(market_index, generation);
795 } else if let Err(message) = self.dispatch_subscribe(channel, auth).await {
796 self.schedule_subscription_retry(topic, generation, &message);
797 }
798 }
799 }
800
801 fn queue_subscribe(
802 &mut self,
803 channel: LighterWsChannel,
804 auth: Option<SecretString>,
805 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
806 ) {
807 let topic = Ustr::from(channel.topic_key().as_str());
808 if let LighterWsChannel::OrderBook(market_index) = channel
809 && response_tx.is_some()
810 && let Some(state) = self.book.recovery.get_mut(&market_index)
811 && state.is_failed()
812 {
813 state.reset();
814 }
815
816 if let LighterWsChannel::OrderBook(market_index) = channel
817 && !self.book.writes.contains_key(&market_index)
818 && self
819 .book
820 .recovery
821 .get(&market_index)
822 .and_then(BookRecoveryState::current)
823 .is_some_and(|recovery| !recovery.is_accepted())
824 {
825 return;
826 }
827
828 if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
829 attempt.channel = channel;
830 if matches!(attempt.channel, LighterWsChannel::OrderBook(_))
831 && !self.inflight_subs.contains_key(&topic)
832 && !self
833 .pending_subs
834 .iter()
835 .any(|(pending, _)| *pending == topic)
836 {
837 self.pending_subs.push_back((topic, attempt.generation));
838 }
839
840 let effective_auth = attempt
841 .pending_auth
842 .as_ref()
843 .or(attempt.auth.as_ref())
844 .map(SecretString::expose_secret);
845 let auth_changed = auth
846 .as_ref()
847 .map(SecretString::expose_secret)
848 .is_some_and(|auth| Some(auth) != effective_auth);
849
850 if auth_changed {
851 let is_inflight = self.inflight_subs.get(&topic) == Some(&attempt.generation);
852 if is_inflight {
853 attempt.pending_auth = auth;
854 if let Some(response_tx) = response_tx {
855 attempt.pending_response_txs.push(response_tx);
856 }
857 } else {
858 debug_assert!(attempt.pending_auth.is_none());
859 debug_assert!(attempt.pending_response_txs.is_empty());
860 attempt.auth = auth;
861 if let Some(response_tx) = response_tx {
862 attempt.response_txs.push(response_tx);
863 }
864 }
865 } else if let Some(response_tx) = response_tx {
866 if auth.is_some() && attempt.pending_auth.is_some() {
867 attempt.pending_response_txs.push(response_tx);
868 } else {
869 attempt.response_txs.push(response_tx);
870 }
871 }
872 return;
873 }
874
875 let newly_pending = self.subscriptions.try_mark_subscribe(topic.as_str());
876 if !newly_pending
877 && auth.is_none()
878 && !self
879 .subscriptions
880 .pending_subscribe_topics()
881 .iter()
882 .any(|pending| pending == topic.as_str())
883 {
884 if let Some(response_tx) = response_tx {
885 let _ = response_tx.send(Ok(()));
886 }
887 return;
888 }
889
890 let generation = self.take_subscription_generation();
891 self.subscription_attempts.insert(
892 topic,
893 SubscriptionAttempt {
894 channel,
895 auth,
896 pending_auth: None,
897 generation,
898 retries: 0,
899 response_txs: response_tx.into_iter().collect(),
900 pending_response_txs: Vec::new(),
901 },
902 );
903 self.pending_subs.push_back((topic, generation));
904 }
905
906 fn queue_subscription_retry(&mut self, topic: Ustr, generation: u64) {
907 if self
908 .subscription_attempts
909 .get(&topic)
910 .is_some_and(|attempt| attempt.generation == generation)
911 && self.inflight_subs.get(&topic) != Some(&generation)
912 && !self
913 .pending_subs
914 .iter()
915 .any(|pending| *pending == (topic, generation))
916 {
917 self.pending_subs.push_back((topic, generation));
918 }
919 }
920
921 fn complete_subscription(&mut self, topic: &str, kind: CompletionKind) -> bool {
922 let topic = Ustr::from(topic);
923
924 if self.ignored_completions.get(&topic) == Some(&kind) {
929 self.ignored_completions.remove(&topic);
930 return false;
931 }
932
933 let Some(generation) = self.inflight_subs.get(&topic).copied() else {
934 return false;
935 };
936
937 if !self
938 .subscription_attempts
939 .get(&topic)
940 .is_some_and(|attempt| attempt.generation == generation)
941 {
942 return false;
943 }
944
945 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
946 && !self
947 .book
948 .expected
949 .get(&market_index)
950 .is_some_and(|expected| expected.0 == generation)
951 {
952 return false;
953 }
954
955 self.inflight_subs.remove(&topic);
956 self.subscriptions.confirm_subscribe(topic.as_str());
957
958 if kind == CompletionKind::ControlAck {
962 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
963 && let Some(expected) = self.book.expected.get(&market_index)
964 && expected.0 == generation
965 {
966 self.book.trailing.entry(market_index).or_insert(*expected);
967 }
968
969 self.ignored_completions
970 .insert(topic, CompletionKind::Typed);
971 }
972 let mut attempt = self
973 .subscription_attempts
974 .remove(&topic)
975 .expect("matching subscription attempt disappeared");
976 for response_tx in std::mem::take(&mut attempt.response_txs) {
977 let _ = response_tx.send(Ok(()));
978 }
979
980 if let Some(auth) = attempt.pending_auth.take() {
981 let generation = self.take_subscription_generation();
982 attempt.auth = Some(auth);
983 attempt.generation = generation;
984 attempt.retries = 0;
985 attempt.response_txs = std::mem::take(&mut attempt.pending_response_txs);
986 self.subscription_attempts.insert(topic, attempt);
987 self.pending_subs.push_back((topic, generation));
988 } else {
989 debug_assert!(attempt.pending_response_txs.is_empty());
990 }
991 true
992 }
993
994 fn retry_inflight_subscriptions(&mut self, message: &str) {
995 let mut inflight: Vec<(Ustr, u64)> = self
996 .inflight_subs
997 .iter()
998 .map(|(topic, generation)| (*topic, *generation))
999 .collect();
1000 inflight.sort_unstable_by_key(|(topic, _)| *topic);
1001
1002 for (topic, generation) in inflight {
1003 self.schedule_subscription_retry(topic, generation, message);
1004 }
1005 }
1006
1007 fn fail_inflight_subscriptions(&mut self, message: &str) {
1008 let mut topics: Vec<(Ustr, u64)> = self
1009 .inflight_subs
1010 .iter()
1011 .map(|(topic, generation)| (*topic, *generation))
1012 .collect();
1013 topics.sort_unstable();
1014
1015 for (topic, generation) in topics {
1016 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1017 self.reject_book_subscription(
1018 market_index,
1019 generation,
1020 LighterWsError::Client(message.into()),
1021 );
1022 } else {
1023 self.fail_subscription_attempt(topic, message);
1024 }
1025 }
1026 }
1027
1028 fn schedule_subscription_retry(&mut self, topic: Ustr, generation: u64, message: &str) {
1029 if self.inflight_subs.get(&topic) != Some(&generation) {
1030 return;
1031 }
1032
1033 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1034 self.reject_book_subscription(
1035 market_index,
1036 generation,
1037 LighterWsError::Network(message.into()),
1038 );
1039 return;
1040 }
1041
1042 self.inflight_subs.remove(&topic);
1043 self.subscriptions.mark_failure(topic.as_str());
1044
1045 let Some(attempt) = self.subscription_attempts.get_mut(&topic) else {
1046 return;
1047 };
1048
1049 if attempt.generation != generation {
1050 return;
1051 }
1052
1053 attempt.fold_pending_auth();
1054 if attempt.retries >= SUBSCRIBE_RETRY_MAX {
1055 self.fail_subscription_attempt(topic, message);
1056 return;
1057 }
1058
1059 attempt.retries += 1;
1060 let retry = attempt.retries;
1061 let next_generation = self.take_subscription_generation();
1062 let attempt = self
1063 .subscription_attempts
1064 .get_mut(&topic)
1065 .expect("subscription attempt disappeared before retry");
1066 attempt.generation = next_generation;
1067
1068 let delay = SUBSCRIBE_RETRY_BASE_BACKOFF.saturating_mul(1_u32 << (retry - 1));
1069 self.subscription_retries.push(Box::pin(async move {
1070 tokio::time::sleep(delay).await;
1071 (topic, next_generation)
1072 }));
1073 }
1074
1075 fn fail_subscription_attempt(&mut self, topic: Ustr, message: &str) {
1076 self.inflight_subs.remove(&topic);
1077 self.pending_subs.retain(|(pending, _)| *pending != topic);
1078 self.subscriptions.mark_unsubscribe(topic.as_str());
1079 self.subscriptions.confirm_unsubscribe(topic.as_str());
1080
1081 if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1082 let attempts = attempt.retries + 1;
1083 for response_tx in attempt
1084 .response_txs
1085 .into_iter()
1086 .chain(attempt.pending_response_txs)
1087 {
1088 let _ = response_tx.send(Err(format!(
1089 "subscription {topic} failed after {attempts} attempts: {message}",
1090 )));
1091 }
1092 }
1093 }
1094
1095 fn cancel_subscription_attempt(&mut self, topic: &str, message: &str) {
1096 let topic = Ustr::from(topic);
1097 self.inflight_subs.remove(&topic);
1098 self.pending_subs.retain(|(pending, _)| *pending != topic);
1099 if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1100 for response_tx in attempt
1101 .response_txs
1102 .into_iter()
1103 .chain(attempt.pending_response_txs)
1104 {
1105 let _ = response_tx.send(Err(format!("{message}: {topic}")));
1106 }
1107 }
1108 }
1109
1110 fn reset_subscription_attempts_after_reconnect(&mut self) {
1111 for topic in self.subscriptions.all_topics() {
1112 self.subscriptions.mark_failure(&topic);
1113 }
1114 self.pending_subs.clear();
1115 self.inflight_subs.clear();
1116 self.ignored_completions.clear();
1119
1120 let mut topics: Vec<Ustr> = self.subscription_attempts.keys().copied().collect();
1121 topics.sort_unstable();
1122 for topic in topics {
1123 let generation = self.take_subscription_generation();
1124 let attempt = self
1125 .subscription_attempts
1126 .get_mut(&topic)
1127 .expect("subscription attempt disappeared during reconnect");
1128 attempt.fold_pending_auth();
1129 attempt.generation = generation;
1130 attempt.retries = 0;
1131
1132 if order_book_market_index_from_topic(topic.as_str()).is_none() {
1133 self.pending_subs.push_back((topic, generation));
1134 }
1135 }
1136 }
1137
1138 fn take_subscription_generation(&mut self) -> u64 {
1139 let generation = self.next_subscription_generation;
1140 self.next_subscription_generation =
1141 self.next_subscription_generation.wrapping_add(1).max(1);
1142 generation
1143 }
1144
1145 fn handle_control_value(
1150 &mut self,
1151 value: &serde_json::Value,
1152 ) -> (bool, Option<NautilusWsMessage>) {
1153 if value.get("type").and_then(|v| v.as_str()) == Some("jsonapi/sendtxbatch")
1154 || value
1155 .get("id")
1156 .and_then(|v| v.as_str())
1157 .is_some_and(|id| id.starts_with(CANCEL_BATCH_ID_PREFIX))
1158 {
1159 let body = value.get("error").unwrap_or(value);
1160 let id = value.get("id").and_then(|v| v.as_str());
1161 let code = body.get("code").and_then(|v| v.as_i64());
1162
1163 let (Some(id), Some(code)) = (id, code) else {
1164 log::warn!("Ignoring malformed Lighter sendTxBatch response");
1165 return (true, None);
1166 };
1167
1168 return (
1169 true,
1170 Some(NautilusWsMessage::SendTxBatchResult {
1171 connection_epoch: 0,
1172 id: id.to_string(),
1173 code,
1174 message: body
1175 .get("message")
1176 .and_then(|v| v.as_str())
1177 .unwrap_or("batch rejected")
1178 .to_string(),
1179 tx_hashes: value
1180 .get("tx_hash")
1181 .and_then(|v| serde_json::from_value(v.clone()).ok())
1182 .unwrap_or_default(),
1183 }),
1184 );
1185 }
1186
1187 if let Some(error) = already_subscribed_error(value) {
1188 self.confirm_already_subscribed(error);
1189 return (true, None);
1190 }
1191 let subscription_code = subscription_error_code(value);
1192
1193 if subscription_code == Some(LIGHTER_ERROR_CODE_WS_RATE_LIMITED) {
1194 self.retry_inflight_subscriptions(&format!(
1195 "venue rejected the WebSocket subscribe with code \
1196 {LIGHTER_ERROR_CODE_WS_RATE_LIMITED}",
1197 ));
1198 return (true, None);
1199 }
1200
1201 if subscription_code == Some(LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED) {
1202 self.fail_inflight_subscriptions(&format!(
1203 "venue rejected the WebSocket subscribe with code \
1204 {LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED}",
1205 ));
1206 return (true, None);
1207 }
1208
1209 let kind = value.get("type").and_then(|v| v.as_str()).unwrap_or("");
1210
1211 match kind {
1212 CTRL_TYPE_CONNECTED => {
1213 log::debug!("Lighter WebSocket handshake complete");
1214 (true, None)
1215 }
1216 CTRL_TYPE_PING | CTRL_TYPE_PONG => (true, None),
1217 CTRL_TYPE_SEND_TX => {
1218 let raw_code = value.get("code").and_then(|v| v.as_u64());
1219 match raw_code {
1220 Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) => {
1221 log_integrator_not_approved();
1222 (
1223 true,
1224 Some(send_tx_rejected_from_value(
1225 value,
1226 SendTxRejectionSource::Ack,
1227 )),
1228 )
1229 }
1230 Some(200) => {
1231 log::debug!("Lighter WebSocket sendTx ack: {value}");
1232 let tx_hash = value
1233 .get("tx_hash")
1234 .and_then(|v| v.as_str())
1235 .map(str::to_string);
1236 (
1237 true,
1238 Some(NautilusWsMessage::SendTxAck {
1239 connection_epoch: 0,
1240 tx_hash,
1241 code: 200,
1242 }),
1243 )
1244 }
1245 Some(_) => {
1246 log::error!("Lighter sendTx rejected: {value}");
1247 (
1248 true,
1249 Some(send_tx_rejected_from_value(
1250 value,
1251 SendTxRejectionSource::Ack,
1252 )),
1253 )
1254 }
1255 None => {
1256 log::warn!(
1257 "Ignoring malformed Lighter sendTx response without numeric code: {value}",
1258 );
1259 (true, None)
1260 }
1261 }
1262 }
1263 CTRL_TYPE_SUBSCRIBED | CTRL_TYPE_UNSUBSCRIBED => {
1264 if let Some(topic) = value.get("channel").and_then(|v| v.as_str()) {
1265 if kind == CTRL_TYPE_SUBSCRIBED {
1266 self.complete_subscription(topic, CompletionKind::ControlAck);
1267 } else {
1268 let was_pending_unsubscribe = self
1269 .subscriptions
1270 .pending_unsubscribe_topics()
1271 .iter()
1272 .any(|pending| pending == topic);
1273 self.subscriptions.confirm_unsubscribe(topic);
1274
1275 if was_pending_unsubscribe {
1276 if let Some(market_index) = order_book_market_index_from_topic(topic) {
1278 self.book.clear_cached_order_book(market_index);
1279 }
1280
1281 if let Some(key) = candle_market_and_resolution_from_topic(topic) {
1282 self.last_candles.remove(&key);
1283 }
1284 }
1285 }
1286 }
1287 (true, None)
1288 }
1289 CTRL_TYPE_ERROR => {
1290 let code = value.get("code").and_then(|v| v.as_u64());
1291 if code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1292 log_integrator_not_approved();
1293 } else {
1294 log::warn!("Lighter WebSocket error frame: {value}");
1295 }
1296
1297 if is_sendtx_error_code(code) {
1298 (
1299 true,
1300 Some(send_tx_rejected_from_value(
1301 value,
1302 SendTxRejectionSource::BareError,
1303 )),
1304 )
1305 } else {
1306 (true, None)
1307 }
1308 }
1309 _ => {
1310 if let Some(error) = value.get("error") {
1311 let nested_code = error.get("code").and_then(|v| v.as_u64());
1312 if nested_code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1313 log_integrator_not_approved();
1314 } else {
1315 log::warn!("Lighter WebSocket error frame: {value}");
1316 }
1317 let rejected = is_sendtx_error_code(nested_code).then(|| {
1318 send_tx_rejected_from_nested_error(error, SendTxRejectionSource::BareError)
1319 });
1320 return (true, rejected);
1321 }
1322 (false, None)
1323 }
1324 }
1325 }
1326
1327 fn confirm_already_subscribed(&mut self, error: &serde_json::Value) {
1328 let Some(topic) = error
1329 .get("message")
1330 .and_then(|value| value.as_str())
1331 .and_then(|message| message.strip_prefix("Already Subscribed to : "))
1332 else {
1333 log::debug!(
1334 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1335 );
1336 return;
1337 };
1338
1339 if !self.complete_subscription(topic, CompletionKind::AlreadySubscribed) {
1340 log::debug!(
1341 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1342 );
1343 return;
1344 }
1345
1346 log::debug!(
1347 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}, topic={topic}",
1348 );
1349 }
1350
1351 fn handle_frame(
1352 &mut self,
1353 frame: LighterWsFrame,
1354 ts_init: UnixNanos,
1355 ) -> Vec<NautilusWsMessage> {
1356 match frame {
1357 LighterWsFrame::OrderBookSnapshot {
1358 channel,
1359 order_book,
1360 timestamp,
1361 ..
1362 } => self.handle_order_book(channel, &order_book, timestamp, true, ts_init),
1363 LighterWsFrame::OrderBook {
1364 channel,
1365 order_book,
1366 timestamp,
1367 ..
1368 } => self.handle_order_book(channel, &order_book, timestamp, false, ts_init),
1369 LighterWsFrame::TickerSnapshot {
1370 channel,
1371 ticker,
1372 timestamp,
1373 ..
1374 }
1375 | LighterWsFrame::Ticker {
1376 channel,
1377 ticker,
1378 timestamp,
1379 ..
1380 } => self.handle_ticker(channel, &ticker, timestamp, ts_init),
1381 LighterWsFrame::TradeSnapshot {
1382 trades,
1383 liquidation_trades,
1384 ..
1385 }
1386 | LighterWsFrame::Trade {
1387 trades,
1388 liquidation_trades,
1389 ..
1390 } => self.handle_trades(&trades, &liquidation_trades, ts_init),
1391 LighterWsFrame::AccountOrders { ref orders, .. }
1392 | LighterWsFrame::AccountAllOrders { ref orders, .. } => {
1393 if self.exec_account.is_none() {
1394 return raw_message(&frame);
1395 }
1396 let mut msgs = self.handle_account_orders(orders, ts_init);
1397 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1398 AccountStream::Orders,
1399 ));
1400 msgs
1401 }
1402 LighterWsFrame::AccountAllTradesSnapshot { .. } => {
1410 if self.exec_account.is_none() {
1411 return raw_message(&frame);
1412 }
1413 log::debug!(
1414 "Skipping Lighter account_all_trades snapshot frame; \
1415 reconcile historical fills via HTTP",
1416 );
1417 vec![NautilusWsMessage::AccountStreamFirstFrame(
1418 AccountStream::Trades,
1419 )]
1420 }
1421 LighterWsFrame::AccountAllTrades { ref trades, .. } => {
1422 if self.exec_account.is_none() {
1423 return raw_message(&frame);
1424 }
1425 let mut msgs = self.handle_account_trades(trades.values().flatten(), ts_init);
1426 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1427 AccountStream::Trades,
1428 ));
1429 msgs
1430 }
1431 LighterWsFrame::AccountAllPositionsSnapshot { ref positions, .. } => {
1432 if self.exec_account.is_none() {
1433 return raw_message(&frame);
1434 }
1435 let mut msgs =
1436 self.handle_account_positions(positions, ts_init, PositionFrameType::Snapshot);
1437 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1438 AccountStream::Positions,
1439 ));
1440 msgs
1441 }
1442 LighterWsFrame::AccountAllPositions { ref positions, .. } => {
1443 if self.exec_account.is_none() {
1444 return raw_message(&frame);
1445 }
1446 self.handle_account_positions(positions, ts_init, PositionFrameType::Update)
1447 }
1448 LighterWsFrame::AccountAllAssets {
1449 ref assets,
1450 timestamp,
1451 ..
1452 } => {
1453 if self.exec_account.is_none() {
1454 return raw_message(&frame);
1455 }
1456 let mut msgs = self.handle_account_assets(assets, timestamp, ts_init);
1457 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1458 AccountStream::Assets,
1459 ));
1460 msgs
1461 }
1462 LighterWsFrame::UserStats {
1463 ref stats,
1464 timestamp,
1465 ..
1466 } => {
1467 if self.exec_account.is_none() {
1468 return raw_message(&frame);
1469 }
1470 let mut msgs = self.handle_user_stats(stats, timestamp, ts_init);
1471 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1472 AccountStream::UserStats,
1473 ));
1474 msgs
1475 }
1476 LighterWsFrame::MarketStats {
1477 ref market_stats,
1478 timestamp,
1479 ..
1480 } => self.handle_market_stats(market_stats, timestamp, ts_init),
1481 LighterWsFrame::SpotMarketStats {
1482 ref spot_market_stats,
1483 timestamp,
1484 ..
1485 } => self.handle_spot_market_stats(spot_market_stats, timestamp, ts_init),
1486 LighterWsFrame::CandleSnapshot {
1487 channel,
1488 ref candles,
1489 ..
1490 }
1491 | LighterWsFrame::Candle {
1492 channel,
1493 ref candles,
1494 ..
1495 } => self.handle_candles(channel, candles, ts_init),
1496 LighterWsFrame::Height { .. } => raw_message(&frame),
1497 }
1498 }
1499
1500 fn handle_order_book(
1501 &mut self,
1502 channel: Ustr,
1503 book: &LighterWsOrderBook,
1504 timestamp: u64,
1505 is_snapshot: bool,
1506 ts_init: UnixNanos,
1507 ) -> Vec<NautilusWsMessage> {
1508 let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1509 log::debug!("Lighter order_book frame missing market index in channel '{channel}'");
1510 return Vec::new();
1511 };
1512
1513 let Some(instrument) = self.instruments.get(&market_index) else {
1514 log::debug!("No instrument cached for Lighter market_index={market_index}");
1515 return Vec::new();
1516 };
1517
1518 match self.book.validate_sequence(market_index, book, is_snapshot) {
1519 BookSequenceOutcome::Accept => self.book.apply(
1520 market_index,
1521 instrument,
1522 book,
1523 timestamp,
1524 is_snapshot,
1525 ts_init,
1526 ),
1527 BookSequenceOutcome::Suppress => Vec::new(),
1528 BookSequenceOutcome::Recover => {
1529 self.start_book_recovery(market_index);
1530 Vec::new()
1531 }
1532 }
1533 }
1534
1535 fn queue_book_replacement(&mut self, market_index: i64, write: BookWrite) {
1536 if write.cancel.is_cancelled() || !self.order_book_stream_is_referenced(market_index) {
1537 return;
1538 }
1539
1540 self.book.clear_cached_order_book(market_index);
1541 self.book.expected.remove(&market_index);
1542 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1543 self.retire_book_attempt(&topic);
1544 self.subscriptions.mark_failure(&topic);
1545 self.book.writes.insert(market_index, write);
1546 self.queue_subscribe(LighterWsChannel::OrderBook(market_index), None, None);
1547 }
1548
1549 fn retire_book_attempt(&mut self, topic: &str) {
1550 let topic = Ustr::from(topic);
1551 self.inflight_subs.remove(&topic);
1552 self.pending_subs.retain(|(pending, _)| *pending != topic);
1553 if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
1554 attempt.generation = self.next_subscription_generation;
1556 self.next_subscription_generation =
1557 self.next_subscription_generation.wrapping_add(1).max(1);
1558 self.pending_subs.push_back((topic, attempt.generation));
1559 }
1560 }
1561
1562 fn reject_book_subscription(
1563 &mut self,
1564 market_index: i64,
1565 generation: u64,
1566 error: LighterWsError,
1567 ) {
1568 if !self
1569 .book
1570 .expected
1571 .get(&market_index)
1572 .is_some_and(|expected| expected.0 == generation)
1573 || self
1574 .book
1575 .recovery
1576 .get(&market_index)
1577 .and_then(BookRecoveryState::current)
1578 .is_some_and(|recovery| recovery.gate.lock().is_closed())
1579 {
1580 return;
1581 }
1582
1583 self.reject_book(market_index, error);
1584 }
1585
1586 fn reject_book(&mut self, market_index: i64, error: LighterWsError) {
1587 self.book.expected.remove(&market_index);
1588 self.book.clear_cached_order_book(market_index);
1589
1590 if let Some(recovery) = self
1591 .book
1592 .recovery
1593 .get(&market_index)
1594 .and_then(BookRecoveryState::current)
1595 && !recovery.is_accepted()
1596 {
1597 recovery.gate.lock().close();
1598 recovery
1599 .outcome
1600 .send_replace(BookRecoveryOutcome::Rejected(error));
1601 } else if matches!(error, LighterWsError::Client(_)) {
1602 self.book
1603 .recovery
1604 .entry(market_index)
1605 .or_default()
1606 .fail(None);
1607 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1608 self.cancel_subscription_attempt(&topic, &error.to_string());
1609 log::error!(
1610 "Lighter book subscription rejected for market_index={market_index}: {error}"
1611 );
1612 } else {
1613 self.start_book_recovery(market_index);
1614 }
1615 }
1616
1617 fn start_book_recovery(&mut self, market_index: i64) {
1618 if !self.order_book_stream_is_referenced(market_index) {
1619 return;
1620 }
1621
1622 let Some(cmd_tx) = self.cmd_tx.clone() else {
1623 return;
1624 };
1625
1626 let Some(recovery) = self.book.recovery.entry(market_index).or_default().claim() else {
1627 return;
1628 };
1629
1630 recovery.gate.lock().close();
1631 self.book.initial.remove(&market_index);
1632 self.book.expected.remove(&market_index);
1633 self.book.clear_cached_order_book(market_index);
1634 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1635 self.inflight_subs.remove(&Ustr::from(topic.as_str()));
1636 self.pending_subs
1637 .retain(|(pending, _)| pending.as_str() != topic);
1638 self.book.work.push(recovery::recover(
1639 market_index,
1640 recovery,
1641 cmd_tx,
1642 self.book_snapshot_timeout,
1643 ));
1644 }
1645
1646 fn complete_typed_subscription(&mut self, topic: &str, epoch: u64) -> bool {
1647 if let Some(market_index) = order_book_market_index_from_topic(topic)
1648 && self
1649 .book
1650 .expected
1651 .get(&market_index)
1652 .is_some_and(|expected| expected.1 != epoch)
1653 {
1654 return false;
1655 }
1656
1657 if !self.book_snapshot_matches(topic, epoch) {
1658 if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1659 self.ignored_completions.remove(&Ustr::from(topic));
1660 }
1661
1662 return false;
1663 }
1664
1665 self.complete_subscription(topic, CompletionKind::Typed);
1666 true
1667 }
1668
1669 fn book_snapshot_matches(&mut self, topic: &str, epoch: u64) -> bool {
1670 let Some(market_index) = order_book_market_index_from_topic(topic) else {
1671 return true;
1672 };
1673
1674 let source =
1675 if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1676 self.book.trailing.remove(&market_index)
1677 } else {
1678 self.inflight_subs
1679 .get(&Ustr::from(topic))
1680 .map(|generation| (*generation, epoch))
1681 };
1682
1683 source.is_some_and(|source| {
1684 source.1 == epoch && Some(source) == self.book.expected.get(&market_index).copied()
1685 })
1686 }
1687
1688 fn send_book_subscribe(&mut self, market_index: i64, generation: u64) {
1689 let write = self.book.writes.remove(&market_index);
1690 let cancel = write
1691 .as_ref()
1692 .map_or_else(CancellationToken::new, |write| write.cancel.clone());
1693 if write.is_none() {
1694 self.book.initial.insert(
1695 market_index,
1696 PendingSnapshot {
1697 deadline: None,
1698 cancel: cancel.clone(),
1699 gate: SnapshotGate::default(),
1700 },
1701 );
1702 }
1703
1704 let client = self.inner.clone();
1705 let subscriptions = self.subscriptions.clone();
1706 let book_snapshot_timeout = self.book_snapshot_timeout;
1707 self.book.work.push(recovery::subscribe(
1708 market_index,
1709 generation,
1710 cancel,
1711 write,
1712 client,
1713 subscriptions,
1714 book_snapshot_timeout,
1715 ));
1716 }
1717
1718 fn complete_book_send(
1719 &mut self,
1720 market_index: i64,
1721 generation: u64,
1722 cancel: CancellationToken,
1723 write: Option<BookWrite>,
1724 result: Result<u64, LighterWsError>,
1725 ) {
1726 let topic = Ustr::from(
1727 LighterWsChannel::OrderBook(market_index)
1728 .topic_key()
1729 .as_str(),
1730 );
1731
1732 if cancel.is_cancelled()
1733 || self.inflight_subs.get(&topic) != Some(&generation)
1734 || !self.order_book_stream_is_referenced(market_index)
1735 {
1736 return;
1737 }
1738
1739 match result {
1740 Ok(epoch) => {
1741 self.book.expected.insert(market_index, (generation, epoch));
1742
1743 if let Some(write) = write {
1744 write.gate.open();
1745 let _ = write.completion.send(Ok(()));
1746 } else {
1747 self.book.work.push(recovery::wait_for_snapshot(
1748 market_index,
1749 cancel,
1750 self.book_snapshot_timeout,
1751 ));
1752 }
1753 }
1754 Err(e) => {
1755 if let Some(write) = write {
1756 let _ = write.completion.send(Err(e));
1757 } else {
1758 self.reject_book(market_index, e);
1759 }
1760 }
1761 }
1762 }
1763
1764 fn order_book_stream_is_referenced(&self, market_index: i64) -> bool {
1765 let channel = LighterWsChannel::OrderBook(market_index);
1766 self.subscriptions.get_reference_count(&channel.topic_key()) > 0
1767 && (self.book.delta_subs.contains(&market_index)
1768 || self.book.depth_subs.contains(&market_index))
1769 }
1770
1771 fn handle_ticker(
1772 &self,
1773 channel: Ustr,
1774 ticker: &super::messages::LighterTicker,
1775 timestamp: u64,
1776 ts_init: UnixNanos,
1777 ) -> Vec<NautilusWsMessage> {
1778 let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1784 log::debug!("Lighter ticker frame missing market index in channel '{channel}'");
1785 return Vec::new();
1786 };
1787
1788 let Some(instrument) = self.instruments.get(&market_index) else {
1789 log::debug!("No instrument cached for Lighter ticker market_index={market_index}");
1790 return Vec::new();
1791 };
1792
1793 match parse_ws_quote_tick(ticker, instrument, timestamp, ts_init) {
1794 Ok(Some(quote)) => vec![NautilusWsMessage::Quote(quote)],
1795 Ok(None) => {
1796 log::debug!(
1797 "Skipping Lighter ticker for market_index={market_index}: one-sided book",
1798 );
1799 Vec::new()
1800 }
1801 Err(e) => {
1802 log::error!("Error parsing Lighter ticker frame: {e}");
1803 Vec::new()
1804 }
1805 }
1806 }
1807
1808 fn handle_trades(
1809 &self,
1810 trades: &[crate::http::models::LighterTrade],
1811 liquidation_trades: &[crate::http::models::LighterTrade],
1812 ts_init: UnixNanos,
1813 ) -> Vec<NautilusWsMessage> {
1814 let Some(market_index) = trades
1818 .first()
1819 .or_else(|| liquidation_trades.first())
1820 .map(|t| t.market_id)
1821 else {
1822 return Vec::new();
1823 };
1824
1825 let Some(instrument) = self.instruments.get(&market_index) else {
1826 log::debug!("No instrument cached for Lighter trade market_index={market_index}");
1827 return Vec::new();
1828 };
1829
1830 let mut ticks = Vec::with_capacity(trades.len() + liquidation_trades.len());
1831 for trade in trades.iter().chain(liquidation_trades.iter()) {
1832 match parse_ws_trade_tick(trade, instrument, ts_init) {
1833 Ok(tick) => ticks.push(tick),
1834 Err(e) => log::error!("Error parsing Lighter trade tick: {e}"),
1835 }
1836 }
1837
1838 if ticks.is_empty() {
1839 Vec::new()
1840 } else {
1841 vec![NautilusWsMessage::Trades(ticks)]
1842 }
1843 }
1844
1845 fn handle_market_stats(
1846 &self,
1847 payload: &super::messages::LighterMarketStatsPayload,
1848 timestamp: u64,
1849 ts_init: UnixNanos,
1850 ) -> Vec<NautilusWsMessage> {
1851 match payload {
1852 super::messages::LighterMarketStatsPayload::All(stats) => stats
1853 .values()
1854 .flat_map(|stats| self.handle_one_market_stats(stats, timestamp, ts_init))
1855 .collect(),
1856 super::messages::LighterMarketStatsPayload::One(stats) => {
1857 self.handle_one_market_stats(stats, timestamp, ts_init)
1858 }
1859 }
1860 }
1861
1862 fn handle_one_market_stats(
1863 &self,
1864 stats: &super::messages::LighterMarketStats,
1865 timestamp: u64,
1866 ts_init: UnixNanos,
1867 ) -> Vec<NautilusWsMessage> {
1868 let Some(instrument) = self.instruments.get(&stats.market_id) else {
1869 log::debug!(
1870 "No instrument cached for Lighter market_stats market_id={}",
1871 stats.market_id,
1872 );
1873 return Vec::new();
1874 };
1875
1876 let mut messages = Vec::with_capacity(3);
1877
1878 match parse_ws_mark_price_update(stats, instrument, timestamp, ts_init) {
1879 Ok(mark_price) => messages.push(NautilusWsMessage::MarkPrice(mark_price)),
1880 Err(e) => log::error!("Error parsing Lighter mark price: {e}"),
1881 }
1882
1883 match parse_ws_index_price_update(stats, instrument, timestamp, ts_init) {
1884 Ok(index_price) => messages.push(NautilusWsMessage::IndexPrice(index_price)),
1885 Err(e) => log::error!("Error parsing Lighter index price: {e}"),
1886 }
1887
1888 match parse_ws_funding_rate_update(stats, instrument, timestamp, ts_init) {
1889 Ok(funding_rate) => messages.push(NautilusWsMessage::FundingRate(funding_rate)),
1890 Err(e) => log::error!("Error parsing Lighter funding rate: {e}"),
1891 }
1892
1893 messages
1894 }
1895
1896 fn handle_spot_market_stats(
1897 &self,
1898 payload: &super::messages::LighterSpotMarketStatsPayload,
1899 timestamp: u64,
1900 ts_init: UnixNanos,
1901 ) -> Vec<NautilusWsMessage> {
1902 match payload {
1903 super::messages::LighterSpotMarketStatsPayload::All(stats) => stats
1904 .values()
1905 .filter_map(|stats| self.handle_one_spot_market_stats(stats, timestamp, ts_init))
1906 .collect(),
1907 super::messages::LighterSpotMarketStatsPayload::One(stats) => self
1908 .handle_one_spot_market_stats(stats, timestamp, ts_init)
1909 .into_iter()
1910 .collect(),
1911 }
1912 }
1913
1914 fn handle_one_spot_market_stats(
1915 &self,
1916 stats: &super::messages::LighterSpotMarketStats,
1917 timestamp: u64,
1918 ts_init: UnixNanos,
1919 ) -> Option<NautilusWsMessage> {
1920 let Some(instrument) = self.instruments.get(&stats.market_id) else {
1921 log::debug!(
1922 "No instrument cached for Lighter spot_market_stats market_id={}",
1923 stats.market_id,
1924 );
1925 return None;
1926 };
1927
1928 match parse_ws_spot_index_price_update(stats, instrument, timestamp, ts_init) {
1929 Ok(index_price) => Some(NautilusWsMessage::IndexPrice(index_price)),
1930 Err(e) => {
1931 log::error!("Error parsing Lighter spot index price: {e}");
1932 None
1933 }
1934 }
1935 }
1936
1937 fn handle_candles(
1938 &mut self,
1939 channel: Ustr,
1940 candles: &[LighterWsCandle],
1941 ts_init: UnixNanos,
1942 ) -> Vec<NautilusWsMessage> {
1943 let Some((market_index, resolution)) =
1944 candle_market_and_resolution_from_topic(channel.as_str())
1945 else {
1946 log::warn!("Lighter candle frame with unparsable channel `{channel}`");
1947 return Vec::new();
1948 };
1949
1950 let Some(instrument) = self.instruments.get(&market_index) else {
1951 log::debug!("No instrument cached for Lighter candle market_index={market_index}");
1952 return Vec::new();
1953 };
1954
1955 let key = (market_index, resolution);
1956 let mut emitted = Vec::new();
1957
1958 for candle in candles {
1959 let previous = self.last_candles.get(&key).cloned();
1960 match previous {
1961 None => {}
1962 Some(prev) if candle.t > prev.t => {
1963 match parse_ws_bar(instrument, &prev, resolution, ts_init) {
1965 Ok(bar) => emitted.push(NautilusWsMessage::Bar(bar)),
1966 Err(e) => log::error!("Error parsing Lighter candle bar: {e}"),
1967 }
1968 }
1969 Some(prev) if candle.t < prev.t => continue,
1970 Some(_) => {}
1971 }
1972 self.last_candles.insert(key, candle.clone());
1973 }
1974
1975 emitted
1976 }
1977
1978 fn handle_account_orders(
1979 &self,
1980 orders_by_market: &AHashMap<Ustr, Vec<LighterOrder>>,
1981 _ts_init: UnixNanos,
1982 ) -> Vec<NautilusWsMessage> {
1983 if self.exec_account.is_none() {
1984 log::debug!("Lighter account_orders frame skipped: no execution context set");
1985 return Vec::new();
1986 }
1987
1988 let mut reports = Vec::new();
1989
1990 for orders in orders_by_market.values() {
1991 for order in orders {
1992 if !self.instruments.contains_key(&order.market_index) {
1993 log::debug!(
1994 "No instrument cached for Lighter order market_index={}",
1995 order.market_index,
1996 );
1997 continue;
1998 }
1999
2000 reports.push(ExecutionReport::Order(order.clone()));
2001 }
2002 }
2003
2004 if reports.is_empty() {
2005 Vec::new()
2006 } else {
2007 vec![NautilusWsMessage::ExecutionReports(reports)]
2008 }
2009 }
2010
2011 fn handle_account_trades<'a>(
2012 &self,
2013 trades: impl IntoIterator<Item = &'a LighterTrade>,
2014 _ts_init: UnixNanos,
2015 ) -> Vec<NautilusWsMessage> {
2016 let Some((_account_id, account_index)) = self.exec_account else {
2017 log::debug!("Lighter account_trades frame skipped: no execution context set");
2018 return Vec::new();
2019 };
2020
2021 let mut reports = Vec::new();
2022
2023 for trade in trades {
2024 if !self.instruments.contains_key(&trade.market_id) {
2025 log::debug!(
2026 "No instrument cached for Lighter account trade market_id={}",
2027 trade.market_id,
2028 );
2029 continue;
2030 }
2031
2032 if trade.bid_account_id != account_index && trade.ask_account_id != account_index {
2036 continue;
2037 }
2038
2039 reports.push(ExecutionReport::Fill(trade.clone()));
2040 }
2041
2042 if reports.is_empty() {
2043 Vec::new()
2044 } else {
2045 vec![NautilusWsMessage::ExecutionReports(reports)]
2046 }
2047 }
2048
2049 fn handle_account_positions(
2050 &self,
2051 positions: &AHashMap<Ustr, LighterPosition>,
2052 ts_init: UnixNanos,
2053 frame_type: PositionFrameType,
2054 ) -> Vec<NautilusWsMessage> {
2055 let Some((account_id, _)) = self.exec_account else {
2056 log::debug!("Lighter account_positions frame skipped: no execution context set");
2057 return Vec::new();
2058 };
2059
2060 let ts_event = ts_init;
2063
2064 let mut reports = Vec::new();
2065 let mut skipped_market_ids = Vec::new();
2066 let mut closed_market_ids = Vec::new();
2067
2068 for position in positions.values() {
2069 if position.position.is_zero() {
2070 if matches!(frame_type, PositionFrameType::Update) {
2071 closed_market_ids.push(position.market_id);
2072 }
2073 continue;
2074 }
2075
2076 let Some(instrument) = self.instruments.get(&position.market_id) else {
2077 log::debug!(
2078 "No instrument cached for Lighter position market_id={}",
2079 position.market_id,
2080 );
2081
2082 skipped_market_ids.push(position.market_id);
2083 continue;
2084 };
2085
2086 match parse_ws_position_status_report(
2087 position, instrument, account_id, ts_event, ts_init,
2088 ) {
2089 Ok(report) => reports.push(report),
2090 Err(e) => {
2091 skipped_market_ids.push(position.market_id);
2092 log::error!("Error parsing Lighter position status report: {e}");
2093 }
2094 }
2095 }
2096
2097 match frame_type {
2098 PositionFrameType::Snapshot => {
2099 vec![NautilusWsMessage::PositionSnapshot {
2101 reports,
2102 skipped_market_ids,
2103 }]
2104 }
2105 PositionFrameType::Update => vec![NautilusWsMessage::PositionUpdate {
2106 reports,
2107 closed_market_ids,
2108 skipped_market_ids,
2109 }],
2110 }
2111 }
2112
2113 fn handle_account_assets(
2114 &self,
2115 assets: &AHashMap<Ustr, LighterAsset>,
2116 timestamp_ms: u64,
2117 ts_init: UnixNanos,
2118 ) -> Vec<NautilusWsMessage> {
2119 let Some((account_id, _)) = self.exec_account else {
2120 log::debug!("Lighter account_assets frame skipped: no execution context set");
2121 return Vec::new();
2122 };
2123
2124 let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2125 Ok(ts) => ts,
2126 Err(e) => {
2127 log::error!("Invalid Lighter account_assets timestamp {timestamp_ms}: {e}");
2128 return Vec::new();
2129 }
2130 };
2131
2132 self.account_state_reconciler.update_assets(assets);
2133 self.emit_unified_account_state(account_id, ts_event, ts_init)
2134 }
2135
2136 fn handle_user_stats(
2137 &self,
2138 stats: &LighterUserStats,
2139 timestamp_ms: u64,
2140 ts_init: UnixNanos,
2141 ) -> Vec<NautilusWsMessage> {
2142 let Some((account_id, _)) = self.exec_account else {
2143 log::debug!("Lighter user_stats frame skipped: no execution context set");
2144 return Vec::new();
2145 };
2146
2147 let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2148 Ok(ts) => ts,
2149 Err(e) => {
2150 log::error!("Invalid Lighter user_stats timestamp {timestamp_ms}: {e}");
2151 return Vec::new();
2152 }
2153 };
2154
2155 self.account_state_reconciler.update_user_stats(stats);
2156 self.emit_unified_account_state(account_id, ts_event, ts_init)
2157 }
2158
2159 fn emit_unified_account_state(
2164 &self,
2165 account_id: AccountId,
2166 ts_event: UnixNanos,
2167 ts_init: UnixNanos,
2168 ) -> Vec<NautilusWsMessage> {
2169 match self
2170 .account_state_reconciler
2171 .build_state(account_id, ts_event, ts_init)
2172 {
2173 Some(Ok(state)) => vec![NautilusWsMessage::AccountState(Box::new(state))],
2174 Some(Err(e)) => {
2175 log::error!("Error building unified Lighter account state: {e}");
2176 Vec::new()
2177 }
2178 None => Vec::new(),
2179 }
2180 }
2181}
2182
2183#[derive(Clone, Copy)]
2184enum PositionFrameType {
2185 Snapshot,
2186 Update,
2187}
2188
2189fn raw_message(frame: &LighterWsFrame) -> Vec<NautilusWsMessage> {
2190 let value = serde_json::to_value(frame).unwrap_or(serde_json::Value::Null);
2191 vec![NautilusWsMessage::Raw(value)]
2192}
2193
2194fn is_sendtx_error_code(code: Option<u64>) -> bool {
2197 code.is_some_and(|c| LIGHTER_ERROR_CODE_TX_RANGE.contains(&c))
2198}
2199
2200fn already_subscribed_error(value: &serde_json::Value) -> Option<&serde_json::Value> {
2201 if value.get("code").and_then(|code| code.as_u64())
2202 == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2203 {
2204 Some(value)
2205 } else {
2206 value.get("error").filter(|e| {
2207 e.get("code").and_then(|code| code.as_u64())
2208 == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2209 })
2210 }
2211}
2212
2213fn subscription_error_code(value: &serde_json::Value) -> Option<u64> {
2214 value
2215 .get("code")
2216 .and_then(|code| code.as_u64())
2217 .or_else(|| {
2218 value
2219 .get("error")
2220 .and_then(|e| e.get("code"))
2221 .and_then(|code| code.as_u64())
2222 })
2223}
2224
2225fn typed_subscribe_topic(text: &str) -> Option<&str> {
2226 let header = serde_json::from_str::<LighterWsFrameHeader<'_>>(text).ok()?;
2227 header
2228 .kind
2229 .starts_with("subscribed/")
2230 .then_some(header.channel?)
2231}
2232
2233fn send_tx_rejected_from_value(
2236 value: &serde_json::Value,
2237 source: SendTxRejectionSource,
2238) -> NautilusWsMessage {
2239 let code = value.get("code").and_then(|v| v.as_i64());
2240 let message = value
2241 .get("message")
2242 .and_then(|v| v.as_str())
2243 .unwrap_or("")
2244 .to_string();
2245 let tx_hash = value
2246 .get("tx_hash")
2247 .and_then(|v| v.as_str())
2248 .map(str::to_string);
2249 NautilusWsMessage::SendTxRejected {
2250 connection_epoch: 0,
2251 source,
2252 code,
2253 message,
2254 tx_hash,
2255 }
2256}
2257
2258fn send_tx_rejected_from_nested_error(
2261 error: &serde_json::Value,
2262 source: SendTxRejectionSource,
2263) -> NautilusWsMessage {
2264 let code = error.get("code").and_then(|v| v.as_i64());
2265 let message = error
2266 .get("message")
2267 .and_then(|v| v.as_str())
2268 .unwrap_or("")
2269 .to_string();
2270 NautilusWsMessage::SendTxRejected {
2271 connection_epoch: 0,
2272 source,
2273 code,
2274 message,
2275 tx_hash: None,
2276 }
2277}
2278
2279fn log_integrator_not_approved() {
2280 log::error!(
2281 "Lighter venue rejected with code {LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED} \
2282 'integrator is not approved'.\n\
2283 Tagged orders require Nautilus integrator approval. \
2284 See: {LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL}",
2285 );
2286}
2287
2288fn market_index_from_topic(topic: &str) -> Option<i64> {
2289 let (_, rest) = topic.split_once(':')?;
2290 rest.parse::<i64>().ok()
2291}
2292
2293fn candle_market_and_resolution_from_topic(topic: &str) -> Option<(i64, LighterCandleResolution)> {
2294 let (channel, rest) = topic.split_once(':')?;
2295 if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::Candle) {
2296 return None;
2297 }
2298 let (market, res) = rest.split_once(':')?;
2299 let market_index = market.parse::<i64>().ok()?;
2300 let resolution = res.parse::<LighterCandleResolution>().ok()?;
2301 Some((market_index, resolution))
2302}
2303
2304fn order_book_market_index_from_topic(topic: &str) -> Option<i64> {
2305 let (channel, rest) = topic.split_once(':')?;
2306 if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::OrderBook) {
2307 return None;
2308 }
2309
2310 rest.parse::<i64>().ok()
2311}
2312
2313pub(crate) fn should_retry_lighter_ws_error(error: &LighterWsError) -> bool {
2314 match error {
2315 LighterWsError::Network(_) => true,
2316 LighterWsError::Transport(send_error) => match send_error {
2321 SendError::Timeout => true,
2322 SendError::InvalidInput(_)
2323 | SendError::Closed
2324 | SendError::ConnectionChanged
2325 | SendError::BrokenPipe(_)
2326 | SendError::WriteTimeout => false,
2327 },
2328 LighterWsError::Authentication(_)
2329 | LighterWsError::Parse(_)
2330 | LighterWsError::Client(_)
2331 | LighterWsError::SendTxOutcomeUnknown(_) => false,
2332 }
2333}
2334
2335pub(crate) fn create_lighter_ws_timeout_error(_msg: String) -> LighterWsError {
2336 LighterWsError::Transport(SendError::Timeout)
2339}
2340
2341#[cfg(test)]
2342mod tests {
2343 use std::time::Duration;
2344
2345 use log::{Level, LevelFilter, Log, Metadata, Record};
2346 use nautilus_model::{
2347 enums::{AccountType, BookAction, RecordFlag},
2348 identifiers::{InstrumentId, Symbol, Venue},
2349 instruments::{CryptoPerpetual, CurrencyPair},
2350 types::{Currency, Money, Price, Quantity},
2351 };
2352 use parking_lot::Mutex;
2353 use rstest::rstest;
2354 use rust_decimal::Decimal;
2355 use serde_json::json;
2356
2357 use super::*;
2358 use crate::{
2359 common::enums::{LighterCandleResolution, LighterTxType},
2360 websocket::messages::{LighterMarketSelection, LighterWsCandle, LighterWsChannel},
2361 };
2362
2363 const SECRET_MARKER: &str = "426426426";
2364
2365 struct OutboundLogCapture {
2366 messages: Mutex<Vec<String>>,
2367 }
2368
2369 static OUTBOUND_LOG_CAPTURE: OutboundLogCapture = OutboundLogCapture {
2370 messages: Mutex::new(Vec::new()),
2371 };
2372
2373 impl OutboundLogCapture {
2374 fn clear(&self) {
2375 self.messages.lock().clear();
2376 }
2377
2378 fn messages(&self) -> Vec<String> {
2379 self.messages.lock().clone()
2380 }
2381 }
2382
2383 impl Log for OutboundLogCapture {
2384 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
2385 metadata.level() == Level::Debug
2386 && metadata.target() == "nautilus_lighter::websocket::handler"
2387 }
2388
2389 fn log(&self, record: &Record<'_>) {
2390 if self.enabled(record.metadata()) {
2391 let message = record.args().to_string();
2392 if message.starts_with("Sending Lighter unsubscribe") {
2393 self.messages.lock().push(message);
2394 }
2395 }
2396 }
2397
2398 fn flush(&self) {}
2399 }
2400
2401 const WS_ACCOUNT_ORDERS_UPDATE: &str =
2402 include_str!("../../test_data/ws_account_orders_update.json");
2403 const WS_ACCOUNT_ALL_TRADES_UPDATE: &str =
2404 include_str!("../../test_data/ws_account_all_trades_update.json");
2405 const WS_ACCOUNT_ALL_POSITIONS_UPDATE: &str =
2406 include_str!("../../test_data/ws_account_all_positions_update.json");
2407 const WS_ACCOUNT_ALL_ASSETS_UPDATE: &str =
2408 include_str!("../../test_data/ws_account_all_assets_update.json");
2409 const WS_USER_STATS_UPDATE: &str = include_str!("../../test_data/ws_user_stats_update.json");
2410 const WS_ACCOUNT_ALL_ASSETS_WITH_POSITION: &str =
2411 include_str!("../../test_data/ws_account_all_assets_with_position.json");
2412 const WS_USER_STATS_WITH_POSITION: &str =
2413 include_str!("../../test_data/ws_user_stats_with_position.json");
2414 const WS_MARKET_STATS_UPDATE_SINGLE: &str =
2415 include_str!("../../test_data/ws_market_stats_update_single.json");
2416 const WS_MARKET_STATS_UPDATE_ALL: &str =
2417 include_str!("../../test_data/ws_market_stats_update_all.json");
2418 const WS_SPOT_MARKET_STATS_UPDATE_SINGLE: &str =
2419 include_str!("../../test_data/ws_spot_market_stats_update_single.json");
2420 const WS_SPOT_MARKET_STATS_UPDATE_ALL: &str =
2421 include_str!("../../test_data/ws_spot_market_stats_update_all.json");
2422 const WS_CANDLE_SUBSCRIBED: &str = include_str!("../../test_data/ws_candle_subscribed.json");
2423 const WS_SPOT_STATS_SUBSCRIBED_BAD_BODY: &str =
2424 include_str!("../../test_data/ws_spot_market_stats_subscribed_single_bad_body.json");
2425 const WS_BOOK_SUBSCRIBED_BAD_BODY: &str =
2426 include_str!("../../test_data/ws_order_book_subscribed_bad_body.json");
2427
2428 fn handle_control_text(
2429 handler: &mut FeedHandler,
2430 text: &str,
2431 ) -> (bool, Option<NautilusWsMessage>) {
2432 let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
2433 return (false, None);
2434 };
2435 handler.handle_control_value(&value)
2436 }
2437
2438 fn stub_eth_perp_instrument() -> InstrumentAny {
2439 let instrument_id = InstrumentId::new(Symbol::new("ETH-PERP"), Venue::new("LIGHTER"));
2440 InstrumentAny::CryptoPerpetual(
2441 CryptoPerpetual::builder()
2442 .instrument_id(instrument_id)
2443 .raw_symbol(Symbol::new("ETH-PERP"))
2444 .base_currency(Currency::from("ETH"))
2445 .quote_currency(Currency::from("USDC"))
2446 .settlement_currency(Currency::from("USDC"))
2447 .is_inverse(false)
2448 .price_precision(2)
2449 .size_precision(4)
2450 .price_increment(Price::from("0.01"))
2451 .size_increment(Quantity::from("0.0001"))
2452 .ts_event(UnixNanos::default())
2453 .ts_init(UnixNanos::default())
2454 .build()
2455 .unwrap(),
2456 )
2457 }
2458
2459 fn stub_eth_spot_instrument() -> InstrumentAny {
2460 let instrument_id = InstrumentId::new(Symbol::new("ETH-SPOT"), Venue::new("LIGHTER"));
2461 InstrumentAny::CurrencyPair(
2462 CurrencyPair::builder()
2463 .instrument_id(instrument_id)
2464 .raw_symbol(Symbol::new("ETH-SPOT"))
2465 .base_currency(Currency::from("ETH"))
2466 .quote_currency(Currency::from("USDC"))
2467 .price_precision(2)
2468 .size_precision(4)
2469 .price_increment(Price::from("0.01"))
2470 .size_increment(Quantity::from("0.0001"))
2471 .ts_event(UnixNanos::default())
2472 .ts_init(UnixNanos::default())
2473 .build()
2474 .unwrap(),
2475 )
2476 }
2477
2478 fn make_handler_with_account() -> FeedHandler {
2479 let signal = Arc::new(AtomicBool::new(false));
2480 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
2481 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
2482 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
2483 let mut handler =
2484 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2485 handler.instruments.insert(0, stub_eth_perp_instrument());
2486 handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
2487 handler
2488 }
2489
2490 #[rstest]
2491 #[case::cached_book(false)]
2492 #[case::queued_replacement(true)]
2493 #[tokio::test]
2494 async fn book_terminal_recovery_failure_clears_cache_and_suppresses_late_snapshot(
2495 #[case] queued: bool,
2496 ) {
2497 let mut handler = make_handler_with_account();
2498 handler.book.delta_subs.insert(0);
2499 let frame: LighterWsFrame = serde_json::from_str(include_str!(
2500 "../../test_data/ws_order_book_subscribed.json"
2501 ))
2502 .unwrap();
2503 assert_eq!(
2504 handler
2505 .handle_frame(frame.clone(), UnixNanos::from(1))
2506 .len(),
2507 1
2508 );
2509 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2510 handler.subscriptions.add_reference("order_book:0");
2511 let (completion, mut completed) = tokio::sync::oneshot::channel();
2512
2513 if queued {
2514 saturate_subscription_gate(&mut handler);
2515 handler.queue_book_replacement(
2516 0,
2517 BookWrite {
2518 cancel: recovery.cancellation.child_token(),
2519 gate: recovery.gate.clone(),
2520 completion,
2521 },
2522 );
2523 } else {
2524 drop(completion);
2525 }
2526
2527 handler
2528 .book
2529 .work
2530 .push(Box::pin(std::future::ready(BookWorkResult::Recovery {
2531 market_index: 0,
2532 recovery: recovery.clone(),
2533 result: Err(LighterWsError::Client("retry budget exhausted".into())),
2534 })));
2535
2536 assert!(handler.next().await.is_none());
2537 let messages = handler.handle_frame(frame, UnixNanos::from(2));
2538
2539 assert!(handler.book.recovery[&0].is_failed());
2540 assert!(recovery.cancellation.is_cancelled());
2541 assert!(!handler.book.states.contains_key(&0));
2542 assert!(!handler.book.snapshots_seen.contains(&0));
2543 assert!(messages.is_empty());
2544 assert!(!handler.book.writes.contains_key(&0));
2545 assert_eq!(
2546 completed.try_recv().unwrap_err(),
2547 tokio::sync::oneshot::error::TryRecvError::Closed
2548 );
2549
2550 handler.inflight_subs.clear();
2551 let (response, _result) = tokio::sync::oneshot::channel();
2552 handler.queue_subscribe(LighterWsChannel::OrderBook(0), None, Some(response));
2553 handler.pump_pending_subscribes().await;
2554
2555 assert!(!handler.book.recovery[&0].is_failed());
2556 assert!(handler.book.initial.contains_key(&0));
2557 assert!(!handler.book.initial[&0].cancel.is_cancelled());
2558 }
2559
2560 #[rstest]
2561 #[case::unconfirmed_permanent(false, false, false)]
2562 #[case::closed_gate_permanent(true, false, false)]
2563 #[case::confirmed_permanent(true, true, false)]
2564 #[case::unconfirmed_retry(false, false, true)]
2565 #[case::closed_gate_retry(true, false, true)]
2566 #[case::confirmed_retry(true, true, true)]
2567 fn book_venue_rejection_requires_confirmed_write(
2568 #[case] confirmed: bool,
2569 #[case] gate_open: bool,
2570 #[case] retry: bool,
2571 ) {
2572 let mut handler = make_handler_with_account();
2573 let (topic, generation) =
2574 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2575 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2576 assert!(recovery.begin_replacement());
2577
2578 if gate_open {
2579 recovery.gate.open();
2580 }
2581
2582 if !confirmed {
2583 handler.book.expected.remove(&0);
2584 }
2585
2586 if retry {
2587 handler.retry_inflight_subscriptions("venue rejection");
2588 } else {
2589 handler.fail_inflight_subscriptions("venue rejection");
2590 }
2591
2592 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
2593 assert!(!handler.book.recovery[&0].is_failed());
2594 assert!(!recovery.cancellation.is_cancelled());
2595 assert!(recovery.gate.lock().is_closed());
2596
2597 match &*recovery.outcome.borrow() {
2598 BookRecoveryOutcome::Pending => assert!(!confirmed || !gate_open),
2599 BookRecoveryOutcome::Rejected(LighterWsError::Network(message)) => {
2600 assert!(confirmed && gate_open && retry);
2601 assert_eq!(message, "venue rejection");
2602 }
2603 BookRecoveryOutcome::Rejected(LighterWsError::Client(message)) => {
2604 assert!(confirmed && gate_open && !retry);
2605 assert_eq!(message, "venue rejection");
2606 }
2607 outcome => panic!("unexpected recovery outcome: {outcome:?}"),
2608 }
2609 }
2610
2611 #[rstest]
2612 fn book_explicit_subscribe_restarts_after_failed_initial_subscription() {
2613 let mut handler = make_handler_with_account();
2614 handler.book.delta_subs.insert(0);
2615 handler.book.recovery.entry(0).or_default().fail(None);
2616 let (response, mut result) = tokio::sync::oneshot::channel();
2617 let (topic, generation) = mark_subscription_inflight(
2618 &mut handler,
2619 LighterWsChannel::OrderBook(0),
2620 Some(response),
2621 );
2622 assert!(handler.complete_typed_subscription(topic.as_str(), 0));
2623 let frame: LighterWsFrame = serde_json::from_str(include_str!(
2624 "../../test_data/ws_order_book_subscribed.json"
2625 ))
2626 .unwrap();
2627 let messages = handler.handle_frame(frame, UnixNanos::from(1));
2628
2629 assert_eq!(result.try_recv(), Ok(Ok(())));
2630 assert!(!handler.book.recovery[&0].is_failed());
2631 assert_eq!(handler.book.expected[&0], (generation, 0));
2632 assert_eq!(messages.len(), 1);
2633 assert!(handler.book.snapshots_seen.contains(&0));
2634 }
2635
2636 #[tokio::test]
2637 async fn book_queued_write_skips_after_last_reference_disappears() {
2638 let mut handler = make_handler_with_account();
2639 handler.book.delta_subs.insert(0);
2640 let channel = LighterWsChannel::OrderBook(0);
2641 handler.subscriptions.add_reference(&channel.topic_key());
2642 let (_, generation) = mark_subscription_inflight(&mut handler, channel.clone(), None);
2643 handler.send_book_subscribe(0, generation);
2644 assert!(handler.subscriptions.remove_reference(&channel.topic_key()));
2645
2646 let BookWorkResult::Sent { result, .. } = handler.book.work.next().await.unwrap() else {
2647 panic!("expected send completion");
2648 };
2649
2650 assert_eq!(
2651 result.unwrap_err().to_string(),
2652 "client error: book subscription cancelled"
2653 );
2654 assert!(handler.inner.is_none());
2655 }
2656
2657 #[rstest]
2658 fn book_wrong_epoch_snapshot_preserves_current_completion() {
2659 let mut handler = make_handler_with_account();
2660 let (tx, mut rx) = tokio::sync::oneshot::channel();
2661 let (topic, generation) =
2662 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2663 handler.book.expected.insert(0, (generation, 12));
2664
2665 assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
2666 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
2667 assert_eq!(
2668 rx.try_recv(),
2669 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2670 );
2671 assert!(handler.complete_typed_subscription(topic.as_str(), 12));
2672 assert_eq!(rx.try_recv(), Ok(Ok(())));
2673 assert!(!handler.inflight_subs.contains_key(&topic));
2674 }
2675
2676 #[rstest]
2677 fn book_control_ack_preserves_snapshot_but_not_successor_completion() {
2678 let mut handler = make_handler_with_account();
2679 let (topic, old_generation) =
2680 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2681 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2682 assert!(handler.complete_typed_subscription(topic.as_str(), 0));
2683 assert_eq!(handler.book.expected[&0], (old_generation, 0));
2684
2685 handler.subscriptions.mark_failure(topic.as_str());
2686 let (_, generation) =
2687 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2688 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2689 handler.subscriptions.mark_failure(topic.as_str());
2690 let (tx, mut rx) = tokio::sync::oneshot::channel();
2691 let (_, successor) =
2692 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2693
2694 assert_ne!(generation, successor);
2695 assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
2696 assert_eq!(handler.inflight_subs.get(&topic), Some(&successor));
2697 assert_eq!(
2698 rx.try_recv(),
2699 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2700 );
2701 assert!(handler.complete_typed_subscription(topic.as_str(), 0));
2702 assert_eq!(rx.try_recv(), Ok(Ok(())));
2703 }
2704
2705 #[rstest]
2706 fn book_old_epoch_does_not_consume_current_trailing_snapshot() {
2707 let mut handler = make_handler_with_account();
2708 let (topic, generation) =
2709 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2710 handler.book.expected.insert(0, (generation, 12));
2711 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2712
2713 assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
2714 assert_eq!(handler.book.trailing.get(&0), Some(&(generation, 12)));
2715 assert_eq!(
2716 handler.ignored_completions.get(&topic),
2717 Some(&CompletionKind::Typed)
2718 );
2719 assert!(handler.complete_typed_subscription(topic.as_str(), 12));
2720 assert!(!handler.book.trailing.contains_key(&0));
2721 assert!(!handler.ignored_completions.contains_key(&topic));
2722 }
2723
2724 #[rstest]
2725 fn book_ack_before_write_cannot_complete_subscription() {
2726 let mut handler = make_handler_with_account();
2727 let (topic, generation) =
2728 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2729 handler.book.expected.clear();
2730
2731 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2732 assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
2733 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
2734 }
2735
2736 #[rstest]
2737 fn book_empty_snapshot_replaces_cached_levels_for_both_consumers() {
2738 let mut handler = make_handler_with_account();
2739 handler.book.delta_subs.insert(0);
2740 handler.book.depth_subs.insert(0);
2741 let frame: LighterWsFrame = serde_json::from_str(include_str!(
2742 "../../test_data/ws_order_book_subscribed.json"
2743 ))
2744 .unwrap();
2745 assert_eq!(
2746 handler
2747 .handle_frame(frame.clone(), UnixNanos::from(1))
2748 .len(),
2749 2
2750 );
2751
2752 let LighterWsFrame::OrderBookSnapshot { order_book, .. } = &frame else {
2753 panic!("expected book snapshot");
2754 };
2755
2756 let mut empty = order_book.clone();
2757 empty.bids.clear();
2758 empty.asks.clear();
2759 empty.nonce += 1;
2760 let messages = handler.handle_order_book(
2761 Ustr::from("order_book:0"),
2762 &empty,
2763 123,
2764 true,
2765 UnixNanos::from(2),
2766 );
2767
2768 assert_eq!(messages.len(), 2);
2769
2770 let NautilusWsMessage::Deltas(deltas) = &messages[0] else {
2771 panic!("expected deltas");
2772 };
2773
2774 assert_eq!(deltas.deltas.len(), 1);
2775 assert_eq!(deltas.deltas[0].action, BookAction::Clear);
2776 assert_eq!(deltas.deltas[0].sequence, empty.nonce as u64);
2777 assert_eq!(
2778 deltas.deltas[0].flags,
2779 RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8
2780 );
2781
2782 let NautilusWsMessage::Depth(depth) = &messages[1] else {
2783 panic!("expected depth");
2784 };
2785
2786 assert!(depth.bids.is_empty());
2787 assert!(depth.asks.is_empty());
2788 assert!(depth.bid_counts.is_empty());
2789 assert!(depth.ask_counts.is_empty());
2790 assert!(handler.book.states[&0].book.bids.is_empty());
2791 assert!(handler.book.states[&0].book.asks.is_empty());
2792 }
2793
2794 #[tokio::test]
2795 async fn book_initial_deadline_is_cancelled_by_replacement_or_unsubscribe() {
2796 let mut handler = make_handler_with_account();
2797 handler.book.delta_subs.insert(0);
2798 let channel = LighterWsChannel::OrderBook(0);
2799 handler.subscriptions.add_reference(&channel.topic_key());
2800 let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
2801 let cancel = CancellationToken::new();
2802 handler.book.initial.insert(
2803 0,
2804 PendingSnapshot {
2805 deadline: None,
2806 cancel: cancel.clone(),
2807 gate: SnapshotGate::default(),
2808 },
2809 );
2810
2811 handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
2812 handler.book.cancel(0);
2813
2814 let BookWorkResult::Initial {
2815 cancel: completed,
2816 market_index,
2817 } = handler.book.work.next().await.unwrap()
2818 else {
2819 panic!("expected deadline completion");
2820 };
2821
2822 assert_eq!(market_index, 0);
2823 assert!(completed.is_cancelled());
2824 assert!(cancel.is_cancelled());
2825 assert!(handler.book.recovery.is_empty());
2826 assert!(handler.book.expected.is_empty());
2827 }
2828
2829 #[tokio::test]
2830 async fn book_initial_deadline_expires_without_cancellation() {
2831 let mut handler = make_handler_with_account();
2832 handler.book_snapshot_timeout = Duration::from_millis(50);
2833 handler.book.delta_subs.insert(0);
2834 let channel = LighterWsChannel::OrderBook(0);
2835 handler.subscriptions.add_reference(&channel.topic_key());
2836 let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
2837 let cancel = CancellationToken::new();
2838 handler.book.initial.insert(
2839 0,
2840 PendingSnapshot {
2841 deadline: None,
2842 cancel: cancel.clone(),
2843 gate: SnapshotGate::default(),
2844 },
2845 );
2846
2847 handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
2848
2849 let BookWorkResult::Initial {
2850 market_index,
2851 cancel: completed,
2852 } = handler.book.work.next().await.unwrap()
2853 else {
2854 panic!("expected deadline completion");
2855 };
2856
2857 assert_eq!(market_index, 0);
2858 assert!(!completed.is_cancelled());
2859 assert!(!cancel.is_cancelled());
2860 }
2861
2862 #[tokio::test]
2863 async fn book_reconnect_retains_recovery_owner_and_retires_old_writes() {
2864 let signal = Arc::new(AtomicBool::new(false));
2865 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2866 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
2867 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
2868 let mut handler =
2869 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2870 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2871 handler.book.expected.insert(0, (3, 9));
2872 raw_tx
2873 .send((10, Message::Text(RECONNECTED.into())))
2874 .unwrap();
2875 assert!(matches!(
2876 handler.next().await,
2877 Some(NautilusWsMessage::Reconnected {
2878 connection_epoch: 10
2879 })
2880 ));
2881
2882 assert!(Arc::ptr_eq(
2883 handler.book.recovery[&0].current().unwrap(),
2884 &recovery
2885 ));
2886 assert!(!recovery.cancellation.is_cancelled());
2887 assert!(recovery.gate.lock().is_closed());
2888 assert!(matches!(
2889 *recovery.outcome.borrow(),
2890 BookRecoveryOutcome::Rejected(LighterWsError::Network(_))
2891 ));
2892 assert!(handler.book.expected.is_empty());
2893 assert!(handler.book.writes.is_empty());
2894 }
2895
2896 #[rstest]
2897 fn book_recovery_is_scoped_to_handler_and_market() {
2898 let mut first = make_handler_with_account();
2899 let mut second = make_handler_with_account();
2900 let recovery = first.book.recovery.entry(0).or_default().claim().unwrap();
2901 let other_market = first.book.recovery.entry(1).or_default().claim().unwrap();
2902 let other_socket = second.book.recovery.entry(0).or_default().claim().unwrap();
2903 first.book.cancel(0);
2904
2905 assert!(recovery.cancellation.is_cancelled());
2906 assert!(!other_market.cancellation.is_cancelled());
2907 assert!(!other_socket.cancellation.is_cancelled());
2908 }
2909
2910 #[rstest]
2911 #[tokio::test]
2912 async fn test_outbound_unsubscribe_log_omits_payload_body() {
2913 log::set_logger(&OUTBOUND_LOG_CAPTURE).expect("test logger already installed");
2914 log::set_max_level(LevelFilter::Debug);
2915
2916 let handler = make_handler_with_account();
2917 let account_index = SECRET_MARKER.parse::<i64>().unwrap();
2918 let channel = LighterWsChannel::AccountAll(account_index);
2919 let payload_len = serde_json::to_string(&LighterWsRequest::unsubscribe(
2920 channel.subscription_channel(),
2921 ))
2922 .unwrap()
2923 .len();
2924 OUTBOUND_LOG_CAPTURE.clear();
2925
2926 handler.dispatch_unsubscribe(channel).await;
2927
2928 let messages = OUTBOUND_LOG_CAPTURE.messages();
2929
2930 assert!(
2931 messages
2932 .iter()
2933 .all(|message| !message.contains(SECRET_MARKER)),
2934 "outbound logs exposed the secret marker: {messages:?}"
2935 );
2936 assert!(
2937 messages.iter().any(|message| {
2938 message == &format!("Sending Lighter unsubscribe ({payload_len} bytes)")
2939 }),
2940 "unsubscribe metadata missing or inaccurate: {messages:?}"
2941 );
2942 }
2943
2944 fn mark_subscription_inflight(
2945 handler: &mut FeedHandler,
2946 channel: LighterWsChannel,
2947 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
2948 ) -> (Ustr, u64) {
2949 handler.queue_subscribe(channel, None, response_tx);
2950 let (topic, generation) = handler
2951 .pending_subs
2952 .pop_front()
2953 .expect("subscription was not queued");
2954 handler.inflight_subs.insert(topic, generation);
2955 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
2956 handler.book.expected.insert(market_index, (generation, 0));
2957 }
2958
2959 (topic, generation)
2960 }
2961
2962 fn saturate_subscription_gate(handler: &mut FeedHandler) {
2963 for i in 0..SUBSCRIBE_INFLIGHT_MAX {
2964 handler
2965 .inflight_subs
2966 .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
2967 }
2968 }
2969
2970 fn strip_account_marker(mut msgs: Vec<NautilusWsMessage>) -> Vec<NautilusWsMessage> {
2975 if matches!(
2976 msgs.last(),
2977 Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
2978 ) {
2979 msgs.pop();
2980 }
2981 msgs
2982 }
2983
2984 #[rstest]
2985 fn handle_frame_routes_account_orders_to_execution_reports() {
2986 let mut handler = make_handler_with_account();
2987 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
2988
2989 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
2990
2991 assert_eq!(messages.len(), 1);
2992 match &messages[0] {
2993 NautilusWsMessage::ExecutionReports(reports) => {
2994 assert_eq!(reports.len(), 1);
2995 match &reports[0] {
2996 super::ExecutionReport::Order(order) => {
2997 assert_eq!(order.order_id, "281476929510110");
2998 assert_eq!(order.client_order_id, "42");
2999 }
3000 other => panic!("expected order report, was {other:?}"),
3001 }
3002 }
3003 other => panic!("expected execution reports, was {other:?}"),
3004 }
3005 }
3006
3007 #[rstest]
3008 fn handle_frame_routes_account_trades_to_execution_reports() {
3009 let mut handler = make_handler_with_account();
3010 let frame: super::LighterWsFrame =
3011 serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3012
3013 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3014
3015 assert_eq!(messages.len(), 1);
3016 match &messages[0] {
3017 NautilusWsMessage::ExecutionReports(reports) => {
3018 assert_eq!(reports.len(), 1);
3019 match &reports[0] {
3020 super::ExecutionReport::Fill(fill) => {
3021 assert_eq!(fill.bid_id_str.as_deref(), Some("562947905631053"),);
3027 }
3028 other => panic!("expected fill report, was {other:?}"),
3029 }
3030 }
3031 other => panic!("expected execution reports, was {other:?}"),
3032 }
3033 }
3034
3035 #[rstest]
3036 fn handle_frame_routes_account_positions_to_update_without_readiness_marker() {
3037 let mut handler = make_handler_with_account();
3038 let frame: super::LighterWsFrame =
3039 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3040
3041 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3042
3043 assert_eq!(messages.len(), 1);
3044 match &messages[0] {
3045 NautilusWsMessage::PositionUpdate {
3046 reports,
3047 closed_market_ids,
3048 skipped_market_ids,
3049 } => {
3050 assert!(closed_market_ids.is_empty());
3051 assert!(skipped_market_ids.is_empty());
3052 assert_eq!(reports.len(), 1);
3053 assert_eq!(reports[0].quantity, Quantity::from("1.5000"));
3054 }
3055 other => panic!("expected position update, was {other:?}"),
3056 }
3057 }
3058
3059 #[rstest]
3060 fn handle_frame_tolerates_unknown_position_margin_mode() {
3061 let mut handler = make_handler_with_account();
3062 let mut frame_json: serde_json::Value =
3063 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3064 frame_json["positions"]["0"]["margin_mode"] = json!(99);
3065 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3066
3067 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3068
3069 assert_eq!(messages.len(), 1);
3070 match &messages[0] {
3071 NautilusWsMessage::PositionUpdate {
3072 reports,
3073 closed_market_ids,
3074 skipped_market_ids,
3075 } => {
3076 assert!(closed_market_ids.is_empty());
3077 assert!(skipped_market_ids.is_empty());
3078 assert_eq!(reports.len(), 1);
3079 }
3080 other => panic!("expected position update, was {other:?}"),
3081 }
3082 }
3083
3084 #[rstest]
3085 fn handle_frame_routes_empty_account_positions_to_empty_update() {
3086 let mut handler = make_handler_with_account();
3087 let frame_json = serde_json::json!({
3088 "type": "update/account_all_positions",
3089 "channel": "account_all_positions:1234",
3090 "positions": {},
3091 "shares": [],
3092 "last_funding_round": null,
3093 "last_funding_discount": null,
3094 });
3095 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3096
3097 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3098
3099 assert_eq!(messages.len(), 1);
3100 match &messages[0] {
3101 NautilusWsMessage::PositionUpdate {
3102 reports,
3103 closed_market_ids,
3104 skipped_market_ids,
3105 } => {
3106 assert!(closed_market_ids.is_empty());
3107 assert!(skipped_market_ids.is_empty());
3108 assert!(reports.is_empty());
3109 }
3110 other => panic!("expected empty position update, was {other:?}"),
3111 }
3112 }
3113
3114 #[rstest]
3115 fn handle_frame_routes_zero_account_position_to_closed_update() {
3116 let mut handler = make_handler_with_account();
3117 let mut frame_json: serde_json::Value =
3118 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3119 frame_json["positions"]["0"]["position"] = json!("0.0000");
3120 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3121
3122 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3123
3124 assert_eq!(messages.len(), 1);
3125 match &messages[0] {
3126 NautilusWsMessage::PositionUpdate {
3127 reports,
3128 closed_market_ids,
3129 skipped_market_ids,
3130 } => {
3131 assert!(reports.is_empty());
3132 assert!(skipped_market_ids.is_empty());
3133 assert_eq!(closed_market_ids, &[0]);
3134 }
3135 other => panic!("expected closed position update, was {other:?}"),
3136 }
3137 }
3138
3139 #[rstest]
3140 fn handle_frame_marks_account_positions_incomplete_when_position_instrument_uncached() {
3141 let mut handler = make_handler_with_account();
3142 let mut frame_json: serde_json::Value =
3143 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3144 frame_json["type"] = json!("subscribed/account_all_positions");
3145 frame_json["positions"]["0"]["market_id"] = json!(999);
3146 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3147
3148 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3149
3150 assert_eq!(messages.len(), 1);
3151 match &messages[0] {
3152 NautilusWsMessage::PositionSnapshot {
3153 reports,
3154 skipped_market_ids,
3155 } => {
3156 assert_eq!(skipped_market_ids, &[999]);
3157 assert!(reports.is_empty());
3158 }
3159 other => panic!("expected incomplete position snapshot, was {other:?}"),
3160 }
3161 }
3162
3163 #[rstest]
3164 fn handle_frame_marks_account_positions_incomplete_when_position_parse_fails() {
3165 let mut handler = make_handler_with_account();
3166 let mut frame_json: serde_json::Value =
3167 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3168 frame_json["type"] = json!("subscribed/account_all_positions");
3169 frame_json["positions"]["0"]["position"] = json!("-1.5000");
3170 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3171
3172 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3173
3174 assert_eq!(messages.len(), 1);
3175 match &messages[0] {
3176 NautilusWsMessage::PositionSnapshot {
3177 reports,
3178 skipped_market_ids,
3179 } => {
3180 assert_eq!(skipped_market_ids, &[0]);
3181 assert!(reports.is_empty());
3182 }
3183 other => panic!("expected incomplete position snapshot, was {other:?}"),
3184 }
3185 }
3186
3187 #[rstest]
3188 fn handle_frame_marks_position_update_incomplete_when_position_parse_fails() {
3189 let mut handler = make_handler_with_account();
3190 let mut frame_json: serde_json::Value =
3191 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3192 frame_json["positions"]["0"]["position"] = json!("-1.5000");
3193 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3194
3195 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3196
3197 assert_eq!(messages.len(), 1);
3198 match &messages[0] {
3199 NautilusWsMessage::PositionUpdate {
3200 reports,
3201 closed_market_ids,
3202 skipped_market_ids,
3203 } => {
3204 assert!(reports.is_empty());
3205 assert!(closed_market_ids.is_empty());
3206 assert_eq!(skipped_market_ids, &[0]);
3207 }
3208 other => panic!("expected incomplete position update, was {other:?}"),
3209 }
3210 }
3211
3212 #[rstest]
3213 fn handle_frame_routes_subscribed_account_all_positions_snapshot() {
3214 let mut handler = make_handler_with_account();
3215 let frame_json = serde_json::json!({
3216 "type": "subscribed/account_all_positions",
3217 "channel": "account_all_positions:1234",
3218 "positions": {
3219 "0": {
3220 "allocated_margin": "0.000000",
3221 "avg_entry_price": "0.111230",
3222 "initial_margin_fraction": "10.00",
3223 "liquidation_price": "0.100598",
3224 "margin_mode": 0,
3225 "market_id": 0,
3226 "open_order_count": 0,
3227 "pending_order_count": 0,
3228 "position": "100",
3229 "position_tied_order_count": 0,
3230 "position_value": "11.123000",
3231 "realized_pnl": "0.000000",
3232 "sign": 1,
3233 "symbol": "ETH",
3234 "total_discount": "0.000000",
3235 "total_funding_paid_out": "0.000000",
3236 "unrealized_pnl": "0.000000"
3237 }
3238 },
3239 });
3240 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3241
3242 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3243
3244 assert_eq!(messages.len(), 1);
3245 match &messages[0] {
3246 NautilusWsMessage::PositionSnapshot {
3247 reports,
3248 skipped_market_ids,
3249 } => {
3250 assert!(skipped_market_ids.is_empty());
3251 assert_eq!(reports.len(), 1);
3252 assert_eq!(reports[0].quantity, Quantity::from("100"));
3253 }
3254 other => panic!("expected position snapshot, was {other:?}"),
3255 }
3256 }
3257
3258 #[rstest]
3259 #[case::connected(serde_json::json!({"type": "connected", "session_id": "x"}), true, false)]
3260 #[case::ping(serde_json::json!({"type": "ping"}), true, false)]
3261 #[case::pong(serde_json::json!({"type": "pong"}), true, false)]
3262 #[case::send_tx_ack(
3263 serde_json::json!({"type": "jsonapi/sendtx", "code": 200, "tx_hash": "abc"}),
3264 true,
3265 true,
3266 )]
3267 #[case::send_tx_without_code(
3268 serde_json::json!({"type": "jsonapi/sendtx", "tx_hash": "abc"}),
3269 true,
3270 false,
3271 )]
3272 #[case::send_tx_with_nonnumeric_code(
3273 serde_json::json!({"type": "jsonapi/sendtx", "code": "200", "tx_hash": "abc"}),
3274 true,
3275 false,
3276 )]
3277 #[case::error_frame(
3278 serde_json::json!({"type": "error", "code": 21727, "message": "invalid client order index"}),
3279 true,
3280 true,
3281 )]
3282 #[case::error_frame_integrator_not_approved(
3283 serde_json::json!({"type": "error", "code": 21149, "message": "integrator is not approved"}),
3284 true,
3285 true,
3286 )]
3287 #[case::send_tx_ack_integrator_not_approved(
3288 serde_json::json!({"type": "jsonapi/sendtx", "code": 21149, "message": "integrator is not approved"}),
3289 true,
3290 true,
3291 )]
3292 #[case::wrapped_error_integrator_not_approved(
3293 serde_json::json!({"error": {"code": 21149, "message": "integrator is not approved"}}),
3294 true,
3295 true,
3296 )]
3297 #[case::subscription_error_frame(
3298 serde_json::json!({"type": "error", "code": 30003, "message": "Already Subscribed to : ticker:3"}),
3299 true,
3300 false,
3301 )]
3302 #[case::wrapped_subscription_error(
3303 serde_json::json!({"error": {"code": 30003, "message": "Already Subscribed to : ticker:3"}}),
3304 true,
3305 false,
3306 )]
3307 #[case::codeless_error_frame(
3308 serde_json::json!({"type": "error", "message": "unclassifiable"}),
3309 true,
3310 false,
3311 )]
3312 #[case::unknown_type(
3313 serde_json::json!({"type": "something_unexpected", "payload": "x"}),
3314 false,
3315 false,
3316 )]
3317 #[case::no_type_field(
3318 serde_json::json!({"error": {"code": 21702, "message": "invalid price"}}),
3319 true,
3320 true,
3321 )]
3322 fn handle_control_text_tri_state(
3323 #[case] payload: serde_json::Value,
3324 #[case] expected_matched: bool,
3325 #[case] expected_has_msg: bool,
3326 ) {
3327 let mut handler = make_handler_with_account();
3334 let text = payload.to_string();
3335 let (matched, msg) = handle_control_text(&mut handler, &text);
3336 assert_eq!(matched, expected_matched, "matched flag");
3337 assert_eq!(msg.is_some(), expected_has_msg, "msg presence");
3338 }
3339
3340 #[rstest]
3341 #[case::ack(serde_json::json!({"type":"jsonapi/sendtxbatch","id":"cancel-batch:abc","code":200,"tx_hash":["abc","def"]}), 200, "batch rejected", vec!["abc", "def"])]
3342 #[case::error(serde_json::json!({"id":"cancel-batch:abc","error":{"code":21104,"message":"invalid nonce"}}), 21104, "invalid nonce", vec![])]
3343 fn batch_response_preserves_correlation_and_epoch(
3344 #[case] value: serde_json::Value,
3345 #[case] expected_code: i64,
3346 #[case] expected_message: &str,
3347 #[case] expected_hashes: Vec<&str>,
3348 ) {
3349 let mut handler = make_handler_with_account();
3350 let (matched, message) = handler.handle_control_value(&value);
3351 assert!(matched);
3352
3353 match message.unwrap().with_connection_epoch(7) {
3354 NautilusWsMessage::SendTxBatchResult {
3355 connection_epoch,
3356 id,
3357 code,
3358 message,
3359 tx_hashes,
3360 } => {
3361 assert_eq!(connection_epoch, 7);
3362 assert_eq!(id, "cancel-batch:abc");
3363 assert_eq!(code, expected_code);
3364 assert_eq!(message, expected_message);
3365 assert_eq!(tx_hashes, expected_hashes);
3366 }
3367 message => panic!("expected batch response, was {message:?}"),
3368 }
3369 }
3370
3371 #[rstest]
3372 fn handle_control_text_sendtx_success_emits_typed_ack() {
3373 let mut handler = make_handler_with_account();
3374 let payload = serde_json::json!({
3375 "type": "jsonapi/sendtx",
3376 "code": 200,
3377 "tx_hash": "0000abcd",
3378 })
3379 .to_string();
3380
3381 let (_, msg) = handle_control_text(&mut handler, &payload);
3382
3383 match msg.expect("SendTxAck emitted") {
3384 NautilusWsMessage::SendTxAck { tx_hash, code, .. } => {
3385 assert_eq!(code, 200);
3386 assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3387 }
3388 other => panic!("expected SendTxAck, was {other:?}"),
3389 }
3390 }
3391
3392 #[rstest]
3393 fn handle_control_text_sendtx_failure_emits_ack_sourced_rejection() {
3394 let mut handler = make_handler_with_account();
3395 let payload = serde_json::json!({
3396 "type": "jsonapi/sendtx",
3397 "code": 21727,
3398 "message": "invalid client order index",
3399 })
3400 .to_string();
3401
3402 let (_, msg) = handle_control_text(&mut handler, &payload);
3403
3404 match msg.expect("SendTxRejected emitted") {
3405 NautilusWsMessage::SendTxRejected {
3406 source,
3407 code,
3408 message,
3409 tx_hash,
3410 ..
3411 } => {
3412 assert_eq!(source, SendTxRejectionSource::Ack);
3413 assert_eq!(code, Some(21727));
3414 assert_eq!(message, "invalid client order index");
3415 assert_eq!(tx_hash, None);
3416 }
3417 other => panic!("expected SendTxRejected, was {other:?}"),
3418 }
3419 }
3420
3421 #[rstest]
3422 fn handle_control_text_sendtx_failure_carries_echoed_tx_hash() {
3423 let mut handler = make_handler_with_account();
3424 let payload = serde_json::json!({
3425 "type": "jsonapi/sendtx",
3426 "code": 21727,
3427 "message": "invalid client order index",
3428 "tx_hash": "0000abcd",
3429 })
3430 .to_string();
3431
3432 let (_, msg) = handle_control_text(&mut handler, &payload);
3433
3434 match msg.expect("SendTxRejected emitted") {
3435 NautilusWsMessage::SendTxRejected { tx_hash, .. } => {
3436 assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3437 }
3438 other => panic!("expected SendTxRejected, was {other:?}"),
3439 }
3440 }
3441
3442 #[rstest]
3443 fn handle_control_text_bare_error_frame_emits_bare_error_rejection() {
3444 let mut handler = make_handler_with_account();
3445 let payload = serde_json::json!({
3446 "type": "error",
3447 "code": 21702,
3448 "message": "invalid price",
3449 })
3450 .to_string();
3451
3452 let (_, msg) = handle_control_text(&mut handler, &payload);
3453
3454 match msg.expect("SendTxRejected emitted") {
3455 NautilusWsMessage::SendTxRejected {
3456 source,
3457 code,
3458 message,
3459 tx_hash,
3460 ..
3461 } => {
3462 assert_eq!(source, SendTxRejectionSource::BareError);
3463 assert_eq!(code, Some(21702));
3464 assert_eq!(message, "invalid price");
3465 assert_eq!(tx_hash, None);
3466 }
3467 other => panic!("expected SendTxRejected, was {other:?}"),
3468 }
3469 }
3470
3471 #[rstest]
3472 fn handle_control_text_wrapped_error_emits_bare_error_rejection() {
3473 let mut handler = make_handler_with_account();
3475 let payload = serde_json::json!({
3476 "error": {"code": 21149, "message": "integrator is not approved"},
3477 })
3478 .to_string();
3479
3480 let (_, msg) = handle_control_text(&mut handler, &payload);
3481
3482 match msg.expect("SendTxRejected emitted") {
3483 NautilusWsMessage::SendTxRejected {
3484 source,
3485 code,
3486 message,
3487 tx_hash,
3488 ..
3489 } => {
3490 assert_eq!(source, SendTxRejectionSource::BareError);
3491 assert_eq!(code, Some(21149));
3492 assert_eq!(message, "integrator is not approved");
3493 assert_eq!(tx_hash, None);
3494 }
3495 other => panic!("expected SendTxRejected, was {other:?}"),
3496 }
3497 }
3498
3499 #[rstest]
3500 fn handle_frame_emits_no_account_state_until_both_streams_seen() {
3501 let mut handler = make_handler_with_account();
3505 let assets_only: super::LighterWsFrame =
3506 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3507
3508 let messages = strip_account_marker(handler.handle_frame(assets_only, UnixNanos::from(11)));
3509
3510 assert!(
3511 messages.is_empty(),
3512 "expected no AccountState before user_stats arrives, received {messages:?}"
3513 );
3514 }
3515
3516 #[rstest]
3517 fn handle_frame_routes_account_assets_and_user_stats_to_unified_state() {
3518 let mut handler = make_handler_with_account();
3525 let assets_frame: super::LighterWsFrame =
3526 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3527 let user_stats_frame: super::LighterWsFrame =
3528 serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3529
3530 let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3531 let messages =
3532 strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3533
3534 assert_eq!(messages.len(), 1);
3535 match &messages[0] {
3536 NautilusWsMessage::AccountState(state) => {
3537 let usdc = Currency::get_or_create_crypto("USDC");
3538 assert_eq!(state.account_type, AccountType::Margin);
3539 assert_eq!(state.base_currency, None);
3540 assert_eq!(state.balances.len(), 1);
3541 assert_eq!(state.balances[0].currency, usdc);
3542 assert_eq!(state.balances[0].total, Money::from("50.000000 USDC"));
3545 assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3546 assert_eq!(state.balances[0].free, Money::from("50.000000 USDC"));
3547 assert_eq!(state.margins.len(), 1);
3548 assert_eq!(state.margins[0].currency, usdc);
3549 assert_eq!(state.margins[0].initial, Money::from("0 USDC"));
3550 assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3551 assert!(state.margins[0].instrument_id.is_none());
3552 assert!(state.is_reported);
3553 }
3554 other => panic!("expected account state, was {other:?}"),
3555 }
3556 }
3557
3558 #[rstest]
3559 fn handle_frame_unified_state_reflects_open_position() {
3560 let mut handler = make_handler_with_account();
3571 let assets_frame: super::LighterWsFrame =
3572 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_WITH_POSITION).unwrap();
3573 let user_stats_frame: super::LighterWsFrame =
3574 serde_json::from_str(WS_USER_STATS_WITH_POSITION).unwrap();
3575
3576 let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3577 let messages =
3578 strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3579
3580 assert_eq!(messages.len(), 1);
3581 match &messages[0] {
3582 NautilusWsMessage::AccountState(state) => {
3583 assert_eq!(state.account_type, AccountType::Margin);
3584 assert_eq!(state.base_currency, None);
3585 assert_eq!(state.balances.len(), 1);
3586 assert_eq!(state.balances[0].total, Money::from("49.99536956 USDC"));
3590 assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3591 assert_eq!(state.balances[0].free, Money::from("49.99536956 USDC"));
3592 assert_eq!(state.margins.len(), 1);
3593 assert_eq!(state.margins[0].initial, Money::from("0.82705500 USDC"));
3595 assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3596 assert!(state.margins[0].instrument_id.is_none());
3597 }
3598 other => panic!("expected account state, was {other:?}"),
3599 }
3600 }
3601
3602 #[rstest]
3603 fn handle_frame_emits_account_stream_first_frame_marker_per_variant() {
3604 let mut handler = make_handler_with_account();
3610 let orders_frame: super::LighterWsFrame =
3611 serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3612 let trades_frame: super::LighterWsFrame =
3613 serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3614 let positions_frame: super::LighterWsFrame = serde_json::from_value({
3615 let mut value: serde_json::Value =
3616 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3617 value["type"] = json!("subscribed/account_all_positions");
3618 value
3619 })
3620 .unwrap();
3621 let assets_frame: super::LighterWsFrame =
3622 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3623 let user_stats_frame: super::LighterWsFrame =
3624 serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3625
3626 let cases = [
3627 (orders_frame, AccountStream::Orders),
3628 (trades_frame, AccountStream::Trades),
3629 (positions_frame, AccountStream::Positions),
3630 (assets_frame, AccountStream::Assets),
3631 (user_stats_frame, AccountStream::UserStats),
3632 ];
3633
3634 for (frame, expected) in cases {
3635 let msgs = handler.handle_frame(frame, UnixNanos::from(11));
3636 let marker = msgs
3637 .iter()
3638 .find(|m| matches!(m, NautilusWsMessage::AccountStreamFirstFrame(_)))
3639 .unwrap_or_else(|| panic!("missing marker for {expected:?}"));
3640 match marker {
3641 NautilusWsMessage::AccountStreamFirstFrame(stream) => {
3642 assert_eq!(*stream, expected);
3643 }
3644 other => panic!("expected AccountStreamFirstFrame, was {other:?}"),
3645 }
3646 assert!(
3649 matches!(
3650 msgs.last(),
3651 Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3652 ),
3653 "marker must trail typed reports for {expected:?}",
3654 );
3655 }
3656 }
3657
3658 #[rstest]
3659 fn handle_frame_account_orders_without_context_falls_back_to_raw() {
3660 let signal = Arc::new(AtomicBool::new(false));
3661 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3662 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3663 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3664 let mut handler =
3665 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3666 handler.instruments.insert(0, stub_eth_perp_instrument());
3667 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3671 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3672
3673 assert_eq!(messages.len(), 1);
3674 match &messages[0] {
3675 NautilusWsMessage::Raw(value) => {
3676 assert_eq!(value["type"], "update/account_orders");
3677 }
3678 other => panic!("expected raw fallback, was {other:?}"),
3679 }
3680 }
3681
3682 #[rstest]
3683 fn handle_frame_account_orders_skips_unknown_market() {
3684 let signal = Arc::new(AtomicBool::new(false));
3690 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3691 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3692 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3693 let mut handler =
3694 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3695 handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
3696 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3699 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3700
3701 assert!(messages.is_empty());
3702 }
3703
3704 #[rstest]
3705 fn handle_frame_account_assets_invalid_timestamp_returns_empty() {
3706 let mut handler = make_handler_with_account();
3707 let frame_json = r#"{
3714 "type": "update/account_all_assets",
3715 "channel": "account_all_assets:1234",
3716 "timestamp": 18446744073709551615,
3717 "assets": {
3718 "0": {
3719 "symbol": "USDC",
3720 "asset_id": 0,
3721 "balance": "100.000000",
3722 "locked_balance": "1.000000"
3723 }
3724 }
3725 }"#;
3726 let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
3727
3728 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3729
3730 assert!(messages.is_empty());
3731 }
3732
3733 #[rstest]
3734 fn handle_frame_account_all_orders_routes_to_execution_reports() {
3735 let mut handler = make_handler_with_account();
3739 let frame_json = r#"{
3740 "type": "update/account_all_orders",
3741 "channel": "account_all_orders:1234",
3742 "orders": {
3743 "0": [{
3744 "order_index": 281476929510110,
3745 "client_order_index": 42,
3746 "order_id": "281476929510110",
3747 "client_order_id": "42",
3748 "market_index": 0,
3749 "owner_account_index": 1234,
3750 "initial_base_amount": "0.0050",
3751 "price": "2352.74",
3752 "nonce": 9182390020,
3753 "remaining_base_amount": "0.0050",
3754 "is_ask": true,
3755 "base_size": 50,
3756 "base_price": 235274,
3757 "filled_base_amount": "0.0000",
3758 "filled_quote_amount": "0.000000",
3759 "side": "sell",
3760 "type": "limit",
3761 "time_in_force": "good-till-time",
3762 "reduce_only": false,
3763 "trigger_price": "0.00",
3764 "order_expiry": 1780360584479,
3765 "status": "open",
3766 "trigger_status": "na",
3767 "trigger_time": 0,
3768 "parent_order_index": 0,
3769 "parent_order_id": "0",
3770 "to_trigger_order_id_0": "0",
3771 "to_trigger_order_id_1": "0",
3772 "to_cancel_order_id_0": "0",
3773 "integrator_fee_collector_index": "0",
3774 "integrator_taker_fee": "0",
3775 "integrator_maker_fee": "0",
3776 "block_height": 227535532,
3777 "timestamp": 1777941383576,
3778 "created_at": 1777941383576,
3779 "updated_at": 1777941383576,
3780 "transaction_time": 1777941383576735
3781 }]
3782 }
3783 }"#;
3784 let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
3785
3786 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3787
3788 assert_eq!(messages.len(), 1);
3789 match &messages[0] {
3790 NautilusWsMessage::ExecutionReports(reports) => {
3791 assert_eq!(reports.len(), 1);
3792 match &reports[0] {
3793 super::ExecutionReport::Order(order) => {
3794 assert_eq!(order.order_id, "281476929510110");
3795 }
3796 other => panic!("expected order report, was {other:?}"),
3797 }
3798 }
3799 other => panic!("expected execution reports, was {other:?}"),
3800 }
3801 }
3802
3803 fn snapshot_trade_frame_json() -> &'static str {
3804 r#"{
3805 "type": "subscribed/account_all_trades",
3806 "channel": "account_all_trades:1234",
3807 "trades": [{
3808 "trade_id": 19209006902,
3809 "trade_id_str": "19209006902",
3810 "tx_hash": "000000128b1ee814",
3811 "type": "trade",
3812 "market_id": 0,
3813 "size": "0.1336",
3814 "price": "2352.73",
3815 "usd_amount": "314.324728",
3816 "ask_id": 281476929510102,
3817 "bid_id": 562947905631053,
3818 "ask_client_id": 0,
3819 "bid_client_id": 7001011966,
3820 "ask_account_id": 91249,
3821 "bid_account_id": 1234,
3822 "is_maker_ask": true,
3823 "block_height": 227535535,
3824 "timestamp": 1777941384181,
3825 "transaction_time": 1777941384181586
3826 }],
3827 "total_volume": "100.0",
3828 "monthly_volume": "100.0",
3829 "weekly_volume": "100.0",
3830 "daily_volume": "100.0"
3831 }"#
3832 }
3833
3834 #[rstest]
3835 fn handle_frame_account_all_trades_snapshot_is_dropped_with_context() {
3836 let mut handler = make_handler_with_account();
3837 let frame: super::LighterWsFrame =
3842 serde_json::from_str(snapshot_trade_frame_json()).unwrap();
3843
3844 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3845
3846 assert!(messages.is_empty());
3847 }
3848
3849 #[rstest]
3850 fn handle_frame_account_all_trades_snapshot_falls_back_to_raw_without_context() {
3851 let signal = Arc::new(AtomicBool::new(false));
3852 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3853 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3854 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3855 let mut handler =
3856 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3857 handler.instruments.insert(0, stub_eth_perp_instrument());
3858 let frame: super::LighterWsFrame =
3862 serde_json::from_str(snapshot_trade_frame_json()).unwrap();
3863 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3864
3865 assert_eq!(messages.len(), 1);
3866 match &messages[0] {
3867 NautilusWsMessage::Raw(value) => {
3868 assert_eq!(value["type"], "subscribed/account_all_trades");
3869 }
3870 other => panic!("expected raw fallback, was {other:?}"),
3871 }
3872 }
3873
3874 #[rstest]
3875 fn handle_frame_market_stats_emits_mark_index_and_funding_updates() {
3876 let mut handler = make_handler_with_account();
3877 let frame: super::LighterWsFrame =
3878 serde_json::from_str(WS_MARKET_STATS_UPDATE_SINGLE).unwrap();
3879
3880 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3881
3882 assert_eq!(messages.len(), 3);
3883 match &messages[0] {
3884 NautilusWsMessage::MarkPrice(update) => {
3885 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
3886 assert_eq!(update.value, Price::from("2064.47"));
3887 assert_eq!(update.ts_event, UnixNanos::from(1_774_883_844_933_000_000));
3888 }
3889 event => panic!("expected mark price update, was {event:?}"),
3890 }
3891
3892 match &messages[1] {
3893 NautilusWsMessage::IndexPrice(update) => {
3894 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
3895 assert_eq!(update.value, Price::from("2064.48"));
3896 }
3897 event => panic!("expected index price update, was {event:?}"),
3898 }
3899
3900 match &messages[2] {
3901 NautilusWsMessage::FundingRate(update) => {
3902 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
3903 assert_eq!(update.rate, Decimal::new(1, 6));
3904 assert_eq!(update.next_funding_ns, None);
3905 }
3906 event => panic!("expected funding rate update, was {event:?}"),
3907 }
3908 }
3909
3910 #[rstest]
3911 fn handle_frame_market_stats_all_emits_mark_index_and_funding_updates() {
3912 let mut handler = make_handler_with_account();
3913 let frame: super::LighterWsFrame =
3914 serde_json::from_str(WS_MARKET_STATS_UPDATE_ALL).unwrap();
3915
3916 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3917
3918 assert_eq!(messages.len(), 3);
3919 assert!(matches!(&messages[0], NautilusWsMessage::MarkPrice(_)));
3920 assert!(matches!(&messages[1], NautilusWsMessage::IndexPrice(_)));
3921 assert!(matches!(&messages[2], NautilusWsMessage::FundingRate(_)));
3922
3923 match &messages[0] {
3924 NautilusWsMessage::MarkPrice(update) => {
3925 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
3926 assert_eq!(update.value, Price::from("2064.47"));
3927 }
3928 event => panic!("expected mark price update, was {event:?}"),
3929 }
3930
3931 match &messages[2] {
3932 NautilusWsMessage::FundingRate(update) => {
3933 assert_eq!(update.rate, Decimal::new(1, 6));
3934 assert_eq!(update.next_funding_ns, None);
3935 }
3936 event => panic!("expected funding rate update, was {event:?}"),
3937 }
3938 }
3939
3940 #[rstest]
3941 fn handle_frame_spot_market_stats_emits_index_update() {
3942 let signal = Arc::new(AtomicBool::new(false));
3943 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3944 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3945 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3946 let mut handler =
3947 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3948 handler.instruments.insert(2048, stub_eth_spot_instrument());
3949 let frame: super::LighterWsFrame =
3950 serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_SINGLE).unwrap();
3951
3952 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3953
3954 assert_eq!(messages.len(), 1);
3955 match &messages[0] {
3956 NautilusWsMessage::IndexPrice(update) => {
3957 assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
3958 assert_eq!(update.value, Price::from("1.00"));
3959 }
3960 event => panic!("expected spot index price update, was {event:?}"),
3961 }
3962 }
3963
3964 #[rstest]
3965 fn handle_frame_spot_market_stats_all_emits_index_update() {
3966 let signal = Arc::new(AtomicBool::new(false));
3967 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3968 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3969 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3970 let mut handler =
3971 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3972 handler.instruments.insert(2048, stub_eth_spot_instrument());
3973 let frame: super::LighterWsFrame =
3974 serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_ALL).unwrap();
3975
3976 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3977
3978 assert_eq!(messages.len(), 1);
3979 match &messages[0] {
3980 NautilusWsMessage::IndexPrice(update) => {
3981 assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
3982 assert_eq!(update.value, Price::from("1.00"));
3983 }
3984 event => panic!("expected spot index price update, was {event:?}"),
3985 }
3986 }
3987
3988 #[rstest]
3989 #[case(LighterWsChannel::OrderBook(0), "order_book:0", "order_book/0")]
3990 #[case(LighterWsChannel::Trade(7), "trade:7", "trade/7")]
3991 #[case(LighterWsChannel::Ticker(2), "ticker:2", "ticker/2")]
3992 #[case(LighterWsChannel::Height, "height", "height")]
3993 #[case(
3994 LighterWsChannel::MarketStats(LighterMarketSelection::All),
3995 "market_stats:all",
3996 "market_stats/all"
3997 )]
3998 #[case(
3999 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(2048)),
4000 "spot_market_stats:2048",
4001 "spot_market_stats/2048"
4002 )]
4003 #[case(
4004 LighterWsChannel::OrderBook(4095),
4005 "order_book:4095",
4006 "order_book/4095"
4007 )]
4008 #[case(
4009 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(50_000)),
4010 "spot_market_stats:50000",
4011 "spot_market_stats/50000"
4012 )]
4013 #[case(
4014 LighterWsChannel::Candle {
4015 market_index: 40_000,
4016 resolution: LighterCandleResolution::OneHour,
4017 },
4018 "candle:40000:1h",
4019 "candle/40000/1h"
4020 )]
4021 #[case(
4022 LighterWsChannel::AccountOrders { market_index: 0, account_index: 1234 },
4023 "account_orders:0:1234",
4024 "account_orders/0/1234",
4025 )]
4026 fn topic_and_subscription_round_trip(
4027 #[case] channel: LighterWsChannel,
4028 #[case] expected_topic: &str,
4029 #[case] expected_subscription: &str,
4030 ) {
4031 assert_eq!(channel.topic_key(), expected_topic);
4032 assert_eq!(channel.subscription_channel(), expected_subscription);
4033 }
4034
4035 #[rstest]
4036 #[case("order_book:0", Some(0))]
4037 #[case("trade:42", Some(42))]
4038 #[case("order_book:4095", Some(4095))]
4039 #[case("trade:40000", Some(40_000))]
4040 #[case("market_stats:50000", Some(50_000))]
4041 #[case("height", None)]
4042 #[case("malformed", None)]
4043 fn market_index_extraction(#[case] topic: &str, #[case] expected: Option<i64>) {
4044 assert_eq!(market_index_from_topic(topic), expected);
4045 }
4046
4047 #[rstest]
4048 #[case("order_book:0", Some(0))]
4049 #[case("order_book:42", Some(42))]
4050 #[case("order_book:40000", Some(40_000))]
4051 #[case("trade:42", None)]
4052 #[case("ticker:2", None)]
4053 #[case("market_stats:0", None)]
4054 #[case("height", None)]
4055 #[case("order_book:not-an-int", None)]
4056 fn order_book_market_index_only_matches_order_book_channel(
4057 #[case] topic: &str,
4058 #[case] expected: Option<i64>,
4059 ) {
4060 assert_eq!(order_book_market_index_from_topic(topic), expected);
4061 }
4062
4063 #[rstest]
4064 #[case(LighterWsChannel::AccountAll(1234), true)]
4065 #[case(LighterWsChannel::OrderBook(0), false)]
4066 #[case(LighterWsChannel::AccountAllPositions(1), true)]
4067 #[case(LighterWsChannel::Trade(0), false)]
4068 fn requires_auth_classification(#[case] channel: LighterWsChannel, #[case] expected: bool) {
4069 assert_eq!(channel.requires_auth(), expected);
4070 }
4071
4072 #[rstest]
4073 fn handler_command_subscribe_debug_redacts_auth_token() {
4074 let token = "schnorr-signature-bytes-do-not-leak";
4075 let cmd = HandlerCommand::Subscribe {
4076 channel: LighterWsChannel::AccountAll(1234),
4077 auth: Some(SecretString::from(token)),
4078 response_tx: None,
4079 };
4080
4081 let dbg = format!("{cmd:?}");
4082
4083 assert!(
4084 !dbg.contains(token),
4085 "Debug output must not contain the auth token, found: {dbg}",
4086 );
4087 assert!(dbg.contains(REDACTED));
4088 }
4089
4090 #[tokio::test]
4091 async fn send_tx_command_returns_handler_send_error_without_active_client() {
4092 let signal = Arc::new(AtomicBool::new(false));
4093 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4094 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4095 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4096 let mut handler =
4097 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4098 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4099
4100 cmd_tx
4101 .send(HandlerCommand::SendTx {
4102 tx_type: LighterTxType::CreateOrder as u8,
4103 tx_info: serde_json::value::RawValue::from_string(
4104 r#"{"AccountIndex":12345,"Nonce":42}"#.to_string(),
4105 )
4106 .unwrap(),
4107 connection_epoch: 0,
4108 response_tx,
4109 })
4110 .unwrap();
4111 drop(cmd_tx);
4112 drop(raw_tx);
4113
4114 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4115 .await
4116 .expect("timed out waiting for handler to drain command");
4117 let result = response_rx.await.expect("sendTx response channel closed");
4118
4119 assert!(next.is_none());
4120 let Err(LighterWsError::Client(message)) = result else {
4121 panic!("expected client send error, was {result:?}");
4122 };
4123 assert!(message.contains("no active WebSocket client"));
4124 }
4125
4126 #[tokio::test]
4127 async fn resubscribe_order_book_command_skips_when_reference_removed() {
4128 let signal = Arc::new(AtomicBool::new(false));
4129 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4130 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4131 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4132 let subscriptions = SubscriptionState::new(':');
4133 let topic = LighterWsChannel::OrderBook(0).topic_key();
4134 assert!(subscriptions.add_reference(&topic));
4135 assert!(subscriptions.remove_reference(&topic));
4136
4137 let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions.clone());
4138 handler.book.delta_subs.insert(0);
4139
4140 let (completion, result) = tokio::sync::oneshot::channel();
4141 cmd_tx
4142 .send(HandlerCommand::RecoverBook {
4143 market_index: 0,
4144 cancel: CancellationToken::new(),
4145 gate: SnapshotGate::default(),
4146 completion,
4147 })
4148 .expect("queue resync");
4149 drop(cmd_tx);
4150 drop(raw_tx);
4151
4152 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4153 .await
4154 .expect("timed out waiting for handler to drain command");
4155
4156 assert!(next.is_none());
4157 assert!(subscriptions.pending_subscribe_topics().is_empty());
4158 assert!(subscriptions.pending_unsubscribe_topics().is_empty());
4159 assert!(result.await.is_err());
4160 }
4161
4162 #[tokio::test]
4163 async fn resubscribe_order_book_queues_when_inflight_is_at_cap() {
4164 let signal = Arc::new(AtomicBool::new(false));
4165 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4166 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4167 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4168 let subscriptions = SubscriptionState::new(':');
4169 let topic = LighterWsChannel::OrderBook(0).topic_key();
4170 assert!(subscriptions.add_reference(&topic));
4171
4172 let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions);
4173 handler.book.delta_subs.insert(0);
4174 saturate_subscription_gate(&mut handler);
4175
4176 let (completion, _rx) = tokio::sync::oneshot::channel();
4177 handler.queue_book_replacement(
4178 0,
4179 BookWrite {
4180 cancel: CancellationToken::new(),
4181 gate: SnapshotGate::default(),
4182 completion,
4183 },
4184 );
4185
4186 handler.pump_pending_subscribes().await;
4187
4188 assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4189 assert!(
4190 handler
4191 .pending_subs
4192 .iter()
4193 .any(|(topic, _)| *topic == Ustr::from("order_book:0")),
4194 );
4195 }
4196
4197 fn stub_candle(
4198 t: i64,
4199 open: i64,
4200 high: i64,
4201 low: i64,
4202 close: i64,
4203 volume_ticks: i64,
4204 ) -> LighterWsCandle {
4205 LighterWsCandle {
4206 t,
4207 o: Decimal::new(open, 2),
4208 h: Decimal::new(high, 2),
4209 l: Decimal::new(low, 2),
4210 c: Decimal::new(close, 2),
4211 v: Decimal::new(volume_ticks, 4),
4212 quote_volume: Decimal::ZERO,
4213 i: 0,
4214 }
4215 }
4216
4217 fn candle_frame(channel: &str, candle: LighterWsCandle, is_snapshot: bool) -> LighterWsFrame {
4218 if is_snapshot {
4219 LighterWsFrame::CandleSnapshot {
4220 channel: Ustr::from(channel),
4221 candles: vec![candle],
4222 timestamp: 0,
4223 }
4224 } else {
4225 LighterWsFrame::Candle {
4226 channel: Ustr::from(channel),
4227 candles: vec![candle],
4228 timestamp: 0,
4229 }
4230 }
4231 }
4232
4233 #[rstest]
4234 fn handle_candles_first_observation_caches_without_emit() {
4235 let mut handler = make_handler_with_account();
4236 let frame = candle_frame(
4237 "candle:0:1m",
4238 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
4239 true,
4240 );
4241
4242 let messages = handler.handle_frame(frame, UnixNanos::from(99));
4243
4244 assert!(messages.is_empty(), "first observation must not emit");
4245 let key = (0_i64, LighterCandleResolution::OneMinute);
4246 assert_eq!(handler.last_candles.get(&key).map(|c| c.t), Some(1_000_000));
4247 }
4248
4249 #[rstest]
4250 fn handle_candles_t_advance_emits_bar_for_previous_candle() {
4251 let mut handler = make_handler_with_account();
4252 let prev = stub_candle(1_000_000, 10_000, 11_000, 9_900, 10_500, 10_000);
4253 let next = stub_candle(1_060_000, 10_500, 10_600, 10_450, 10_550, 20_000);
4254 let next_t = next.t;
4255 handler.handle_frame(candle_frame("candle:0:1m", prev, true), UnixNanos::from(1));
4256
4257 let messages =
4258 handler.handle_frame(candle_frame("candle:0:1m", next, false), UnixNanos::from(2));
4259
4260 assert_eq!(messages.len(), 1);
4261 match &messages[0] {
4262 NautilusWsMessage::Bar(bar) => {
4263 assert_eq!(bar.open, Price::from("100.00"));
4265 assert_eq!(bar.high, Price::from("110.00"));
4266 assert_eq!(bar.low, Price::from("99.00"));
4267 assert_eq!(bar.close, Price::from("105.00"));
4268 assert_eq!(bar.volume, Quantity::from("1.0000"));
4269 assert_eq!(bar.ts_event, UnixNanos::from(1_000_000 * 1_000_000));
4270 }
4271 other => panic!("expected Bar message, was {other:?}"),
4272 }
4273 let cached = handler
4274 .last_candles
4275 .get(&(0_i64, LighterCandleResolution::OneMinute))
4276 .expect("cache populated");
4277 assert_eq!(cached.t, next_t);
4278 }
4279
4280 #[rstest]
4281 fn handle_candles_same_t_updates_cache_without_emit() {
4282 let mut handler = make_handler_with_account();
4283 let initial = stub_candle(1_000_000, 10_000, 10_050, 9_950, 10_025, 5_000);
4284 let same_t_updated = stub_candle(1_000_000, 10_000, 10_100, 9_950, 10_075, 7_500);
4285 let same_t_h = same_t_updated.h;
4286 let same_t_c = same_t_updated.c;
4287 handler.handle_frame(
4288 candle_frame("candle:0:1m", initial, true),
4289 UnixNanos::from(1),
4290 );
4291
4292 let messages = handler.handle_frame(
4293 candle_frame("candle:0:1m", same_t_updated, false),
4294 UnixNanos::from(2),
4295 );
4296
4297 assert!(messages.is_empty(), "same-`t` update must not emit");
4298 let cached = handler
4299 .last_candles
4300 .get(&(0_i64, LighterCandleResolution::OneMinute))
4301 .expect("cache populated");
4302 assert_eq!(cached.h, same_t_h);
4303 assert_eq!(cached.c, same_t_c);
4304 }
4305
4306 #[rstest]
4307 fn handle_candles_regressed_t_is_skipped() {
4308 let mut handler = make_handler_with_account();
4309 let initial = stub_candle(2_000_000, 10_000, 10_000, 10_000, 10_000, 5_000);
4310 let regressed = stub_candle(1_000_000, 9_000, 9_000, 9_000, 9_000, 5_000);
4311 let initial_t = initial.t;
4312 handler.handle_frame(
4313 candle_frame("candle:0:1m", initial, true),
4314 UnixNanos::from(1),
4315 );
4316
4317 let messages = handler.handle_frame(
4318 candle_frame("candle:0:1m", regressed, false),
4319 UnixNanos::from(2),
4320 );
4321
4322 assert!(messages.is_empty(), "regressed `t` must not emit");
4323 let cached = handler
4324 .last_candles
4325 .get(&(0_i64, LighterCandleResolution::OneMinute))
4326 .expect("cache populated");
4327 assert_eq!(cached.t, initial_t);
4329 }
4330
4331 #[rstest]
4332 fn handle_candles_unknown_market_returns_empty() {
4333 let mut handler = make_handler_with_account();
4334 let frame = candle_frame(
4335 "candle:99:1m",
4336 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 5_000),
4337 true,
4338 );
4339
4340 let messages = handler.handle_frame(frame, UnixNanos::from(1));
4341
4342 assert!(messages.is_empty());
4343 }
4344
4345 #[rstest]
4346 fn handle_unsubscribe_ack_clears_only_matching_candle_key() {
4347 let mut handler = make_handler_with_account();
4348 handler.last_candles.insert(
4349 (0, LighterCandleResolution::OneMinute),
4350 stub_candle(1, 0, 0, 0, 0, 0),
4351 );
4352 handler.last_candles.insert(
4353 (0, LighterCandleResolution::FiveMinute),
4354 stub_candle(2, 0, 0, 0, 0, 0),
4355 );
4356 handler.subscriptions.mark_unsubscribe("candle:0:1m");
4357
4358 let payload = json!({"type": "unsubscribed", "channel": "candle:0:1m"});
4359 let (matched, _) = handle_control_text(&mut handler, &payload.to_string());
4360
4361 assert!(matched);
4362 assert!(
4363 handler
4364 .last_candles
4365 .get(&(0, LighterCandleResolution::OneMinute))
4366 .is_none(),
4367 );
4368 assert!(
4369 handler
4370 .last_candles
4371 .get(&(0, LighterCandleResolution::FiveMinute))
4372 .is_some(),
4373 );
4374 }
4375
4376 #[rstest]
4377 #[case::well_formed("candle:0:1m", Some((0, LighterCandleResolution::OneMinute)))]
4378 #[case::weekly("candle:3:1w", Some((3, LighterCandleResolution::OneWeek)))]
4379 #[case::widened_id("candle:40000:1h", Some((40_000, LighterCandleResolution::OneHour)))]
4380 #[case::other_kind("order_book:0", None)]
4381 #[case::missing_resolution("candle:0", None)]
4382 #[case::bad_market("candle:notanint:1m", None)]
4383 #[case::bad_resolution("candle:0:bogus", None)]
4384 fn test_candle_market_and_resolution_from_topic(
4385 #[case] topic: &str,
4386 #[case] expected: Option<(i64, LighterCandleResolution)>,
4387 ) {
4388 assert_eq!(candle_market_and_resolution_from_topic(topic), expected);
4389 }
4390
4391 #[rstest]
4392 #[case::network_retries(LighterWsError::Network("disconnected".into()), true)]
4393 #[case::auth_does_not_retry(LighterWsError::Authentication("bad token".into()), false)]
4394 #[case::parse_does_not_retry(LighterWsError::Parse("bad json".into()), false)]
4395 #[case::client_does_not_retry(LighterWsError::Client("no active WebSocket client".into()), false)]
4396 #[case::transport_closed_does_not_retry(LighterWsError::Transport(SendError::Closed), false)]
4397 #[case::transport_invalid_input_does_not_retry(
4398 LighterWsError::Transport(SendError::InvalidInput("pong payload too large".into())),
4399 false
4400 )]
4401 #[case::transport_connection_changed_does_not_retry(
4402 LighterWsError::Transport(SendError::ConnectionChanged),
4403 false
4404 )]
4405 #[case::transport_timeout_retries(LighterWsError::Transport(SendError::Timeout), true)]
4406 #[case::transport_write_timeout_does_not_retry(
4407 LighterWsError::Transport(SendError::WriteTimeout),
4408 false
4409 )]
4410 #[case::transport_broken_pipe_does_not_retry(
4411 LighterWsError::Transport(SendError::BrokenPipe(
4412 "writer closed".into(),
4413 )),
4414 false,
4415 )]
4416 fn test_should_retry_lighter_ws_error(#[case] error: LighterWsError, #[case] expected: bool) {
4417 assert_eq!(should_retry_lighter_ws_error(&error), expected);
4418 }
4419
4420 #[rstest]
4422 #[case::closed(SendError::Closed)]
4423 #[case::invalid_input(SendError::InvalidInput("pong payload too large".into()))]
4424 #[case::timeout(SendError::Timeout)]
4425 #[case::write_timeout(SendError::WriteTimeout)]
4426 #[case::connection_changed(SendError::ConnectionChanged)]
4427 #[case::broken_pipe(SendError::BrokenPipe("writer dropped".into()))]
4428 fn send_error_converts_into_transport_variant(#[case] send_error: SendError) {
4429 let err: LighterWsError = send_error.into();
4430 assert!(
4431 matches!(err, LighterWsError::Transport(_)),
4432 "expected Transport variant, was {err:?}",
4433 );
4434 }
4435
4436 #[tokio::test]
4437 async fn subscribe_command_parks_in_pending_subs_when_inflight_at_cap() {
4438 let signal = Arc::new(AtomicBool::new(false));
4439 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4440 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4441 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4442 let mut handler =
4443 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4444
4445 saturate_subscription_gate(&mut handler);
4447
4448 cmd_tx
4449 .send(HandlerCommand::Subscribe {
4450 channel: LighterWsChannel::Candle {
4451 market_index: 0,
4452 resolution: LighterCandleResolution::OneMinute,
4453 },
4454 auth: None,
4455 response_tx: None,
4456 })
4457 .expect("queue subscribe");
4458 drop(cmd_tx);
4459 drop(raw_tx);
4460
4461 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4462 .await
4463 .expect("timed out waiting for handler to drain command");
4464
4465 assert!(next.is_none());
4466 assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4467 assert_eq!(handler.pending_subs.len(), 1);
4468 assert_eq!(handler.subscription_attempts.len(), 1);
4469 assert_eq!(handler.pending_subs[0].0, Ustr::from("candle:0:1m"));
4470 }
4471
4472 #[tokio::test]
4473 async fn unsubscribe_drops_queued_subscribe_while_gate_full() {
4474 let signal = Arc::new(AtomicBool::new(false));
4475 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4476 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4477 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4478 let mut handler =
4479 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4480
4481 saturate_subscription_gate(&mut handler);
4483
4484 cmd_tx
4485 .send(HandlerCommand::Subscribe {
4486 channel: LighterWsChannel::Trade(0),
4487 auth: None,
4488 response_tx: None,
4489 })
4490 .expect("queue subscribe");
4491 cmd_tx
4492 .send(HandlerCommand::Unsubscribe {
4493 channel: LighterWsChannel::Trade(0),
4494 })
4495 .expect("queue unsubscribe");
4496 drop(cmd_tx);
4497 drop(raw_tx);
4498
4499 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4500 .await
4501 .expect("timed out waiting for handler to drain commands");
4502
4503 assert!(next.is_none());
4504 assert!(handler.pending_subs.is_empty());
4505 assert!(handler.subscription_attempts.is_empty());
4506 }
4507
4508 #[tokio::test]
4509 async fn reconnect_requeues_attempt_with_new_generation() {
4510 let signal = Arc::new(AtomicBool::new(false));
4511 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4512 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4513 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4514 let mut handler =
4515 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4516
4517 saturate_subscription_gate(&mut handler);
4518 handler.queue_subscribe(LighterWsChannel::Trade(0), None, None);
4519 let old_generation = handler.pending_subs[0].1;
4520
4521 raw_tx
4522 .send((7, Message::Text(RECONNECTED.to_string().into())))
4523 .expect("queue reconnect sentinel");
4524
4525 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4526 .await
4527 .expect("timed out waiting for reconnect");
4528
4529 assert!(matches!(next, Some(NautilusWsMessage::Reconnected { .. })));
4530 assert!(handler.inflight_subs.is_empty());
4531 assert_eq!(handler.pending_subs.len(), 1);
4532 assert_eq!(handler.pending_subs[0].0, Ustr::from("trade:0"));
4533 assert_ne!(handler.pending_subs[0].1, old_generation);
4534 assert_eq!(
4535 handler.subscription_attempts[&Ustr::from("trade:0")].generation,
4536 handler.pending_subs[0].1,
4537 );
4538 }
4539
4540 #[rstest]
4541 #[tokio::test]
4542 async fn pump_releases_inflight_slot_and_schedules_send_failure_retry() {
4543 let signal = Arc::new(AtomicBool::new(false));
4544 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4545 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4546 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4547 let mut handler =
4548 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4549
4550 for market_index in 0..3 {
4552 handler.queue_subscribe(LighterWsChannel::Trade(market_index), None, None);
4553 }
4554
4555 handler.pump_pending_subscribes().await;
4556
4557 assert!(handler.pending_subs.is_empty());
4558 assert!(handler.inflight_subs.is_empty());
4559 assert_eq!(handler.subscription_attempts.len(), 3);
4560 assert!(
4561 handler
4562 .subscription_attempts
4563 .values()
4564 .all(|attempt| attempt.retries == 1),
4565 );
4566 assert_eq!(handler.subscription_retries.len(), 3);
4567 }
4568
4569 #[rstest]
4570 fn subscribed_control_frame_releases_inflight_slot() {
4571 let mut handler = make_handler_with_account();
4572 mark_subscription_inflight(
4573 &mut handler,
4574 LighterWsChannel::Candle {
4575 market_index: 0,
4576 resolution: LighterCandleResolution::OneMinute,
4577 },
4578 None,
4579 );
4580
4581 let (matched, msg) = handle_control_text(
4582 &mut handler,
4583 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
4584 );
4585
4586 assert!(matched);
4587 assert!(msg.is_none());
4588 assert!(
4589 !handler
4590 .inflight_subs
4591 .contains_key(&Ustr::from("candle:0:1m"))
4592 );
4593 assert!(handler.subscription_attempts.is_empty());
4594 }
4595
4596 #[rstest]
4597 #[case::top_level(
4598 r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_orders:12345"}"#
4599 )]
4600 #[case::nested(
4601 r#"{"error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4602 )]
4603 #[case::nested_typed(
4604 r#"{"type":"error","error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4605 )]
4606 fn already_subscribed_confirms_matching_inflight_topic(#[case] payload: &str) {
4607 let mut handler = make_handler_with_account();
4608 mark_subscription_inflight(
4609 &mut handler,
4610 LighterWsChannel::AccountAllOrders(12345),
4611 None,
4612 );
4613
4614 let (matched, msg) = handle_control_text(&mut handler, payload);
4615
4616 assert!(matched);
4617 assert!(msg.is_none());
4618 assert!(handler.inflight_subs.is_empty());
4619 assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
4620 assert_eq!(handler.subscriptions.len(), 1);
4621 }
4622
4623 #[rstest]
4624 fn already_subscribed_does_not_confirm_unmatched_topic() {
4625 let mut handler = make_handler_with_account();
4626 let inflight = "account_all_orders:12345";
4627 mark_subscription_inflight(
4628 &mut handler,
4629 LighterWsChannel::AccountAllOrders(12345),
4630 None,
4631 );
4632
4633 let (matched, msg) = handle_control_text(
4634 &mut handler,
4635 r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_trades:12345"}"#,
4636 );
4637
4638 assert!(matched);
4639 assert!(msg.is_none());
4640 assert!(handler.inflight_subs.contains_key(&Ustr::from(inflight)));
4641 assert_eq!(
4642 handler.subscriptions.pending_subscribe_topics(),
4643 vec![inflight]
4644 );
4645 assert_eq!(handler.subscriptions.len(), 0);
4646 }
4647
4648 #[tokio::test]
4649 async fn duplicate_queued_and_inflight_topics_share_one_generation() {
4650 let mut handler = make_handler_with_account();
4651 let (response_tx_1, mut response_rx_1) = tokio::sync::oneshot::channel();
4652 let (response_tx_2, mut response_rx_2) = tokio::sync::oneshot::channel();
4653 let (response_tx_3, mut response_rx_3) = tokio::sync::oneshot::channel();
4654 let channel = LighterWsChannel::Trade(7);
4655
4656 handler.queue_subscribe(channel.clone(), None, Some(response_tx_1));
4657 let generation = handler.pending_subs[0].1;
4658 handler.queue_subscribe(channel.clone(), None, Some(response_tx_2));
4659
4660 assert_eq!(handler.pending_subs.len(), 1);
4661 assert_eq!(handler.subscription_attempts.len(), 1);
4662 assert_eq!(
4663 handler.subscription_attempts[&Ustr::from("trade:7")]
4664 .response_txs
4665 .len(),
4666 2,
4667 );
4668
4669 let (topic, queued_generation) = handler.pending_subs.pop_front().unwrap();
4670 handler.inflight_subs.insert(topic, queued_generation);
4671 handler.queue_subscribe(channel, None, Some(response_tx_3));
4672
4673 assert_eq!(queued_generation, generation);
4674 assert!(handler.pending_subs.is_empty());
4675 assert_eq!(handler.inflight_subs.len(), 1);
4676 assert_eq!(
4677 handler.subscription_attempts[&Ustr::from("trade:7")]
4678 .response_txs
4679 .len(),
4680 3,
4681 );
4682 assert!(matches!(
4683 response_rx_1.try_recv(),
4684 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4685 ));
4686 assert!(matches!(
4687 response_rx_2.try_recv(),
4688 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4689 ));
4690 assert!(matches!(
4691 response_rx_3.try_recv(),
4692 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4693 ));
4694
4695 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
4696
4697 assert_eq!(response_rx_1.await.unwrap(), Ok(()));
4698 assert_eq!(response_rx_2.await.unwrap(), Ok(()));
4699 assert_eq!(response_rx_3.await.unwrap(), Ok(()));
4700 assert!(handler.inflight_subs.is_empty());
4701 assert!(handler.subscription_attempts.is_empty());
4702 }
4703
4704 #[tokio::test]
4705 async fn changed_inflight_auth_waits_for_serialized_successor() {
4706 let mut handler = make_handler_with_account();
4707 let channel = LighterWsChannel::AccountAllOrders(12345);
4708 let topic = Ustr::from(channel.topic_key().as_str());
4709 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
4710 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
4711
4712 handler.queue_subscribe(
4713 channel.clone(),
4714 Some(SecretString::from("old-token")),
4715 Some(old_tx),
4716 );
4717 let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
4718 handler.inflight_subs.insert(queued_topic, old_generation);
4719 handler.queue_subscribe(
4720 channel,
4721 Some(SecretString::from("fresh-token")),
4722 Some(fresh_tx),
4723 );
4724
4725 let attempt = &handler.subscription_attempts[&topic];
4726 assert_eq!(
4727 attempt.auth.as_ref().map(SecretString::expose_secret),
4728 Some("old-token")
4729 );
4730 assert_eq!(
4731 attempt
4732 .pending_auth
4733 .as_ref()
4734 .map(SecretString::expose_secret),
4735 Some("fresh-token")
4736 );
4737 assert_eq!(attempt.response_txs.len(), 1);
4738 assert_eq!(attempt.pending_response_txs.len(), 1);
4739
4740 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4741 assert_eq!(old_rx.await.unwrap(), Ok(()));
4742 assert!(matches!(
4743 fresh_rx.try_recv(),
4744 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4745 ));
4746 let attempt = &handler.subscription_attempts[&topic];
4747 assert_ne!(attempt.generation, old_generation);
4748 assert_eq!(
4749 attempt.auth.as_ref().map(SecretString::expose_secret),
4750 Some("fresh-token")
4751 );
4752 assert!(attempt.pending_auth.is_none());
4753 assert_eq!(
4754 handler.pending_subs.front(),
4755 Some(&(topic, attempt.generation)),
4756 );
4757
4758 let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
4759 handler.inflight_subs.insert(topic, fresh_generation);
4760
4761 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
4764 assert!(matches!(
4765 fresh_rx.try_recv(),
4766 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4767 ));
4768 assert_eq!(
4769 handler.subscription_attempts[&topic].generation,
4770 fresh_generation,
4771 );
4772
4773 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4774 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
4775 assert!(handler.subscription_attempts.is_empty());
4776 }
4777
4778 #[tokio::test]
4779 async fn stale_typed_frame_does_not_complete_fresh_attempt_after_predecessor_removed() {
4780 let mut handler = make_handler_with_account();
4781 let channel = LighterWsChannel::AccountAllOrders(12345);
4782 let topic = Ustr::from(channel.topic_key().as_str());
4783 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
4784 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
4785
4786 handler.queue_subscribe(
4789 channel.clone(),
4790 Some(SecretString::from("old-token")),
4791 Some(old_tx),
4792 );
4793 let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
4794 handler.inflight_subs.insert(queued_topic, old_generation);
4795 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4796 assert_eq!(old_rx.await.unwrap(), Ok(()));
4797 assert!(handler.subscription_attempts.is_empty());
4798
4799 handler.queue_subscribe(
4801 channel,
4802 Some(SecretString::from("fresh-token")),
4803 Some(fresh_tx),
4804 );
4805 let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
4806 handler.inflight_subs.insert(topic, fresh_generation);
4807
4808 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
4810 assert!(matches!(
4811 fresh_rx.try_recv(),
4812 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4813 ));
4814 assert_eq!(
4815 handler.subscription_attempts[&topic].generation,
4816 fresh_generation,
4817 );
4818
4819 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4820 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
4821 assert!(handler.subscription_attempts.is_empty());
4822 }
4823
4824 #[tokio::test]
4825 async fn changed_queued_auth_updates_existing_generation_before_dispatch() {
4826 let mut handler = make_handler_with_account();
4827 let channel = LighterWsChannel::AccountAllOrders(12345);
4828 let topic = Ustr::from(channel.topic_key().as_str());
4829 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
4830 let (fresh_tx, fresh_rx) = tokio::sync::oneshot::channel();
4831
4832 handler.queue_subscribe(
4833 channel.clone(),
4834 Some(SecretString::from("old-token")),
4835 Some(old_tx),
4836 );
4837 let generation = handler.pending_subs[0].1;
4838 handler.queue_subscribe(
4839 channel,
4840 Some(SecretString::from("fresh-token")),
4841 Some(fresh_tx),
4842 );
4843
4844 assert_eq!(handler.pending_subs.len(), 1);
4845 assert_eq!(handler.pending_subs[0], (topic, generation));
4846 let attempt = &handler.subscription_attempts[&topic];
4847 assert_eq!(attempt.generation, generation);
4848 assert_eq!(
4849 attempt.auth.as_ref().map(SecretString::expose_secret),
4850 Some("fresh-token")
4851 );
4852 assert!(attempt.pending_auth.is_none());
4853 assert_eq!(attempt.response_txs.len(), 2);
4854 assert!(attempt.pending_response_txs.is_empty());
4855
4856 handler.pending_subs.pop_front();
4857 handler.inflight_subs.insert(topic, generation);
4858 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4859 assert_eq!(old_rx.await.unwrap(), Ok(()));
4860 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
4861 }
4862
4863 #[rstest]
4864 fn second_reconnect_folds_pending_auth_into_requeued_generation() {
4865 let mut handler = make_handler_with_account();
4866 let channel = LighterWsChannel::AccountAllOrders(12345);
4867 let topic = Ustr::from(channel.topic_key().as_str());
4868 let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
4869 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
4870
4871 handler.queue_subscribe(
4872 channel.clone(),
4873 Some(SecretString::from("old-token")),
4874 Some(old_tx),
4875 );
4876 handler.reset_subscription_attempts_after_reconnect();
4877 let (_, replay_generation) = handler.pending_subs.pop_front().unwrap();
4878 handler.inflight_subs.insert(topic, replay_generation);
4879 handler.queue_subscribe(
4880 channel,
4881 Some(SecretString::from("fresh-token")),
4882 Some(fresh_tx),
4883 );
4884
4885 let attempt = &handler.subscription_attempts[&topic];
4886 assert_eq!(
4887 attempt.auth.as_ref().map(SecretString::expose_secret),
4888 Some("old-token")
4889 );
4890 assert_eq!(
4891 attempt
4892 .pending_auth
4893 .as_ref()
4894 .map(SecretString::expose_secret),
4895 Some("fresh-token")
4896 );
4897 assert_eq!(attempt.response_txs.len(), 1);
4898 assert_eq!(attempt.pending_response_txs.len(), 1);
4899
4900 handler.reset_subscription_attempts_after_reconnect();
4901
4902 let attempt = &handler.subscription_attempts[&topic];
4903 assert_ne!(attempt.generation, replay_generation);
4904 assert_eq!(
4905 attempt.auth.as_ref().map(SecretString::expose_secret),
4906 Some("fresh-token")
4907 );
4908 assert!(attempt.pending_auth.is_none());
4909 assert_eq!(attempt.response_txs.len(), 2);
4910 assert!(attempt.pending_response_txs.is_empty());
4911 assert_eq!(handler.pending_subs.len(), 1);
4912 assert_eq!(handler.pending_subs[0], (topic, attempt.generation));
4913 assert!(handler.inflight_subs.is_empty());
4914 assert!(matches!(
4915 old_rx.try_recv(),
4916 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4917 ));
4918 assert!(matches!(
4919 fresh_rx.try_recv(),
4920 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4921 ));
4922
4923 let (_, generation) = handler.pending_subs.pop_front().unwrap();
4924 handler.inflight_subs.insert(topic, generation);
4925 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
4926
4927 assert_eq!(old_rx.try_recv(), Ok(Ok(())));
4928 assert_eq!(fresh_rx.try_recv(), Ok(Ok(())));
4929 assert!(handler.subscription_attempts.is_empty());
4930 }
4931
4932 #[tokio::test]
4933 async fn retry_folds_pending_auth_without_resetting_retry_budget() {
4934 let mut handler = make_handler_with_account();
4935 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
4936 handler.set_command_sender(cmd_tx);
4937 let channel = LighterWsChannel::AccountAllOrders(12345);
4938 let topic = Ustr::from(channel.topic_key().as_str());
4939 let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
4940 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
4941
4942 handler.queue_subscribe(
4943 channel.clone(),
4944 Some(SecretString::from("old-token")),
4945 Some(old_tx),
4946 );
4947 let (_, generation) = handler.pending_subs.pop_front().unwrap();
4948 handler.inflight_subs.insert(topic, generation);
4949 handler
4950 .subscription_attempts
4951 .get_mut(&topic)
4952 .unwrap()
4953 .retries = 2;
4954 handler.queue_subscribe(
4955 channel,
4956 Some(SecretString::from("fresh-token")),
4957 Some(fresh_tx),
4958 );
4959
4960 handler.schedule_subscription_retry(topic, generation, "retry");
4961
4962 let attempt = &handler.subscription_attempts[&topic];
4963 assert_eq!(attempt.retries, 3);
4964 assert_ne!(attempt.generation, generation);
4965 assert_eq!(
4966 attempt.auth.as_ref().map(SecretString::expose_secret),
4967 Some("fresh-token")
4968 );
4969 assert!(attempt.pending_auth.is_none());
4970 assert_eq!(attempt.response_txs.len(), 2);
4971 assert!(attempt.pending_response_txs.is_empty());
4972 assert!(handler.pending_subs.is_empty());
4973 assert!(handler.inflight_subs.is_empty());
4974 assert_eq!(handler.subscription_retries.len(), 1);
4975 assert!(matches!(
4976 old_rx.try_recv(),
4977 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4978 ));
4979 assert!(matches!(
4980 fresh_rx.try_recv(),
4981 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
4982 ));
4983 }
4984
4985 #[tokio::test]
4986 async fn confirmed_authenticated_topic_opens_new_generation() {
4987 let mut handler = make_handler_with_account();
4988 let channel = LighterWsChannel::AccountAllOrders(12345);
4989 let topic = channel.topic_key();
4990 handler.subscriptions.mark_subscribe(&topic);
4991 handler.subscriptions.confirm_subscribe(&topic);
4992 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4993
4994 handler.queue_subscribe(
4995 channel,
4996 Some(SecretString::from("rotated-auth-token")),
4997 Some(response_tx),
4998 );
4999
5000 assert_eq!(handler.pending_subs.len(), 1);
5001 assert_eq!(handler.subscription_attempts.len(), 1);
5002 assert_eq!(handler.subscriptions.len(), 1);
5003 assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5004
5005 let (topic, generation) = handler.pending_subs.pop_front().unwrap();
5006 handler.inflight_subs.insert(topic, generation);
5007 handle_control_text(
5008 &mut handler,
5009 r#"{"type":"subscribed","channel":"account_all_orders:12345"}"#,
5010 );
5011
5012 assert_eq!(response_rx.await.unwrap(), Ok(()));
5013 assert!(handler.subscription_attempts.is_empty());
5014 assert_eq!(handler.subscriptions.len(), 1);
5015 }
5016
5017 #[tokio::test]
5018 async fn typed_update_before_ack_does_not_complete_subscription_generation() {
5019 let mut handler = make_handler_with_account();
5020 let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
5021 let (topic, generation) = mark_subscription_inflight(
5022 &mut handler,
5023 LighterWsChannel::Candle {
5024 market_index: 0,
5025 resolution: LighterCandleResolution::OneMinute,
5026 },
5027 Some(response_tx),
5028 );
5029
5030 handler.handle_frame(
5031 candle_frame(
5032 "candle:0:1m",
5033 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
5034 false,
5035 ),
5036 UnixNanos::from(1),
5037 );
5038
5039 assert_eq!(
5040 handler
5041 .last_candles
5042 .get(&(0, LighterCandleResolution::OneMinute))
5043 .map(|candle| candle.t),
5044 Some(1_000_000),
5045 );
5046 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5047 assert_eq!(
5048 handler.subscriptions.pending_subscribe_topics(),
5049 vec!["candle:0:1m"],
5050 );
5051 assert_eq!(handler.subscriptions.len(), 0);
5052 assert!(matches!(
5053 response_rx.try_recv(),
5054 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5055 ));
5056
5057 handle_control_text(
5058 &mut handler,
5059 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5060 );
5061
5062 assert_eq!(response_rx.await.unwrap(), Ok(()));
5063 assert_eq!(handler.subscriptions.len(), 1);
5064 }
5065
5066 #[tokio::test]
5067 async fn typed_subscribed_frame_completes_subscription_generation() {
5068 let signal = Arc::new(AtomicBool::new(false));
5069 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5070 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5071 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5072 let mut handler =
5073 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5074 handler.instruments.insert(0, stub_eth_perp_instrument());
5075 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5076 mark_subscription_inflight(
5077 &mut handler,
5078 LighterWsChannel::Candle {
5079 market_index: 0,
5080 resolution: LighterCandleResolution::OneMinute,
5081 },
5082 Some(response_tx),
5083 );
5084
5085 raw_tx
5086 .send((7, Message::Text(WS_CANDLE_SUBSCRIBED.into())))
5087 .expect("typed subscribed frame");
5088 drop(raw_tx);
5089 drop(cmd_tx);
5090
5091 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5092 .await
5093 .expect("handler did not process typed subscribed frame");
5094
5095 assert!(next.is_none());
5096 assert_eq!(response_rx.await.unwrap(), Ok(()));
5097 assert!(handler.inflight_subs.is_empty());
5098 assert!(handler.subscription_attempts.is_empty());
5099 assert_eq!(handler.subscriptions.len(), 1);
5100 }
5101
5102 #[tokio::test]
5103 async fn unparsable_subscribed_frame_still_completes_subscription() {
5104 let signal = Arc::new(AtomicBool::new(false));
5105 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5106 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5107 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5108 let mut handler =
5109 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5110 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5111 mark_subscription_inflight(
5112 &mut handler,
5113 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(4098)),
5114 Some(response_tx),
5115 );
5116
5117 raw_tx
5118 .send((7, Message::Text(WS_SPOT_STATS_SUBSCRIBED_BAD_BODY.into())))
5119 .expect("unparsable subscribed frame");
5120 drop(raw_tx);
5121 drop(cmd_tx);
5122
5123 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5124 .await
5125 .expect("handler did not process unparsable subscribed frame");
5126
5127 assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5128 let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5129 .await
5130 .expect("subscription was not completed");
5131 assert_eq!(response.unwrap(), Ok(()));
5132 assert!(handler.inflight_subs.is_empty());
5133 assert!(handler.subscription_attempts.is_empty());
5134 assert_eq!(handler.subscriptions.len(), 1);
5135 }
5136
5137 #[tokio::test]
5138 async fn unparsable_book_subscribed_frame_still_completes_subscription() {
5139 let signal = Arc::new(AtomicBool::new(false));
5140 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5141 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5142 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5143 let mut handler =
5144 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5145 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5146 mark_subscription_inflight(
5147 &mut handler,
5148 LighterWsChannel::OrderBook(4095),
5149 Some(response_tx),
5150 );
5151
5152 raw_tx
5155 .send((0, Message::Text(WS_BOOK_SUBSCRIBED_BAD_BODY.into())))
5156 .expect("unparsable book subscribed frame");
5157 drop(raw_tx);
5158 drop(cmd_tx);
5159
5160 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5161 .await
5162 .expect("handler did not process unparsable book subscribed frame");
5163
5164 assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5165 let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5166 .await
5167 .expect("book subscription was not completed");
5168 assert_eq!(response.unwrap(), Ok(()));
5169 assert!(handler.inflight_subs.is_empty());
5170 assert!(handler.subscription_attempts.is_empty());
5171 assert_eq!(handler.subscriptions.len(), 1);
5172 }
5173
5174 #[tokio::test]
5175 async fn failed_subscribe_error_fails_every_waiter_for_the_generation() {
5176 let mut handler = make_handler_with_account();
5177 let (response_tx_1, response_rx_1) = tokio::sync::oneshot::channel();
5178 let (response_tx_2, response_rx_2) = tokio::sync::oneshot::channel();
5179 let channel = LighterWsChannel::MarketStats(LighterMarketSelection::Market(0));
5180 mark_subscription_inflight(&mut handler, channel.clone(), Some(response_tx_1));
5181 handler.queue_subscribe(channel, None, Some(response_tx_2));
5182
5183 let (matched, msg) = handle_control_text(
5184 &mut handler,
5185 r#"{"type":"error","code":30012,"message":"failed to subscribe"}"#,
5186 );
5187
5188 assert!(matched);
5189 assert!(msg.is_none());
5190 let error_1 = response_rx_1.await.unwrap().unwrap_err();
5191 let error_2 = response_rx_2.await.unwrap().unwrap_err();
5192 assert!(error_1.contains("market_stats:0"));
5193 assert!(error_1.contains("30012"));
5194 assert_eq!(error_1, error_2);
5195 assert!(handler.inflight_subs.is_empty());
5196 assert!(handler.subscription_attempts.is_empty());
5197 assert!(handler.subscriptions.is_empty());
5198 }
5199
5200 #[tokio::test]
5201 async fn rate_limit_retry_exhaustion_fails_waiter_and_clears_intent() {
5202 let mut handler = make_handler_with_account();
5203 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5204 let (topic, _) =
5205 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(response_tx));
5206 handler
5207 .subscription_attempts
5208 .get_mut(&topic)
5209 .expect("subscription attempt")
5210 .retries = SUBSCRIBE_RETRY_MAX;
5211
5212 let (matched, msg) = handle_control_text(
5213 &mut handler,
5214 r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
5215 );
5216
5217 assert!(matched);
5218 assert!(msg.is_none());
5219 assert_eq!(
5220 response_rx.await.unwrap(),
5221 Err(
5222 "subscription trade:7 failed after 6 attempts: venue rejected the WebSocket \
5223 subscribe with code 30009"
5224 .to_string(),
5225 ),
5226 );
5227 assert!(handler.inflight_subs.is_empty());
5228 assert!(handler.subscription_attempts.is_empty());
5229 assert!(handler.subscriptions.is_empty());
5230 }
5231
5232 #[tokio::test]
5233 async fn rate_limit_error_retries_each_inflight_topic_with_new_generation() {
5234 let mut handler = make_handler_with_account();
5235 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5236 handler.set_command_sender(cmd_tx);
5237 let (trade_tx, mut trade_rx) = tokio::sync::oneshot::channel();
5238 let (candle_tx, mut candle_rx) = tokio::sync::oneshot::channel();
5239 let (trade_topic, trade_generation) =
5240 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(trade_tx));
5241 let (candle_topic, candle_generation) = mark_subscription_inflight(
5242 &mut handler,
5243 LighterWsChannel::Candle {
5244 market_index: 0,
5245 resolution: LighterCandleResolution::OneMinute,
5246 },
5247 Some(candle_tx),
5248 );
5249
5250 let (matched, msg) = handle_control_text(
5251 &mut handler,
5252 r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
5253 );
5254
5255 assert!(matched);
5256 assert!(msg.is_none());
5257 assert!(handler.inflight_subs.is_empty());
5258 assert!(handler.pending_subs.is_empty());
5259 let trade_retry = &handler.subscription_attempts[&trade_topic];
5260 let candle_retry = &handler.subscription_attempts[&candle_topic];
5261 assert_eq!(trade_retry.retries, 1);
5262 assert_eq!(candle_retry.retries, 1);
5263 assert_ne!(trade_retry.generation, trade_generation);
5264 assert_ne!(candle_retry.generation, candle_generation);
5265 assert_eq!(handler.subscription_retries.len(), 2);
5266 assert!(matches!(
5267 trade_rx.try_recv(),
5268 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5269 ));
5270 assert!(matches!(
5271 candle_rx.try_recv(),
5272 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5273 ));
5274
5275 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5276 assert_eq!(handler.subscription_attempts.len(), 2);
5277 assert_eq!(handler.subscriptions.len(), 0);
5278
5279 let retry_generations = [
5280 (
5281 trade_topic,
5282 handler.subscription_attempts[&trade_topic].generation,
5283 ),
5284 (
5285 candle_topic,
5286 handler.subscription_attempts[&candle_topic].generation,
5287 ),
5288 ];
5289
5290 for (topic, generation) in retry_generations {
5291 handler.queue_subscription_retry(topic, generation);
5292 let queued = handler.pending_subs.pop_front().unwrap();
5293 assert_eq!(queued, (topic, generation));
5294 handler.inflight_subs.insert(topic, generation);
5295 }
5296
5297 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5298 handle_control_text(
5299 &mut handler,
5300 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5301 );
5302
5303 assert_eq!(trade_rx.await.unwrap(), Ok(()));
5304 assert_eq!(candle_rx.await.unwrap(), Ok(()));
5305 assert!(handler.subscription_attempts.is_empty());
5306 assert_eq!(handler.subscriptions.len(), 2);
5307 }
5308}