1use std::{
19 collections::VecDeque,
20 fmt::Debug,
21 future::Future,
22 pin::Pin,
23 sync::{
24 Arc,
25 atomic::{AtomicBool, Ordering},
26 },
27 time::Duration,
28};
29
30use ahash::AHashMap;
31use futures_util::{StreamExt, stream::FuturesUnordered};
32use nautilus_core::{
33 AtomicTime,
34 nanos::UnixNanos,
35 string::secret::{REDACTED, SecretString},
36 time::get_atomic_clock_realtime,
37};
38#[cfg(test)]
39use nautilus_live::book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS;
40use nautilus_live::book::{
41 BookSequenceOutcome,
42 recovery::{BookRecoveryOutcome, BookRecoveryState},
43 snapshot::{PendingSnapshot, SnapshotGate},
44};
45use nautilus_model::{identifiers::AccountId, instruments::InstrumentAny, types::Currency};
46use nautilus_network::{
47 RECONNECTED,
48 error::SendError,
49 retry::{RetryManager, create_websocket_retry_manager},
50 websocket::{SubscriptionState, WebSocketClient},
51};
52use tokio_tungstenite::tungstenite::Message;
53use tokio_util::sync::CancellationToken;
54use ustr::Ustr;
55use zeroize::Zeroize;
56
57use super::{
58 account_state::LighterAccountStateReconciler,
59 error::LighterWsError,
60 messages::{
61 AccountStream, ExecutionReport, LighterAsset, LighterPosition, LighterUserStats,
62 LighterWsCandle, LighterWsChannel, LighterWsChannelKind, LighterWsFrame,
63 LighterWsOrderBook, LighterWsRequest, NautilusWsMessage, SendTxRejectionSource,
64 TX_BATCH_ID_PREFIX,
65 },
66 parse::{
67 parse_ws_bar, parse_ws_funding_rate_update, parse_ws_index_price_update,
68 parse_ws_mark_price_update, parse_ws_position_status_report, parse_ws_quote_tick,
69 parse_ws_spot_index_price_update, parse_ws_trade_tick,
70 },
71};
72use crate::{
73 book::{
74 recovery::{self, BookWorkResult, BookWrite},
75 sync::BookSyncTracker,
76 },
77 common::{
78 consts::{
79 LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED, LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED,
80 LIGHTER_ERROR_CODE_TX_RANGE, LIGHTER_ERROR_CODE_WS_RATE_LIMITED,
81 LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED, LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL,
82 SUBSCRIBE_INFLIGHT_MAX, SUBSCRIBE_RETRY_BASE_BACKOFF, SUBSCRIBE_RETRY_MAX,
83 },
84 enums::LighterCandleResolution,
85 rate_limit::LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY,
86 },
87 http::models::{LighterOrder, LighterTrade},
88};
89
90const CTRL_TYPE_CONNECTED: &str = "connected";
94const CTRL_TYPE_SUBSCRIBED: &str = "subscribed";
95const CTRL_TYPE_UNSUBSCRIBED: &str = "unsubscribed";
96const CTRL_TYPE_PING: &str = "ping";
97const CTRL_TYPE_PONG: &str = "pong";
98const CTRL_TYPE_ERROR: &str = "error";
99const CTRL_TYPE_SEND_TX: &str = "jsonapi/sendtx";
100
101#[derive(serde::Deserialize)]
102struct LighterWsFrameHeader<'a> {
103 #[serde(rename = "type", borrow)]
104 kind: &'a str,
105 #[serde(borrow)]
106 channel: Option<&'a str>,
107}
108
109#[expect(
111 clippy::large_enum_variant,
112 reason = "commands are ephemeral and immediately consumed"
113)]
114pub enum HandlerCommand {
115 SetClient(WebSocketClient),
118 Disconnect,
120 Subscribe {
123 channel: LighterWsChannel,
124 auth: Option<SecretString>,
125 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
126 },
127 Unsubscribe { channel: LighterWsChannel },
129 RecoverBook {
131 market_index: i64,
132 cancel: CancellationToken,
133 gate: SnapshotGate,
134 completion: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
135 },
136 InitializeInstruments(Vec<(i64, InstrumentAny)>),
138 UpdateInstrument {
140 market_index: i64,
141 instrument: InstrumentAny,
142 },
143 SetBookDeltasSub { market_index: i64, subscribed: bool },
146 SetDepthSub { market_index: i64, subscribed: bool },
149 SetExecutionContext {
154 account_id: AccountId,
155 account_index: i64,
156 },
157 SendTx {
163 tx_type: u8,
164 tx_info: Box<serde_json::value::RawValue>,
165 connection_epoch: u64,
166 response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
167 },
168 SendTxBatch {
169 data: super::messages::LighterWsSendTxBatch,
170 connection_epoch: u64,
171 response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
172 },
173}
174
175impl Debug for HandlerCommand {
176 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181 match self {
182 Self::SetClient(_) => f.write_str("SetClient(<WebSocketClient>)"),
183 Self::Disconnect => f.write_str("Disconnect"),
184 Self::Subscribe { channel, auth, .. } => f
185 .debug_struct(stringify!(Subscribe))
186 .field("channel", channel)
187 .field("auth", auth)
188 .finish(),
189 Self::Unsubscribe { channel } => f
190 .debug_struct(stringify!(Unsubscribe))
191 .field("channel", channel)
192 .finish(),
193 Self::RecoverBook { market_index, .. } => f
194 .debug_struct(stringify!(RecoverBook))
195 .field("market_index", market_index)
196 .finish(),
197 Self::InitializeInstruments(instruments) => f
198 .debug_tuple(stringify!(InitializeInstruments))
199 .field(&instruments.len())
200 .finish(),
201 Self::UpdateInstrument { market_index, .. } => f
202 .debug_struct(stringify!(UpdateInstrument))
203 .field("market_index", market_index)
204 .finish(),
205 Self::SetBookDeltasSub {
206 market_index,
207 subscribed,
208 } => f
209 .debug_struct(stringify!(SetBookDeltasSub))
210 .field("market_index", market_index)
211 .field("subscribed", subscribed)
212 .finish(),
213 Self::SetDepthSub {
214 market_index,
215 subscribed,
216 } => f
217 .debug_struct(stringify!(SetDepthSub))
218 .field("market_index", market_index)
219 .field("subscribed", subscribed)
220 .finish(),
221 Self::SetExecutionContext {
222 account_id,
223 account_index,
224 } => f
225 .debug_struct(stringify!(SetExecutionContext))
226 .field("account_id", account_id)
227 .field("account_index", account_index)
228 .finish(),
229 Self::SendTxBatch { .. } => f.write_str("SendTxBatch(<redacted>)"),
230 Self::SendTx { tx_type, .. } => f
231 .debug_struct(stringify!(SendTx))
232 .field("tx_type", tx_type)
233 .field("tx_info", &REDACTED)
234 .finish(),
235 }
236 }
237}
238
239pub(super) struct FeedHandler {
246 clock: &'static AtomicTime,
247 signal: Arc<AtomicBool>,
248 inner: Option<Arc<WebSocketClient>>,
249 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
250 cmd_tx: Option<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>,
251 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
252 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
253 subscriptions: SubscriptionState,
254 retry_manager: RetryManager<LighterWsError>,
255 pending_messages: VecDeque<NautilusWsMessage>,
256 pending_subs: VecDeque<(Ustr, u64)>,
257 inflight_subs: AHashMap<Ustr, u64>,
258 subscription_attempts: AHashMap<Ustr, SubscriptionAttempt>,
259 subscription_retries: FuturesUnordered<SubscriptionRetry>,
260 ignored_completions: AHashMap<Ustr, CompletionKind>,
261 next_subscription_generation: u64,
262 instruments: AHashMap<i64, InstrumentAny>,
263 book: BookSyncTracker,
264 book_snapshot_timeout: Duration,
265 last_candles: AHashMap<(i64, LighterCandleResolution), LighterWsCandle>,
266 exec_account: Option<(AccountId, i64)>,
267 account_state_reconciler: LighterAccountStateReconciler,
268}
269
270type SubscriptionRetry = Pin<Box<dyn Future<Output = (Ustr, u64)> + Send + Sync + 'static>>;
271
272struct SubscriptionAttempt {
273 channel: LighterWsChannel,
274 auth: Option<SecretString>,
275 pending_auth: Option<SecretString>,
276 generation: u64,
277 retries: u8,
278 response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
279 pending_response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
280}
281
282impl SubscriptionAttempt {
283 fn fold_pending_auth(&mut self) {
284 if let Some(auth) = self.pending_auth.take() {
285 self.auth = Some(auth);
286 self.response_txs.append(&mut self.pending_response_txs);
287 } else {
288 debug_assert!(self.pending_response_txs.is_empty());
289 }
290 }
291}
292
293#[derive(Clone, Copy, Debug, PartialEq, Eq)]
301enum CompletionKind {
302 ControlAck,
303 Typed,
304 AlreadySubscribed,
305}
306
307impl FeedHandler {
308 #[cfg(test)]
309 pub(super) fn new(
310 signal: Arc<AtomicBool>,
311 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
312 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
313 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
314 subscriptions: SubscriptionState,
315 ) -> Self {
316 Self::new_with_settlement_currency(
317 signal,
318 cmd_rx,
319 raw_rx,
320 out_tx,
321 subscriptions,
322 Currency::get_or_create_crypto("USDC"),
323 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
324 )
325 }
326
327 pub(super) fn new_with_settlement_currency(
328 signal: Arc<AtomicBool>,
329 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
330 raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
331 out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
332 subscriptions: SubscriptionState,
333 settlement_currency: Currency,
334 book_snapshot_timeout: Duration,
335 ) -> Self {
336 Self {
337 clock: get_atomic_clock_realtime(),
338 signal,
339 inner: None,
340 cmd_rx,
341 cmd_tx: None,
342 raw_rx,
343 out_tx,
344 subscriptions,
345 retry_manager: create_websocket_retry_manager(),
346 pending_messages: VecDeque::new(),
347 pending_subs: VecDeque::new(),
348 inflight_subs: AHashMap::new(),
349 subscription_attempts: AHashMap::new(),
350 subscription_retries: FuturesUnordered::new(),
351 ignored_completions: AHashMap::new(),
352 next_subscription_generation: 1,
353 instruments: AHashMap::new(),
354 book: BookSyncTracker::default(),
355 book_snapshot_timeout,
356 last_candles: AHashMap::new(),
357 exec_account: None,
358 account_state_reconciler: LighterAccountStateReconciler::new_with_settlement_currency(
359 settlement_currency,
360 ),
361 }
362 }
363
364 pub(super) fn send(&self, msg: NautilusWsMessage) -> Result<(), String> {
365 self.out_tx
366 .send(msg)
367 .map_err(|e| format!("Failed to send message: {e}"))
368 }
369
370 pub(super) fn set_command_sender(
371 &mut self,
372 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
373 ) {
374 self.cmd_tx = Some(cmd_tx);
375 }
376
377 pub(super) fn is_stopped(&self) -> bool {
378 self.signal.load(Ordering::Relaxed)
379 }
380
381 async fn send_with_retry(&self, payload: String) -> Result<(), LighterWsError> {
382 self.send_secret_with_retry(SecretString::from(payload))
383 .await
384 }
385
386 async fn send_secret_with_retry(&self, payload: SecretString) -> Result<(), LighterWsError> {
387 if let Some(client) = &self.inner {
388 self.retry_manager
389 .invocation(
390 "websocket_send",
391 || {
392 let payload = payload.clone();
393 async move {
394 client
395 .send_text(
396 payload.expose_secret().to_owned(),
397 Some(LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY.as_slice()),
398 )
399 .await
400 .map_err(LighterWsError::Transport)
401 }
402 },
403 should_retry_lighter_ws_error,
404 |e| create_lighter_ws_timeout_error(e.to_string()),
405 )
406 .execute()
407 .await
408 } else {
409 Err(LighterWsError::Client(
410 "no active WebSocket client".to_string(),
411 ))
412 }
413 }
414
415 async fn send_once(
418 &self,
419 payload: String,
420 connection_epoch: u64,
421 ) -> Result<(), LighterWsError> {
422 if let Some(client) = &self.inner {
423 match client
424 .send_text_on_connection(payload, None, connection_epoch)
425 .await
426 {
427 Err(SendError::BrokenPipe(message)) => {
428 Err(LighterWsError::SendTxOutcomeUnknown(message))
429 }
430 result => result.map_err(LighterWsError::Transport),
431 }
432 } else {
433 Err(LighterWsError::Client(
434 "no active WebSocket client".to_string(),
435 ))
436 }
437 }
438
439 async fn dispatch_subscribe(
440 &self,
441 channel: LighterWsChannel,
442 auth: Option<SecretString>,
443 ) -> Result<(), String> {
444 let topic = channel.topic_key();
445
446 let authed = auth.is_some();
447 let mut request = match auth {
448 Some(token) => LighterWsRequest::subscribe_auth(
449 channel.subscription_channel(),
450 token.expose_secret(),
451 ),
452 None => LighterWsRequest::subscribe(channel.subscription_channel()),
453 };
454
455 let payload = serde_json::to_string(&request);
456 request.zeroize();
457
458 match payload {
459 Ok(payload) => {
460 log::debug!("Sending Lighter subscribe: topic={topic} authed={authed}");
463 let result = if authed {
464 self.send_secret_with_retry(SecretString::from(payload))
465 .await
466 } else {
467 self.send_with_retry(payload).await
468 };
469
470 if let Err(e) = result {
471 log::error!("Error subscribing to {topic}: {e}");
472 Err(e.to_string())
473 } else {
474 Ok(())
475 }
476 }
477 Err(e) => {
478 log::error!("Error serializing subscription for {topic}: {e}");
479 Err(format!("failed to serialize subscription for {topic}: {e}"))
480 }
481 }
482 }
483
484 async fn dispatch_send_tx(
485 &self,
486 tx_type: u8,
487 tx_info: Box<serde_json::value::RawValue>,
488 connection_epoch: u64,
489 ) -> Result<(), LighterWsError> {
490 let request = LighterWsRequest::SendTx {
491 data: super::messages::LighterWsSendTx { tx_type, tx_info },
492 };
493
494 match serde_json::to_string(&request) {
495 Ok(payload) => {
496 log::debug!(
497 "Sending Lighter sendTx: tx_type={tx_type} ({} bytes)",
498 payload.len(),
499 );
500
501 match self.send_once(payload, connection_epoch).await {
502 Ok(()) => Ok(()),
503 Err(e) => {
504 log::debug!("Error dispatching Lighter sendTx (tx_type={tx_type}): {e}");
505 Err(e)
506 }
507 }
508 }
509 Err(e) => {
510 log::error!("Error serializing Lighter sendTx (tx_type={tx_type}): {e}");
511 Err(LighterWsError::Client(format!(
512 "failed to serialize Lighter sendTx: {e}"
513 )))
514 }
515 }
516 }
517
518 async fn dispatch_unsubscribe(&self, channel: LighterWsChannel) {
519 let topic = channel.topic_key();
520 self.subscriptions.mark_unsubscribe(&topic);
521
522 let request = LighterWsRequest::unsubscribe(channel.subscription_channel());
523 match serde_json::to_string(&request) {
524 Ok(payload) => {
525 log::debug!("Sending Lighter unsubscribe ({} bytes)", payload.len());
526 if let Err(e) = self.send_with_retry(payload).await {
527 log::error!("Error unsubscribing from {topic}: {e}");
528 }
529 }
530 Err(e) => {
531 log::error!("Error serializing unsubscription for {topic}: {e}");
532 }
533 }
534 }
535
536 pub(super) async fn next(&mut self) -> Option<NautilusWsMessage> {
537 if let Some(msg) = self.pending_messages.pop_front() {
538 return Some(msg);
539 }
540
541 loop {
542 self.pump_pending_subscribes().await;
544
545 tokio::select! {
546 biased;
547 Some(work) = self.book.work.next(), if !self.book.work.is_empty() => {
548 match work {
549 BookWorkResult::Sent { market_index, generation, cancel, write, result } => {
550 self.complete_book_send(market_index, generation, cancel, write, result);
551 }
552 BookWorkResult::Initial { market_index, cancel } => {
553 if !cancel.is_cancelled() {
554 self.start_book_recovery(market_index);
555 }
556 }
557 BookWorkResult::Recovery => {}
559 }
560 }
561 Some(cmd) = self.cmd_rx.recv() => {
562 match cmd {
563 HandlerCommand::SetClient(client) => {
564 log::debug!("Setting WebSocket client in Lighter handler");
565 self.inner = Some(Arc::new(client));
566 }
567 HandlerCommand::Disconnect => {
568 log::debug!("Lighter handler received disconnect");
569 if let Some(ref client) = self.inner {
570 client.disconnect().await;
571 }
572 self.signal.store(true, Ordering::SeqCst);
573 return None;
574 }
575 HandlerCommand::Subscribe {
576 channel,
577 auth,
578 response_tx,
579 } => {
580 self.queue_subscribe(channel, auth, response_tx);
581 }
582 HandlerCommand::Unsubscribe { channel } => {
583 let topic = channel.topic_key();
586 self.cancel_subscription_attempt(
587 &topic,
588 "subscription cancelled before venue acknowledgement",
589 );
590
591 if let LighterWsChannel::OrderBook(market_index) = &channel {
592 self.book.cancel(*market_index);
593 self.book.clear_cached_order_book(*market_index);
594 }
595 self.dispatch_unsubscribe(channel).await;
596 }
597 HandlerCommand::RecoverBook { market_index, cancel, gate, completion } => {
598 self.queue_book_replacement(market_index, BookWrite { cancel, gate, completion });
599 }
600 HandlerCommand::InitializeInstruments(instruments) => {
601 self.instruments.clear();
602 for (market_index, inst) in instruments {
603 self.instruments.insert(market_index, inst);
604 }
605 }
606 HandlerCommand::UpdateInstrument { market_index, instrument } => {
607 self.instruments.insert(market_index, instrument);
608 }
609 HandlerCommand::SetBookDeltasSub { market_index, subscribed } => {
610 if subscribed {
611 let inserted = self.book.delta_subs.insert(market_index);
612 if inserted
613 && let Some(first) = self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_deltas_snapshot(market_index, instrument, self.clock.get_time_ns()))
614 {
615 return Some(first);
616 }
617 } else {
618 self.book.delta_subs.remove(&market_index);
619 self.cancel_orphaned_book_recovery(market_index);
620 }
621 }
622 HandlerCommand::SetDepthSub { market_index, subscribed } => {
623 if subscribed {
624 let inserted = self.book.depth_subs.insert(market_index);
625 if inserted
626 && let Some(first) =
627 self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_depth_snapshot(market_index, instrument, self.clock.get_time_ns()))
628 {
629 return Some(first);
630 }
631 } else {
632 self.book.depth_subs.remove(&market_index);
633 self.cancel_orphaned_book_recovery(market_index);
634 }
635 }
636 HandlerCommand::SetExecutionContext { account_id, account_index } => {
637 self.exec_account = Some((account_id, account_index));
638 }
639 HandlerCommand::SendTxBatch { data, connection_epoch, response_tx } => {
640 let result = match serde_json::to_string(&LighterWsRequest::SendTxBatch { data }) {
641 Ok(payload) => self.send_once(payload, connection_epoch).await,
642 Err(e) => Err(LighterWsError::Client(format!("failed to serialize Lighter sendTxBatch: {e}"))),
643 };
644 let _ = response_tx.send(result);
645 }
646 HandlerCommand::SendTx {
647 tx_type,
648 tx_info,
649 connection_epoch,
650 response_tx,
651 } => {
652 let result = self
653 .dispatch_send_tx(tx_type, tx_info, connection_epoch)
654 .await;
655
656 if response_tx.send(result).is_err() {
657 log::debug!("Lighter sendTx result receiver dropped");
658 }
659 }
660 }
661 }
662 Some((topic, generation)) = self.subscription_retries.next(),
663 if !self.subscription_retries.is_empty() =>
664 {
665 self.queue_subscription_retry(topic, generation);
666 }
667 Some((connection_epoch, raw_msg)) = self.raw_rx.recv() => {
668 match raw_msg {
669 Message::Text(text) => {
670 if text == RECONNECTED {
671 log::debug!("Received Lighter WebSocket RECONNECTED sentinel");
672 self.book.reset_on_reconnect();
673 self.last_candles.clear();
675 self.reset_subscription_attempts_after_reconnect();
679 self.account_state_reconciler.reset();
680 return Some(NautilusWsMessage::Reconnected {
681 connection_epoch,
682 });
683 }
684
685 let ts_init = self.clock.get_time_ns();
686 let subscribed_topic = typed_subscribe_topic(&text);
687
688 if let Ok(frame) = serde_json::from_str::<LighterWsFrame>(&text) {
689 if let Some(topic) = subscribed_topic
690 && !self.complete_typed_subscription(topic, connection_epoch)
691 {
692 continue;
693 }
694 let messages = self
695 .handle_frame(frame, ts_init)
696 .into_iter()
697 .map(|msg| msg.with_connection_epoch(connection_epoch))
698 .collect();
699
700 if let Some(first) = self.dispatch_results(messages) {
701 return Some(first);
702 }
703 } else if let Ok(value) =
704 serde_json::from_str::<serde_json::Value>(&text)
705 {
706 if let Some(topic) = subscribed_topic {
711 self.complete_typed_subscription(topic, connection_epoch);
712 }
713
714 let (matched, msg) = self.handle_control_value(&value);
715 if let Some(first) = msg {
716 return Some(first.with_connection_epoch(connection_epoch));
717 }
718
719 if !matched {
720 log::warn!("Lighter WS unparsed frame: {value}");
725 return Some(NautilusWsMessage::Raw(value));
726 }
727 } else {
728 log::warn!("Lighter WS non-JSON text: {text}");
729 }
730 }
731 Message::Ping(data) => {
732 if let Some(ref client) = self.inner
733 && let Err(e) = client.send_pong(data.to_vec()).await {
734 log::error!("Error sending Lighter pong: {e}");
735 }
736 }
737 Message::Close(frame) => {
738 log::debug!("Received Lighter WebSocket close frame: {frame:?}");
739 return None;
740 }
741 _ => {}
742 }
743 }
744 else => {
745 log::debug!("Lighter handler shutting down: stream ended or command channel closed");
746 return None;
747 }
748 }
749 }
750 }
751
752 fn dispatch_results(
753 &mut self,
754 mut messages: Vec<NautilusWsMessage>,
755 ) -> Option<NautilusWsMessage> {
756 if messages.is_empty() {
757 return None;
758 }
759 let first = messages.remove(0);
760 for extra in messages {
761 self.pending_messages.push_back(extra);
762 }
763 Some(first)
764 }
765
766 async fn pump_pending_subscribes(&mut self) {
769 while self.inflight_subs.len() < SUBSCRIBE_INFLIGHT_MAX {
770 let Some((topic, generation)) = self.pending_subs.pop_front() else {
771 break;
772 };
773
774 let Some(attempt) = self.subscription_attempts.get(&topic) else {
775 continue;
776 };
777
778 if attempt.generation != generation {
779 continue;
780 }
781
782 let channel = attempt.channel.clone();
783 let auth = attempt.auth.clone();
784 self.inflight_subs.insert(topic, generation);
785 if let LighterWsChannel::OrderBook(market_index) = channel {
786 self.send_book_subscribe(market_index, generation);
787 } else if let Err(message) = self.dispatch_subscribe(channel, auth).await {
788 self.schedule_subscription_retry(topic, generation, &message);
789 }
790 }
791 }
792
793 fn queue_subscribe(
794 &mut self,
795 channel: LighterWsChannel,
796 auth: Option<SecretString>,
797 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
798 ) {
799 let topic = Ustr::from(channel.topic_key().as_str());
800
801 if let LighterWsChannel::OrderBook(market_index) = channel
802 && !self.book.writes.contains_key(&market_index)
803 && self
804 .book
805 .recovery
806 .get(&market_index)
807 .and_then(BookRecoveryState::current)
808 .is_some_and(|recovery| !recovery.is_accepted())
809 {
810 return;
811 }
812
813 if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
814 attempt.channel = channel;
815 if matches!(attempt.channel, LighterWsChannel::OrderBook(_))
816 && !self.inflight_subs.contains_key(&topic)
817 && !self
818 .pending_subs
819 .iter()
820 .any(|(pending, _)| *pending == topic)
821 {
822 self.pending_subs.push_back((topic, attempt.generation));
823 }
824
825 let effective_auth = attempt
826 .pending_auth
827 .as_ref()
828 .or(attempt.auth.as_ref())
829 .map(SecretString::expose_secret);
830 let auth_changed = auth
831 .as_ref()
832 .map(SecretString::expose_secret)
833 .is_some_and(|auth| Some(auth) != effective_auth);
834
835 if auth_changed {
836 let is_inflight = self.inflight_subs.get(&topic) == Some(&attempt.generation);
837 if is_inflight {
838 attempt.pending_auth = auth;
839 if let Some(response_tx) = response_tx {
840 attempt.pending_response_txs.push(response_tx);
841 }
842 } else {
843 debug_assert!(attempt.pending_auth.is_none());
844 debug_assert!(attempt.pending_response_txs.is_empty());
845 attempt.auth = auth;
846 if let Some(response_tx) = response_tx {
847 attempt.response_txs.push(response_tx);
848 }
849 }
850 } else if let Some(response_tx) = response_tx {
851 if auth.is_some() && attempt.pending_auth.is_some() {
852 attempt.pending_response_txs.push(response_tx);
853 } else {
854 attempt.response_txs.push(response_tx);
855 }
856 }
857 return;
858 }
859
860 let newly_pending = self.subscriptions.try_mark_subscribe(topic.as_str());
861 if !newly_pending
862 && auth.is_none()
863 && !self
864 .subscriptions
865 .pending_subscribe_topics()
866 .iter()
867 .any(|pending| pending == topic.as_str())
868 {
869 if let Some(response_tx) = response_tx {
870 let _ = response_tx.send(Ok(()));
871 }
872 return;
873 }
874
875 let generation = self.take_subscription_generation();
876 self.subscription_attempts.insert(
877 topic,
878 SubscriptionAttempt {
879 channel,
880 auth,
881 pending_auth: None,
882 generation,
883 retries: 0,
884 response_txs: response_tx.into_iter().collect(),
885 pending_response_txs: Vec::new(),
886 },
887 );
888 self.pending_subs.push_back((topic, generation));
889 }
890
891 fn queue_subscription_retry(&mut self, topic: Ustr, generation: u64) {
892 if self
893 .subscription_attempts
894 .get(&topic)
895 .is_some_and(|attempt| attempt.generation == generation)
896 && self.inflight_subs.get(&topic) != Some(&generation)
897 && !self
898 .pending_subs
899 .iter()
900 .any(|pending| *pending == (topic, generation))
901 {
902 self.pending_subs.push_back((topic, generation));
903 }
904 }
905
906 fn complete_subscription(&mut self, topic: &str, kind: CompletionKind) -> bool {
907 let topic = Ustr::from(topic);
908
909 if self.ignored_completions.get(&topic) == Some(&kind) {
914 self.ignored_completions.remove(&topic);
915 return false;
916 }
917
918 let Some(generation) = self.inflight_subs.get(&topic).copied() else {
919 return false;
920 };
921
922 if !self
923 .subscription_attempts
924 .get(&topic)
925 .is_some_and(|attempt| attempt.generation == generation)
926 {
927 return false;
928 }
929
930 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
931 && !self
932 .book
933 .expected
934 .get(&market_index)
935 .is_some_and(|expected| expected.0 == generation)
936 {
937 return false;
938 }
939
940 self.inflight_subs.remove(&topic);
941 self.subscriptions.confirm_subscribe(topic.as_str());
942
943 if kind == CompletionKind::ControlAck {
947 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
948 && let Some(expected) = self.book.expected.get(&market_index)
949 && expected.0 == generation
950 {
951 self.book.trailing.entry(market_index).or_insert(*expected);
952 }
953
954 self.ignored_completions
955 .insert(topic, CompletionKind::Typed);
956 }
957 let mut attempt = self
958 .subscription_attempts
959 .remove(&topic)
960 .expect("matching subscription attempt disappeared");
961 for response_tx in std::mem::take(&mut attempt.response_txs) {
962 let _ = response_tx.send(Ok(()));
963 }
964
965 if let Some(auth) = attempt.pending_auth.take() {
966 let generation = self.take_subscription_generation();
967 attempt.auth = Some(auth);
968 attempt.generation = generation;
969 attempt.retries = 0;
970 attempt.response_txs = std::mem::take(&mut attempt.pending_response_txs);
971 self.subscription_attempts.insert(topic, attempt);
972 self.pending_subs.push_back((topic, generation));
973 } else {
974 debug_assert!(attempt.pending_response_txs.is_empty());
975 }
976 true
977 }
978
979 fn retry_inflight_subscriptions(&mut self, message: &str) {
980 let mut inflight: Vec<(Ustr, u64)> = self
981 .inflight_subs
982 .iter()
983 .map(|(topic, generation)| (*topic, *generation))
984 .collect();
985 inflight.sort_unstable_by_key(|(topic, _)| *topic);
986
987 for (topic, generation) in inflight {
988 self.schedule_subscription_retry(topic, generation, message);
989 }
990 }
991
992 fn schedule_subscription_retry(&mut self, topic: Ustr, generation: u64, message: &str) {
993 if self.inflight_subs.get(&topic) != Some(&generation) {
994 return;
995 }
996
997 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
998 self.reject_book_subscription(
999 market_index,
1000 generation,
1001 LighterWsError::Network(message.into()),
1002 );
1003 return;
1004 }
1005
1006 self.inflight_subs.remove(&topic);
1007 self.subscriptions.mark_failure(topic.as_str());
1008
1009 let Some(attempt) = self.subscription_attempts.get_mut(&topic) else {
1010 return;
1011 };
1012
1013 if attempt.generation != generation {
1014 return;
1015 }
1016
1017 attempt.fold_pending_auth();
1018 if attempt.retries >= SUBSCRIBE_RETRY_MAX {
1019 self.fail_subscription_attempt(topic, message);
1020 return;
1021 }
1022
1023 attempt.retries += 1;
1024 let retry = attempt.retries;
1025 let next_generation = self.take_subscription_generation();
1026 let attempt = self
1027 .subscription_attempts
1028 .get_mut(&topic)
1029 .expect("subscription attempt disappeared before retry");
1030 attempt.generation = next_generation;
1031
1032 let delay = SUBSCRIBE_RETRY_BASE_BACKOFF.saturating_mul(1_u32 << (retry - 1));
1033 self.subscription_retries.push(Box::pin(async move {
1034 tokio::time::sleep(delay).await;
1035 (topic, next_generation)
1036 }));
1037 }
1038
1039 fn fail_subscription_attempt(&mut self, topic: Ustr, message: &str) {
1040 self.inflight_subs.remove(&topic);
1041 self.pending_subs.retain(|(pending, _)| *pending != topic);
1042 self.subscriptions.mark_unsubscribe(topic.as_str());
1043 self.subscriptions.confirm_unsubscribe(topic.as_str());
1044
1045 if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1046 let attempts = attempt.retries + 1;
1047 for response_tx in attempt
1048 .response_txs
1049 .into_iter()
1050 .chain(attempt.pending_response_txs)
1051 {
1052 let _ = response_tx.send(Err(format!(
1053 "subscription {topic} failed after {attempts} attempts: {message}",
1054 )));
1055 }
1056 }
1057 }
1058
1059 fn cancel_subscription_attempt(&mut self, topic: &str, message: &str) {
1060 let topic = Ustr::from(topic);
1061 self.inflight_subs.remove(&topic);
1062 self.pending_subs.retain(|(pending, _)| *pending != topic);
1063 if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1064 for response_tx in attempt
1065 .response_txs
1066 .into_iter()
1067 .chain(attempt.pending_response_txs)
1068 {
1069 let _ = response_tx.send(Err(format!("{message}: {topic}")));
1070 }
1071 }
1072 }
1073
1074 fn reset_subscription_attempts_after_reconnect(&mut self) {
1075 for topic in self.subscriptions.all_topics() {
1076 self.subscriptions.mark_failure(&topic);
1077 }
1078 self.pending_subs.clear();
1079 self.inflight_subs.clear();
1080 self.ignored_completions.clear();
1083
1084 let mut topics: Vec<Ustr> = self.subscription_attempts.keys().copied().collect();
1085 topics.sort_unstable();
1086 for topic in topics {
1087 let generation = self.take_subscription_generation();
1088 let attempt = self
1089 .subscription_attempts
1090 .get_mut(&topic)
1091 .expect("subscription attempt disappeared during reconnect");
1092 attempt.fold_pending_auth();
1093 attempt.generation = generation;
1094 attempt.retries = 0;
1095
1096 if order_book_market_index_from_topic(topic.as_str()).is_none() {
1097 self.pending_subs.push_back((topic, generation));
1098 }
1099 }
1100 }
1101
1102 fn take_subscription_generation(&mut self) -> u64 {
1103 let generation = self.next_subscription_generation;
1104 self.next_subscription_generation =
1105 self.next_subscription_generation.wrapping_add(1).max(1);
1106 generation
1107 }
1108
1109 fn handle_control_value(
1114 &mut self,
1115 value: &serde_json::Value,
1116 ) -> (bool, Option<NautilusWsMessage>) {
1117 if value.get("type").and_then(|v| v.as_str()) == Some("jsonapi/sendtxbatch")
1118 || value
1119 .get("id")
1120 .and_then(|v| v.as_str())
1121 .is_some_and(|id| id.starts_with(TX_BATCH_ID_PREFIX))
1122 {
1123 let body = value.get("error").unwrap_or(value);
1124 let id = value.get("id").and_then(|v| v.as_str());
1125 let code = body.get("code").and_then(|v| v.as_i64());
1126
1127 let (Some(id), Some(code)) = (id, code) else {
1128 log::warn!("Ignoring malformed Lighter sendTxBatch response");
1129 return (true, None);
1130 };
1131
1132 return (
1133 true,
1134 Some(NautilusWsMessage::SendTxBatchResult {
1135 connection_epoch: 0,
1136 id: id.to_string(),
1137 code,
1138 message: body
1139 .get("message")
1140 .and_then(|v| v.as_str())
1141 .unwrap_or("batch rejected")
1142 .to_string(),
1143 tx_hashes: value
1144 .get("tx_hash")
1145 .and_then(|v| serde_json::from_value(v.clone()).ok())
1146 .unwrap_or_default(),
1147 }),
1148 );
1149 }
1150
1151 if let Some(error) = already_subscribed_error(value) {
1152 self.confirm_already_subscribed(error);
1153 return (true, None);
1154 }
1155 let subscription_code = subscription_error_code(value);
1156
1157 if subscription_code == Some(LIGHTER_ERROR_CODE_WS_RATE_LIMITED) {
1158 self.retry_inflight_subscriptions(&format!(
1159 "venue rejected the WebSocket subscribe with code \
1160 {LIGHTER_ERROR_CODE_WS_RATE_LIMITED}",
1161 ));
1162 return (true, None);
1163 }
1164
1165 if subscription_code == Some(LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED) {
1166 self.handle_subscribe_failure(value);
1167 return (true, None);
1168 }
1169
1170 let kind = value.get("type").and_then(|v| v.as_str()).unwrap_or("");
1171
1172 match kind {
1173 CTRL_TYPE_CONNECTED => {
1174 log::debug!("Lighter WebSocket handshake complete");
1175 (true, None)
1176 }
1177 CTRL_TYPE_PING | CTRL_TYPE_PONG => (true, None),
1178 CTRL_TYPE_SEND_TX => {
1179 let raw_code = value.get("code").and_then(|v| v.as_u64());
1180 match raw_code {
1181 Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) => {
1182 log_integrator_not_approved();
1183 (
1184 true,
1185 Some(send_tx_rejected_from_value(
1186 value,
1187 SendTxRejectionSource::Ack,
1188 )),
1189 )
1190 }
1191 Some(200) => {
1192 log::debug!("Lighter WebSocket sendTx ack: {value}");
1193 let tx_hash = value
1194 .get("tx_hash")
1195 .and_then(|v| v.as_str())
1196 .map(str::to_string);
1197 (
1198 true,
1199 Some(NautilusWsMessage::SendTxAck {
1200 connection_epoch: 0,
1201 tx_hash,
1202 code: 200,
1203 }),
1204 )
1205 }
1206 Some(_) => (
1207 true,
1208 Some(send_tx_rejected_from_value(
1209 value,
1210 SendTxRejectionSource::Ack,
1211 )),
1212 ),
1213 None => {
1214 log::warn!(
1215 "Ignoring malformed Lighter sendTx response without numeric code: {value}",
1216 );
1217 (true, None)
1218 }
1219 }
1220 }
1221 CTRL_TYPE_SUBSCRIBED | CTRL_TYPE_UNSUBSCRIBED => {
1222 if let Some(topic) = value.get("channel").and_then(|v| v.as_str()) {
1223 if kind == CTRL_TYPE_SUBSCRIBED {
1224 self.complete_subscription(topic, CompletionKind::ControlAck);
1225 } else {
1226 let was_pending_unsubscribe = self
1227 .subscriptions
1228 .pending_unsubscribe_topics()
1229 .iter()
1230 .any(|pending| pending == topic);
1231 self.subscriptions.confirm_unsubscribe(topic);
1232
1233 if was_pending_unsubscribe {
1234 if let Some(market_index) = order_book_market_index_from_topic(topic) {
1236 self.book.clear_cached_order_book(market_index);
1237 }
1238
1239 if let Some(key) = candle_market_and_resolution_from_topic(topic) {
1240 self.last_candles.remove(&key);
1241 }
1242 }
1243 }
1244 }
1245 (true, None)
1246 }
1247 CTRL_TYPE_ERROR => {
1248 let code = value.get("code").and_then(|v| v.as_u64());
1249 if code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1250 log_integrator_not_approved();
1251 } else {
1252 log::warn!("Lighter WebSocket error frame: {value}");
1253 }
1254
1255 if is_sendtx_error_code(code) {
1256 (
1257 true,
1258 Some(send_tx_rejected_from_value(
1259 value,
1260 SendTxRejectionSource::BareError,
1261 )),
1262 )
1263 } else {
1264 (true, None)
1265 }
1266 }
1267 _ => {
1268 if let Some(error) = value.get("error") {
1269 let nested_code = error.get("code").and_then(|v| v.as_u64());
1270 if nested_code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1271 log_integrator_not_approved();
1272 } else {
1273 log::warn!("Lighter WebSocket error frame: {value}");
1274 }
1275 let rejected = is_sendtx_error_code(nested_code).then(|| {
1276 send_tx_rejected_from_nested_error(error, SendTxRejectionSource::BareError)
1277 });
1278 return (true, rejected);
1279 }
1280 (false, None)
1281 }
1282 }
1283 }
1284
1285 fn handle_subscribe_failure(&mut self, value: &serde_json::Value) {
1286 let message = format!(
1287 "venue rejected the WebSocket subscribe with code \
1288 {LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED}",
1289 );
1290
1291 let channel = value
1292 .get("channel")
1293 .and_then(serde_json::Value::as_str)
1294 .or_else(|| {
1295 value
1296 .get("error")
1297 .and_then(|e| e.get("channel"))
1298 .and_then(serde_json::Value::as_str)
1299 })
1300 .filter(|channel| !channel.is_empty());
1301
1302 let Some(channel) = channel else {
1303 log::warn!(
1304 "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel=none: retrying inflight subscriptions"
1305 );
1306 self.retry_inflight_subscriptions(&message);
1307 return;
1308 };
1309
1310 let topic = Ustr::from(channel.replace('/', ":").as_str());
1311
1312 let Some(generation) = self.inflight_subs.get(&topic).copied() else {
1313 log::warn!(
1314 "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel={channel}: no matching inflight subscription"
1315 );
1316 return;
1317 };
1318
1319 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1320 self.reject_book_subscription(
1321 market_index,
1322 generation,
1323 LighterWsError::Client(message),
1324 );
1325 } else {
1326 self.fail_subscription_attempt(topic, &message);
1327 }
1328 }
1329
1330 fn confirm_already_subscribed(&mut self, error: &serde_json::Value) {
1331 let Some(topic) = error
1332 .get("message")
1333 .and_then(|value| value.as_str())
1334 .and_then(|message| message.strip_prefix("Already Subscribed to : "))
1335 else {
1336 log::debug!(
1337 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1338 );
1339 return;
1340 };
1341
1342 if !self.complete_subscription(topic, CompletionKind::AlreadySubscribed) {
1343 log::debug!(
1344 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1345 );
1346 return;
1347 }
1348
1349 log::debug!(
1350 "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}, topic={topic}",
1351 );
1352 }
1353
1354 fn handle_frame(
1355 &mut self,
1356 frame: LighterWsFrame,
1357 ts_init: UnixNanos,
1358 ) -> Vec<NautilusWsMessage> {
1359 match frame {
1360 LighterWsFrame::OrderBookSnapshot {
1361 channel,
1362 order_book,
1363 timestamp,
1364 ..
1365 } => self.handle_order_book(channel, &order_book, timestamp, true, ts_init),
1366 LighterWsFrame::OrderBook {
1367 channel,
1368 order_book,
1369 timestamp,
1370 ..
1371 } => self.handle_order_book(channel, &order_book, timestamp, false, ts_init),
1372 LighterWsFrame::TickerSnapshot {
1373 channel,
1374 ticker,
1375 timestamp,
1376 ..
1377 }
1378 | LighterWsFrame::Ticker {
1379 channel,
1380 ticker,
1381 timestamp,
1382 ..
1383 } => self.handle_ticker(channel, &ticker, timestamp, ts_init),
1384 LighterWsFrame::TradeSnapshot {
1385 trades,
1386 liquidation_trades,
1387 ..
1388 }
1389 | LighterWsFrame::Trade {
1390 trades,
1391 liquidation_trades,
1392 ..
1393 } => self.handle_trades(&trades, &liquidation_trades, ts_init),
1394 LighterWsFrame::AccountOrders { ref orders, .. }
1395 | LighterWsFrame::AccountAllOrders { ref orders, .. } => {
1396 if self.exec_account.is_none() {
1397 return raw_message(&frame);
1398 }
1399 let mut msgs = self.handle_account_orders(orders, ts_init);
1400 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1401 AccountStream::Orders,
1402 ));
1403 msgs
1404 }
1405 LighterWsFrame::AccountAllTradesSnapshot { .. } => {
1413 if self.exec_account.is_none() {
1414 return raw_message(&frame);
1415 }
1416 log::debug!(
1417 "Skipping Lighter account_all_trades snapshot frame; \
1418 reconcile historical fills via HTTP",
1419 );
1420 vec![NautilusWsMessage::AccountStreamFirstFrame(
1421 AccountStream::Trades,
1422 )]
1423 }
1424 LighterWsFrame::AccountAllTrades { ref trades, .. } => {
1425 if self.exec_account.is_none() {
1426 return raw_message(&frame);
1427 }
1428 let mut msgs = self.handle_account_trades(trades.values().flatten(), ts_init);
1429 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1430 AccountStream::Trades,
1431 ));
1432 msgs
1433 }
1434 LighterWsFrame::AccountAllPositionsSnapshot { ref positions, .. } => {
1435 if self.exec_account.is_none() {
1436 return raw_message(&frame);
1437 }
1438 let mut msgs =
1439 self.handle_account_positions(positions, ts_init, PositionFrameType::Snapshot);
1440 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1441 AccountStream::Positions,
1442 ));
1443 msgs
1444 }
1445 LighterWsFrame::AccountAllPositions { ref positions, .. } => {
1446 if self.exec_account.is_none() {
1447 return raw_message(&frame);
1448 }
1449 self.handle_account_positions(positions, ts_init, PositionFrameType::Update)
1450 }
1451 LighterWsFrame::AccountAllAssets {
1452 ref assets,
1453 timestamp,
1454 ..
1455 } => {
1456 if self.exec_account.is_none() {
1457 return raw_message(&frame);
1458 }
1459 let mut msgs = self.handle_account_assets(assets, timestamp, ts_init);
1460 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1461 AccountStream::Assets,
1462 ));
1463 msgs
1464 }
1465 LighterWsFrame::UserStats {
1466 ref stats,
1467 timestamp,
1468 ..
1469 } => {
1470 if self.exec_account.is_none() {
1471 return raw_message(&frame);
1472 }
1473 let mut msgs = self.handle_user_stats(stats, timestamp, ts_init);
1474 msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1475 AccountStream::UserStats,
1476 ));
1477 msgs
1478 }
1479 LighterWsFrame::MarketStats {
1480 ref market_stats,
1481 timestamp,
1482 ..
1483 } => self.handle_market_stats(market_stats, timestamp, ts_init),
1484 LighterWsFrame::SpotMarketStats {
1485 ref spot_market_stats,
1486 timestamp,
1487 ..
1488 } => self.handle_spot_market_stats(spot_market_stats, timestamp, ts_init),
1489 LighterWsFrame::CandleSnapshot {
1490 channel,
1491 ref candles,
1492 ..
1493 }
1494 | LighterWsFrame::Candle {
1495 channel,
1496 ref candles,
1497 ..
1498 } => self.handle_candles(channel, candles, ts_init),
1499 LighterWsFrame::Height { .. } => raw_message(&frame),
1500 }
1501 }
1502
1503 fn handle_order_book(
1504 &mut self,
1505 channel: Ustr,
1506 book: &LighterWsOrderBook,
1507 timestamp: u64,
1508 is_snapshot: bool,
1509 ts_init: UnixNanos,
1510 ) -> Vec<NautilusWsMessage> {
1511 let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1512 log::debug!("Lighter order_book frame missing market index in channel '{channel}'");
1513 return Vec::new();
1514 };
1515
1516 let Some(instrument) = self.instruments.get(&market_index) else {
1517 log::debug!("No instrument cached for Lighter market_index={market_index}");
1518 return Vec::new();
1519 };
1520
1521 match self.book.validate_sequence(market_index, book, is_snapshot) {
1522 BookSequenceOutcome::Accept => self.book.apply(
1523 market_index,
1524 instrument,
1525 book,
1526 timestamp,
1527 is_snapshot,
1528 ts_init,
1529 ),
1530 BookSequenceOutcome::Suppress => Vec::new(),
1531 BookSequenceOutcome::Recover => {
1532 self.start_book_recovery(market_index);
1533 Vec::new()
1534 }
1535 }
1536 }
1537
1538 fn queue_book_replacement(&mut self, market_index: i64, write: BookWrite) {
1539 if write.cancel.is_cancelled() || !self.order_book_stream_is_referenced(market_index) {
1540 return;
1541 }
1542
1543 self.book.clear_cached_order_book(market_index);
1544 self.book.expected.remove(&market_index);
1545 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1546 self.retire_book_attempt(&topic);
1547 self.subscriptions.mark_failure(&topic);
1548 self.book.writes.insert(market_index, write);
1549 self.queue_subscribe(LighterWsChannel::OrderBook(market_index), None, None);
1550 }
1551
1552 fn retire_book_attempt(&mut self, topic: &str) {
1553 let topic = Ustr::from(topic);
1554 self.inflight_subs.remove(&topic);
1555 self.pending_subs.retain(|(pending, _)| *pending != topic);
1556 if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
1557 attempt.generation = self.next_subscription_generation;
1559 self.next_subscription_generation =
1560 self.next_subscription_generation.wrapping_add(1).max(1);
1561 self.pending_subs.push_back((topic, attempt.generation));
1562 }
1563 }
1564
1565 fn reject_book_subscription(
1566 &mut self,
1567 market_index: i64,
1568 generation: u64,
1569 error: LighterWsError,
1570 ) {
1571 if !self
1572 .book
1573 .expected
1574 .get(&market_index)
1575 .is_some_and(|expected| expected.0 == generation)
1576 || self
1577 .book
1578 .recovery
1579 .get(&market_index)
1580 .and_then(BookRecoveryState::current)
1581 .is_some_and(|recovery| recovery.gate.lock().is_closed())
1582 {
1583 return;
1584 }
1585
1586 self.reject_book(market_index, error);
1587 }
1588
1589 fn reject_book(&mut self, market_index: i64, error: LighterWsError) {
1590 self.book.expected.remove(&market_index);
1591 self.book.clear_cached_order_book(market_index);
1592 let topic = Ustr::from(
1593 LighterWsChannel::OrderBook(market_index)
1594 .topic_key()
1595 .as_str(),
1596 );
1597
1598 let waiting = matches!(error, LighterWsError::Client(_))
1599 && self
1600 .subscription_attempts
1601 .get(&topic)
1602 .is_some_and(|attempt| {
1603 !attempt.response_txs.is_empty() || !attempt.pending_response_txs.is_empty()
1604 });
1605
1606 if waiting {
1607 self.book.initial.remove(&market_index);
1610 self.cancel_subscription_attempt(&topic, &error.to_string());
1611 log::error!(
1612 "Lighter book subscription rejected for market_index={market_index}: {error}"
1613 );
1614 }
1615
1616 if let Some(recovery) = self
1617 .book
1618 .recovery
1619 .get(&market_index)
1620 .and_then(BookRecoveryState::current)
1621 && !recovery.is_accepted()
1622 {
1623 self.inflight_subs.remove(&topic);
1625 recovery.gate.lock().close();
1626 recovery
1627 .outcome
1628 .send_replace(BookRecoveryOutcome::Rejected(error));
1629 } else if !waiting {
1630 self.start_book_recovery(market_index);
1631 }
1632 }
1633
1634 fn start_book_recovery(&mut self, market_index: i64) {
1635 if !self.order_book_stream_is_referenced(market_index) {
1636 return;
1637 }
1638
1639 let Some(cmd_tx) = self.cmd_tx.clone() else {
1640 return;
1641 };
1642
1643 let Some(recovery) = self.book.recovery.entry(market_index).or_default().claim() else {
1644 return;
1645 };
1646
1647 recovery.gate.lock().close();
1648 self.book.initial.remove(&market_index);
1649 self.book.expected.remove(&market_index);
1650 self.book.clear_cached_order_book(market_index);
1651 let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1652 self.inflight_subs.remove(&Ustr::from(topic.as_str()));
1653 self.pending_subs
1654 .retain(|(pending, _)| pending.as_str() != topic);
1655 self.book.work.push(recovery::recover(
1656 market_index,
1657 recovery,
1658 cmd_tx,
1659 self.book_snapshot_timeout,
1660 ));
1661 }
1662
1663 fn cancel_orphaned_book_recovery(&mut self, market_index: i64) {
1666 if !self.order_book_stream_is_referenced(market_index) {
1667 self.book.recovery.remove(&market_index);
1668 }
1669 }
1670
1671 fn complete_typed_subscription(&mut self, topic: &str, epoch: u64) -> bool {
1672 if let Some(market_index) = order_book_market_index_from_topic(topic)
1673 && self
1674 .book
1675 .expected
1676 .get(&market_index)
1677 .is_some_and(|expected| expected.1 != epoch)
1678 {
1679 return false;
1680 }
1681
1682 if !self.book_snapshot_matches(topic, epoch) {
1683 if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1684 self.ignored_completions.remove(&Ustr::from(topic));
1685 }
1686
1687 return false;
1688 }
1689
1690 self.complete_subscription(topic, CompletionKind::Typed);
1691 true
1692 }
1693
1694 fn book_snapshot_matches(&mut self, topic: &str, epoch: u64) -> bool {
1695 let Some(market_index) = order_book_market_index_from_topic(topic) else {
1696 return true;
1697 };
1698
1699 let source =
1700 if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1701 self.book.trailing.remove(&market_index)
1702 } else {
1703 self.inflight_subs
1704 .get(&Ustr::from(topic))
1705 .map(|generation| (*generation, epoch))
1706 };
1707
1708 source.is_some_and(|source| {
1709 source.1 == epoch && Some(source) == self.book.expected.get(&market_index).copied()
1710 })
1711 }
1712
1713 fn send_book_subscribe(&mut self, market_index: i64, generation: u64) {
1714 let write = self.book.writes.remove(&market_index);
1715 let cancel = write
1716 .as_ref()
1717 .map_or_else(CancellationToken::new, |write| write.cancel.clone());
1718 if write.is_none() {
1719 self.book.initial.insert(
1720 market_index,
1721 PendingSnapshot {
1722 deadline: None,
1723 cancel: cancel.clone(),
1724 gate: SnapshotGate::default(),
1725 },
1726 );
1727 }
1728
1729 let client = self.inner.clone();
1730 let subscriptions = self.subscriptions.clone();
1731 let book_snapshot_timeout = self.book_snapshot_timeout;
1732 self.book.work.push(recovery::subscribe(
1733 market_index,
1734 generation,
1735 cancel,
1736 write,
1737 client,
1738 subscriptions,
1739 book_snapshot_timeout,
1740 ));
1741 }
1742
1743 fn complete_book_send(
1744 &mut self,
1745 market_index: i64,
1746 generation: u64,
1747 cancel: CancellationToken,
1748 write: Option<BookWrite>,
1749 result: Result<u64, LighterWsError>,
1750 ) {
1751 let topic = Ustr::from(
1752 LighterWsChannel::OrderBook(market_index)
1753 .topic_key()
1754 .as_str(),
1755 );
1756
1757 if cancel.is_cancelled()
1758 || self.inflight_subs.get(&topic) != Some(&generation)
1759 || !self.order_book_stream_is_referenced(market_index)
1760 {
1761 return;
1762 }
1763
1764 match result {
1765 Ok(epoch) => {
1766 self.book.expected.insert(market_index, (generation, epoch));
1767
1768 if let Some(write) = write {
1769 write.gate.open();
1770 let _ = write.completion.send(Ok(()));
1771 } else {
1772 self.book.work.push(recovery::wait_for_snapshot(
1773 market_index,
1774 cancel,
1775 self.book_snapshot_timeout,
1776 ));
1777 }
1778 }
1779 Err(e) => {
1780 if let Some(write) = write {
1781 self.inflight_subs.remove(&topic);
1783 let _ = write.completion.send(Err(e));
1784 } else {
1785 self.reject_book(market_index, e);
1786 }
1787 }
1788 }
1789 }
1790
1791 fn order_book_stream_is_referenced(&self, market_index: i64) -> bool {
1792 let channel = LighterWsChannel::OrderBook(market_index);
1793 self.subscriptions.get_reference_count(&channel.topic_key()) > 0
1794 && (self.book.delta_subs.contains(&market_index)
1795 || self.book.depth_subs.contains(&market_index))
1796 }
1797
1798 fn handle_ticker(
1799 &self,
1800 channel: Ustr,
1801 ticker: &super::messages::LighterTicker,
1802 timestamp: u64,
1803 ts_init: UnixNanos,
1804 ) -> Vec<NautilusWsMessage> {
1805 let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1811 log::debug!("Lighter ticker frame missing market index in channel '{channel}'");
1812 return Vec::new();
1813 };
1814
1815 let Some(instrument) = self.instruments.get(&market_index) else {
1816 log::debug!("No instrument cached for Lighter ticker market_index={market_index}");
1817 return Vec::new();
1818 };
1819
1820 match parse_ws_quote_tick(ticker, instrument, timestamp, ts_init) {
1821 Ok(Some(quote)) => vec![NautilusWsMessage::Quote(quote)],
1822 Ok(None) => {
1823 log::debug!(
1824 "Skipping Lighter ticker for market_index={market_index}: one-sided book",
1825 );
1826 Vec::new()
1827 }
1828 Err(e) => {
1829 log::error!("Error parsing Lighter ticker frame: {e}");
1830 Vec::new()
1831 }
1832 }
1833 }
1834
1835 fn handle_trades(
1836 &self,
1837 trades: &[crate::http::models::LighterTrade],
1838 liquidation_trades: &[crate::http::models::LighterTrade],
1839 ts_init: UnixNanos,
1840 ) -> Vec<NautilusWsMessage> {
1841 let Some(market_index) = trades
1845 .first()
1846 .or_else(|| liquidation_trades.first())
1847 .map(|t| t.market_id)
1848 else {
1849 return Vec::new();
1850 };
1851
1852 let Some(instrument) = self.instruments.get(&market_index) else {
1853 log::debug!("No instrument cached for Lighter trade market_index={market_index}");
1854 return Vec::new();
1855 };
1856
1857 let mut ticks = Vec::with_capacity(trades.len() + liquidation_trades.len());
1858 for trade in trades.iter().chain(liquidation_trades.iter()) {
1859 match parse_ws_trade_tick(trade, instrument, ts_init) {
1860 Ok(tick) => ticks.push(tick),
1861 Err(e) => log::error!("Error parsing Lighter trade tick: {e}"),
1862 }
1863 }
1864
1865 if ticks.is_empty() {
1866 Vec::new()
1867 } else {
1868 vec![NautilusWsMessage::Trades(ticks)]
1869 }
1870 }
1871
1872 fn handle_market_stats(
1873 &self,
1874 payload: &super::messages::LighterMarketStatsPayload,
1875 timestamp: u64,
1876 ts_init: UnixNanos,
1877 ) -> Vec<NautilusWsMessage> {
1878 match payload {
1879 super::messages::LighterMarketStatsPayload::All(stats) => stats
1880 .values()
1881 .flat_map(|stats| self.handle_one_market_stats(stats, timestamp, ts_init))
1882 .collect(),
1883 super::messages::LighterMarketStatsPayload::One(stats) => {
1884 self.handle_one_market_stats(stats, timestamp, ts_init)
1885 }
1886 }
1887 }
1888
1889 fn handle_one_market_stats(
1890 &self,
1891 stats: &super::messages::LighterMarketStats,
1892 timestamp: u64,
1893 ts_init: UnixNanos,
1894 ) -> Vec<NautilusWsMessage> {
1895 let Some(instrument) = self.instruments.get(&stats.market_id) else {
1896 log::debug!(
1897 "No instrument cached for Lighter market_stats market_id={}",
1898 stats.market_id,
1899 );
1900 return Vec::new();
1901 };
1902
1903 let mut messages = Vec::with_capacity(3);
1904
1905 match parse_ws_mark_price_update(stats, instrument, timestamp, ts_init) {
1906 Ok(mark_price) => messages.push(NautilusWsMessage::MarkPrice(mark_price)),
1907 Err(e) => log::error!("Error parsing Lighter mark price: {e}"),
1908 }
1909
1910 match parse_ws_index_price_update(stats, instrument, timestamp, ts_init) {
1911 Ok(index_price) => messages.push(NautilusWsMessage::IndexPrice(index_price)),
1912 Err(e) => log::error!("Error parsing Lighter index price: {e}"),
1913 }
1914
1915 match parse_ws_funding_rate_update(stats, instrument, timestamp, ts_init) {
1916 Ok(funding_rate) => messages.push(NautilusWsMessage::FundingRate(funding_rate)),
1917 Err(e) => log::error!("Error parsing Lighter funding rate: {e}"),
1918 }
1919
1920 messages
1921 }
1922
1923 fn handle_spot_market_stats(
1924 &self,
1925 payload: &super::messages::LighterSpotMarketStatsPayload,
1926 timestamp: u64,
1927 ts_init: UnixNanos,
1928 ) -> Vec<NautilusWsMessage> {
1929 match payload {
1930 super::messages::LighterSpotMarketStatsPayload::All(stats) => stats
1931 .values()
1932 .filter_map(|stats| self.handle_one_spot_market_stats(stats, timestamp, ts_init))
1933 .collect(),
1934 super::messages::LighterSpotMarketStatsPayload::One(stats) => self
1935 .handle_one_spot_market_stats(stats, timestamp, ts_init)
1936 .into_iter()
1937 .collect(),
1938 }
1939 }
1940
1941 fn handle_one_spot_market_stats(
1942 &self,
1943 stats: &super::messages::LighterSpotMarketStats,
1944 timestamp: u64,
1945 ts_init: UnixNanos,
1946 ) -> Option<NautilusWsMessage> {
1947 let Some(instrument) = self.instruments.get(&stats.market_id) else {
1948 log::debug!(
1949 "No instrument cached for Lighter spot_market_stats market_id={}",
1950 stats.market_id,
1951 );
1952 return None;
1953 };
1954
1955 match parse_ws_spot_index_price_update(stats, instrument, timestamp, ts_init) {
1956 Ok(index_price) => Some(NautilusWsMessage::IndexPrice(index_price)),
1957 Err(e) => {
1958 log::error!("Error parsing Lighter spot index price: {e}");
1959 None
1960 }
1961 }
1962 }
1963
1964 fn handle_candles(
1965 &mut self,
1966 channel: Ustr,
1967 candles: &[LighterWsCandle],
1968 ts_init: UnixNanos,
1969 ) -> Vec<NautilusWsMessage> {
1970 let Some((market_index, resolution)) =
1971 candle_market_and_resolution_from_topic(channel.as_str())
1972 else {
1973 log::warn!("Lighter candle frame with unparsable channel `{channel}`");
1974 return Vec::new();
1975 };
1976
1977 let Some(instrument) = self.instruments.get(&market_index) else {
1978 log::debug!("No instrument cached for Lighter candle market_index={market_index}");
1979 return Vec::new();
1980 };
1981
1982 let key = (market_index, resolution);
1983 let mut emitted = Vec::new();
1984
1985 for candle in candles {
1986 let previous = self.last_candles.get(&key).cloned();
1987 match previous {
1988 None => {}
1989 Some(prev) if candle.t > prev.t => {
1990 match parse_ws_bar(instrument, &prev, resolution, ts_init) {
1992 Ok(bar) => emitted.push(NautilusWsMessage::Bar(bar)),
1993 Err(e) => log::error!("Error parsing Lighter candle bar: {e}"),
1994 }
1995 }
1996 Some(prev) if candle.t < prev.t => continue,
1997 Some(_) => {}
1998 }
1999 self.last_candles.insert(key, candle.clone());
2000 }
2001
2002 emitted
2003 }
2004
2005 fn handle_account_orders(
2006 &self,
2007 orders_by_market: &AHashMap<Ustr, Vec<LighterOrder>>,
2008 _ts_init: UnixNanos,
2009 ) -> Vec<NautilusWsMessage> {
2010 if self.exec_account.is_none() {
2011 log::debug!("Lighter account_orders frame skipped: no execution context set");
2012 return Vec::new();
2013 }
2014
2015 let mut reports = Vec::new();
2016
2017 for orders in orders_by_market.values() {
2018 for order in orders {
2019 if !self.instruments.contains_key(&order.market_index) {
2020 log::debug!(
2021 "No instrument cached for Lighter order market_index={}",
2022 order.market_index,
2023 );
2024 continue;
2025 }
2026
2027 reports.push(ExecutionReport::Order(order.clone()));
2028 }
2029 }
2030
2031 if reports.is_empty() {
2032 Vec::new()
2033 } else {
2034 vec![NautilusWsMessage::ExecutionReports(reports)]
2035 }
2036 }
2037
2038 fn handle_account_trades<'a>(
2039 &self,
2040 trades: impl IntoIterator<Item = &'a LighterTrade>,
2041 _ts_init: UnixNanos,
2042 ) -> Vec<NautilusWsMessage> {
2043 let Some((_account_id, account_index)) = self.exec_account else {
2044 log::debug!("Lighter account_trades frame skipped: no execution context set");
2045 return Vec::new();
2046 };
2047
2048 let mut reports = Vec::new();
2049
2050 for trade in trades {
2051 if !self.instruments.contains_key(&trade.market_id) {
2052 log::debug!(
2053 "No instrument cached for Lighter account trade market_id={}",
2054 trade.market_id,
2055 );
2056 continue;
2057 }
2058
2059 if trade.bid_account_id != account_index && trade.ask_account_id != account_index {
2063 continue;
2064 }
2065
2066 reports.push(ExecutionReport::Fill(trade.clone()));
2067 }
2068
2069 if reports.is_empty() {
2070 Vec::new()
2071 } else {
2072 vec![NautilusWsMessage::ExecutionReports(reports)]
2073 }
2074 }
2075
2076 fn handle_account_positions(
2077 &self,
2078 positions: &AHashMap<Ustr, LighterPosition>,
2079 ts_init: UnixNanos,
2080 frame_type: PositionFrameType,
2081 ) -> Vec<NautilusWsMessage> {
2082 let Some((account_id, _)) = self.exec_account else {
2083 log::debug!("Lighter account_positions frame skipped: no execution context set");
2084 return Vec::new();
2085 };
2086
2087 let ts_event = ts_init;
2090
2091 let mut reports = Vec::new();
2092 let mut skipped_market_ids = Vec::new();
2093 let mut closed_market_ids = Vec::new();
2094
2095 for position in positions.values() {
2096 if position.position.is_zero() {
2097 if matches!(frame_type, PositionFrameType::Update) {
2098 closed_market_ids.push(position.market_id);
2099 }
2100 continue;
2101 }
2102
2103 let Some(instrument) = self.instruments.get(&position.market_id) else {
2104 log::debug!(
2105 "No instrument cached for Lighter position market_id={}",
2106 position.market_id,
2107 );
2108
2109 skipped_market_ids.push(position.market_id);
2110 continue;
2111 };
2112
2113 match parse_ws_position_status_report(
2114 position, instrument, account_id, ts_event, ts_init,
2115 ) {
2116 Ok(report) => reports.push(report),
2117 Err(e) => {
2118 skipped_market_ids.push(position.market_id);
2119 log::error!("Error parsing Lighter position status report: {e}");
2120 }
2121 }
2122 }
2123
2124 match frame_type {
2125 PositionFrameType::Snapshot => {
2126 vec![NautilusWsMessage::PositionSnapshot {
2128 reports,
2129 skipped_market_ids,
2130 }]
2131 }
2132 PositionFrameType::Update => vec![NautilusWsMessage::PositionUpdate {
2133 reports,
2134 closed_market_ids,
2135 skipped_market_ids,
2136 }],
2137 }
2138 }
2139
2140 fn handle_account_assets(
2141 &self,
2142 assets: &AHashMap<Ustr, LighterAsset>,
2143 timestamp_ms: u64,
2144 ts_init: UnixNanos,
2145 ) -> Vec<NautilusWsMessage> {
2146 let Some((account_id, _)) = self.exec_account else {
2147 log::debug!("Lighter account_assets frame skipped: no execution context set");
2148 return Vec::new();
2149 };
2150
2151 let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2152 Ok(ts) => ts,
2153 Err(e) => {
2154 log::error!("Invalid Lighter account_assets timestamp {timestamp_ms}: {e}");
2155 return Vec::new();
2156 }
2157 };
2158
2159 self.account_state_reconciler.update_assets(assets);
2160 self.emit_unified_account_state(account_id, ts_event, ts_init)
2161 }
2162
2163 fn handle_user_stats(
2164 &self,
2165 stats: &LighterUserStats,
2166 timestamp_ms: u64,
2167 ts_init: UnixNanos,
2168 ) -> Vec<NautilusWsMessage> {
2169 let Some((account_id, _)) = self.exec_account else {
2170 log::debug!("Lighter user_stats frame skipped: no execution context set");
2171 return Vec::new();
2172 };
2173
2174 let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2175 Ok(ts) => ts,
2176 Err(e) => {
2177 log::error!("Invalid Lighter user_stats timestamp {timestamp_ms}: {e}");
2178 return Vec::new();
2179 }
2180 };
2181
2182 self.account_state_reconciler.update_user_stats(stats);
2183 self.emit_unified_account_state(account_id, ts_event, ts_init)
2184 }
2185
2186 fn emit_unified_account_state(
2191 &self,
2192 account_id: AccountId,
2193 ts_event: UnixNanos,
2194 ts_init: UnixNanos,
2195 ) -> Vec<NautilusWsMessage> {
2196 match self
2197 .account_state_reconciler
2198 .build_state(account_id, ts_event, ts_init)
2199 {
2200 Some(Ok(state)) => vec![NautilusWsMessage::AccountState(Box::new(state))],
2201 Some(Err(e)) => {
2202 log::error!("Error building unified Lighter account state: {e}");
2203 Vec::new()
2204 }
2205 None => Vec::new(),
2206 }
2207 }
2208}
2209
2210#[derive(Clone, Copy)]
2211enum PositionFrameType {
2212 Snapshot,
2213 Update,
2214}
2215
2216fn raw_message(frame: &LighterWsFrame) -> Vec<NautilusWsMessage> {
2217 let value = serde_json::to_value(frame).unwrap_or(serde_json::Value::Null);
2218 vec![NautilusWsMessage::Raw(value)]
2219}
2220
2221fn is_sendtx_error_code(code: Option<u64>) -> bool {
2224 code.is_some_and(|c| LIGHTER_ERROR_CODE_TX_RANGE.contains(&c))
2225}
2226
2227fn already_subscribed_error(value: &serde_json::Value) -> Option<&serde_json::Value> {
2228 if value.get("code").and_then(|code| code.as_u64())
2229 == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2230 {
2231 Some(value)
2232 } else {
2233 value.get("error").filter(|e| {
2234 e.get("code").and_then(|code| code.as_u64())
2235 == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2236 })
2237 }
2238}
2239
2240fn subscription_error_code(value: &serde_json::Value) -> Option<u64> {
2241 value
2242 .get("code")
2243 .and_then(|code| code.as_u64())
2244 .or_else(|| {
2245 value
2246 .get("error")
2247 .and_then(|e| e.get("code"))
2248 .and_then(|code| code.as_u64())
2249 })
2250}
2251
2252fn typed_subscribe_topic(text: &str) -> Option<&str> {
2253 let header = serde_json::from_str::<LighterWsFrameHeader<'_>>(text).ok()?;
2254 header
2255 .kind
2256 .starts_with("subscribed/")
2257 .then_some(header.channel?)
2258}
2259
2260fn send_tx_rejected_from_value(
2263 value: &serde_json::Value,
2264 source: SendTxRejectionSource,
2265) -> NautilusWsMessage {
2266 let code = value.get("code").and_then(|v| v.as_i64());
2267 let message = value
2268 .get("message")
2269 .and_then(|v| v.as_str())
2270 .unwrap_or("")
2271 .to_string();
2272 let tx_hash = value
2273 .get("tx_hash")
2274 .and_then(|v| v.as_str())
2275 .map(str::to_string);
2276 NautilusWsMessage::SendTxRejected {
2277 connection_epoch: 0,
2278 source,
2279 code,
2280 message,
2281 tx_hash,
2282 }
2283}
2284
2285fn send_tx_rejected_from_nested_error(
2288 error: &serde_json::Value,
2289 source: SendTxRejectionSource,
2290) -> NautilusWsMessage {
2291 let code = error.get("code").and_then(|v| v.as_i64());
2292 let message = error
2293 .get("message")
2294 .and_then(|v| v.as_str())
2295 .unwrap_or("")
2296 .to_string();
2297 NautilusWsMessage::SendTxRejected {
2298 connection_epoch: 0,
2299 source,
2300 code,
2301 message,
2302 tx_hash: None,
2303 }
2304}
2305
2306fn log_integrator_not_approved() {
2307 log::error!(
2308 "Lighter venue rejected with code {LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED} \
2309 'integrator is not approved'.\n\
2310 Tagged orders require Nautilus integrator approval. \
2311 See: {LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL}",
2312 );
2313}
2314
2315fn market_index_from_topic(topic: &str) -> Option<i64> {
2316 let (_, rest) = topic.split_once(':')?;
2317 rest.parse::<i64>().ok()
2318}
2319
2320fn candle_market_and_resolution_from_topic(topic: &str) -> Option<(i64, LighterCandleResolution)> {
2321 let (channel, rest) = topic.split_once(':')?;
2322 if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::Candle) {
2323 return None;
2324 }
2325 let (market, res) = rest.split_once(':')?;
2326 let market_index = market.parse::<i64>().ok()?;
2327 let resolution = res.parse::<LighterCandleResolution>().ok()?;
2328 Some((market_index, resolution))
2329}
2330
2331fn order_book_market_index_from_topic(topic: &str) -> Option<i64> {
2332 let (channel, rest) = topic.split_once(':')?;
2333 if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::OrderBook) {
2334 return None;
2335 }
2336
2337 rest.parse::<i64>().ok()
2338}
2339
2340pub(crate) fn should_retry_lighter_ws_error(error: &LighterWsError) -> bool {
2341 match error {
2342 LighterWsError::Network(_) => true,
2343 LighterWsError::Transport(send_error) => match send_error {
2348 SendError::Timeout => true,
2349 SendError::InvalidInput(_)
2350 | SendError::BufferFull
2351 | SendError::Closed
2352 | SendError::ConnectionChanged
2353 | SendError::BrokenPipe(_)
2354 | SendError::WriteTimeout => false,
2355 },
2356 LighterWsError::Authentication(_)
2357 | LighterWsError::Parse(_)
2358 | LighterWsError::Client(_)
2359 | LighterWsError::SendTxOutcomeUnknown(_) => false,
2360 }
2361}
2362
2363pub(crate) fn create_lighter_ws_timeout_error(_msg: String) -> LighterWsError {
2364 LighterWsError::Transport(SendError::Timeout)
2367}
2368
2369#[cfg(test)]
2370mod tests {
2371 use std::time::Duration;
2372
2373 use futures_util::FutureExt;
2374 use log::{Level, LevelFilter, Log, Metadata, Record};
2375 use nautilus_model::{
2376 enums::{AccountType, BookAction, RecordFlag},
2377 identifiers::{InstrumentId, Symbol, Venue},
2378 instruments::{CryptoPerpetual, CurrencyPair},
2379 types::{Currency, Money, Price, Quantity},
2380 };
2381 use parking_lot::Mutex;
2382 use rstest::rstest;
2383 use rust_decimal::Decimal;
2384 use serde_json::json;
2385
2386 use super::*;
2387 use crate::{
2388 common::enums::{LighterCandleResolution, LighterTxType},
2389 websocket::messages::{LighterMarketSelection, LighterWsCandle, LighterWsChannel},
2390 };
2391
2392 const SECRET_MARKER: &str = "426426426";
2393
2394 struct OutboundLogCapture {
2395 messages: Mutex<Vec<String>>,
2396 }
2397
2398 static OUTBOUND_LOG_CAPTURE: OutboundLogCapture = OutboundLogCapture {
2399 messages: Mutex::new(Vec::new()),
2400 };
2401
2402 impl OutboundLogCapture {
2403 fn clear(&self) {
2404 self.messages.lock().clear();
2405 }
2406
2407 fn messages(&self) -> Vec<String> {
2408 self.messages.lock().clone()
2409 }
2410 }
2411
2412 impl Log for OutboundLogCapture {
2413 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
2414 matches!(metadata.level(), Level::Debug | Level::Warn)
2415 && metadata.target() == "nautilus_lighter::websocket::handler"
2416 }
2417
2418 fn log(&self, record: &Record<'_>) {
2419 if self.enabled(record.metadata()) {
2420 let message = record.args().to_string();
2421 if (record.level() == Level::Debug
2422 && message.starts_with("Sending Lighter unsubscribe"))
2423 || (record.level() == Level::Warn
2424 && message.starts_with("Lighter subscribe error"))
2425 {
2426 self.messages.lock().push(message);
2427 }
2428 }
2429 }
2430
2431 fn flush(&self) {}
2432 }
2433
2434 const WS_ACCOUNT_ORDERS_UPDATE: &str =
2435 include_str!("../../test_data/ws_account_orders_update.json");
2436 const WS_ACCOUNT_ALL_TRADES_UPDATE: &str =
2437 include_str!("../../test_data/ws_account_all_trades_update.json");
2438 const WS_ACCOUNT_ALL_POSITIONS_UPDATE: &str =
2439 include_str!("../../test_data/ws_account_all_positions_update.json");
2440 const WS_ACCOUNT_ALL_ASSETS_UPDATE: &str =
2441 include_str!("../../test_data/ws_account_all_assets_update.json");
2442 const WS_USER_STATS_UPDATE: &str = include_str!("../../test_data/ws_user_stats_update.json");
2443 const WS_ACCOUNT_ALL_ASSETS_WITH_POSITION: &str =
2444 include_str!("../../test_data/ws_account_all_assets_with_position.json");
2445 const WS_USER_STATS_WITH_POSITION: &str =
2446 include_str!("../../test_data/ws_user_stats_with_position.json");
2447 const WS_MARKET_STATS_UPDATE_SINGLE: &str =
2448 include_str!("../../test_data/ws_market_stats_update_single.json");
2449 const WS_MARKET_STATS_UPDATE_ALL: &str =
2450 include_str!("../../test_data/ws_market_stats_update_all.json");
2451 const WS_SPOT_MARKET_STATS_UPDATE_SINGLE: &str =
2452 include_str!("../../test_data/ws_spot_market_stats_update_single.json");
2453 const WS_SPOT_MARKET_STATS_UPDATE_ALL: &str =
2454 include_str!("../../test_data/ws_spot_market_stats_update_all.json");
2455 const WS_CANDLE_SUBSCRIBED: &str = include_str!("../../test_data/ws_candle_subscribed.json");
2456 const WS_SPOT_STATS_SUBSCRIBED_BAD_BODY: &str =
2457 include_str!("../../test_data/ws_spot_market_stats_subscribed_single_bad_body.json");
2458 const WS_BOOK_SUBSCRIBED_BAD_BODY: &str =
2459 include_str!("../../test_data/ws_order_book_subscribed_bad_body.json");
2460 const WS_SUBSCRIBE_FAILED: &str =
2461 include_str!("../../test_data/ws_order_book_subscribe_failed.json");
2462 const BOOK_SUBSCRIBE_REJECTION: &str =
2463 "client error: venue rejected the WebSocket subscribe with code 30012: order_book:0";
2464
2465 fn handle_control_text(
2466 handler: &mut FeedHandler,
2467 text: &str,
2468 ) -> (bool, Option<NautilusWsMessage>) {
2469 let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
2470 return (false, None);
2471 };
2472 handler.handle_control_value(&value)
2473 }
2474
2475 fn stub_eth_perp_instrument() -> InstrumentAny {
2476 let instrument_id = InstrumentId::new(Symbol::new("ETH-PERP"), Venue::new("LIGHTER"));
2477 InstrumentAny::CryptoPerpetual(
2478 CryptoPerpetual::builder()
2479 .instrument_id(instrument_id)
2480 .raw_symbol(Symbol::new("ETH-PERP"))
2481 .base_currency(Currency::from("ETH"))
2482 .quote_currency(Currency::from("USDC"))
2483 .settlement_currency(Currency::from("USDC"))
2484 .is_inverse(false)
2485 .price_precision(2)
2486 .size_precision(4)
2487 .price_increment(Price::from("0.01"))
2488 .size_increment(Quantity::from("0.0001"))
2489 .ts_event(UnixNanos::default())
2490 .ts_init(UnixNanos::default())
2491 .build()
2492 .unwrap(),
2493 )
2494 }
2495
2496 fn stub_eth_spot_instrument() -> InstrumentAny {
2497 let instrument_id = InstrumentId::new(Symbol::new("ETH-SPOT"), Venue::new("LIGHTER"));
2498 InstrumentAny::CurrencyPair(
2499 CurrencyPair::builder()
2500 .instrument_id(instrument_id)
2501 .raw_symbol(Symbol::new("ETH-SPOT"))
2502 .base_currency(Currency::from("ETH"))
2503 .quote_currency(Currency::from("USDC"))
2504 .price_precision(2)
2505 .size_precision(4)
2506 .price_increment(Price::from("0.01"))
2507 .size_increment(Quantity::from("0.0001"))
2508 .ts_event(UnixNanos::default())
2509 .ts_init(UnixNanos::default())
2510 .build()
2511 .unwrap(),
2512 )
2513 }
2514
2515 fn make_handler_with_account() -> FeedHandler {
2516 let signal = Arc::new(AtomicBool::new(false));
2517 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
2518 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
2519 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
2520 let mut handler =
2521 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2522 handler.instruments.insert(0, stub_eth_perp_instrument());
2523 handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
2524 handler
2525 }
2526
2527 #[rstest]
2529 fn book_late_snapshot_completes_running_recovery() {
2530 let mut handler = make_handler_with_account();
2531 handler.book.delta_subs.insert(0);
2532 let frame: LighterWsFrame = serde_json::from_str(include_str!(
2533 "../../test_data/ws_order_book_subscribed.json"
2534 ))
2535 .unwrap();
2536 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2537 handler.subscriptions.add_reference("order_book:0");
2538 assert!(recovery.begin_replacement());
2539
2540 let during_write = handler.handle_frame(frame.clone(), UnixNanos::from(1));
2541 recovery.gate.open();
2542 let after_write = handler.handle_frame(frame, UnixNanos::from(2));
2543
2544 assert!(during_write.is_empty());
2545 assert_eq!(after_write.len(), 1);
2546 assert!(recovery.is_accepted());
2547 assert!(handler.book.snapshots_seen.contains(&0));
2548 }
2549
2550 #[rstest]
2551 #[case::unconfirmed_permanent(false, false, false)]
2552 #[case::closed_gate_permanent(true, false, false)]
2553 #[case::confirmed_permanent(true, true, false)]
2554 #[case::unconfirmed_retry(false, false, true)]
2555 #[case::closed_gate_retry(true, false, true)]
2556 #[case::confirmed_retry(true, true, true)]
2557 fn book_venue_rejection_requires_confirmed_write(
2558 #[case] confirmed: bool,
2559 #[case] gate_open: bool,
2560 #[case] retry: bool,
2561 ) {
2562 let mut handler = make_handler_with_account();
2563 let (topic, generation) =
2564 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2565 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2566 assert!(recovery.begin_replacement());
2567
2568 if gate_open {
2569 recovery.gate.open();
2570 }
2571
2572 if !confirmed {
2573 handler.book.expected.remove(&0);
2574 }
2575
2576 if retry {
2577 handler.retry_inflight_subscriptions("venue rejection");
2578 } else {
2579 let value = serde_json::json!({"code": 30012, "channel": "order_book:0"});
2580 handler.handle_control_value(&value);
2581 }
2582
2583 let rejected = confirmed && gate_open;
2584 assert_eq!(
2585 handler.inflight_subs.get(&topic),
2586 (!rejected).then_some(&generation)
2587 );
2588 assert!(!recovery.cancellation.is_cancelled());
2589 assert!(recovery.gate.lock().is_closed());
2590
2591 match &*recovery.outcome.borrow() {
2592 BookRecoveryOutcome::Pending => assert!(!confirmed || !gate_open),
2593 BookRecoveryOutcome::Rejected(LighterWsError::Network(message)) => {
2594 assert!(confirmed && gate_open && retry);
2595 assert_eq!(message, "venue rejection");
2596 }
2597 BookRecoveryOutcome::Rejected(LighterWsError::Client(message)) => {
2598 assert!(confirmed && gate_open && !retry);
2599 assert_eq!(
2600 message,
2601 "venue rejected the WebSocket subscribe with code 30012"
2602 );
2603 }
2604 outcome => panic!("unexpected recovery outcome: {outcome:?}"),
2605 }
2606 }
2607
2608 #[rstest]
2611 #[case::waiting_subscriber(true)]
2612 #[case::replayed(false)]
2613 fn book_client_rejection_recovers_only_without_waiting_subscriber(#[case] waiting: bool) {
2614 let mut handler = make_handler_with_account();
2615 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2616 handler.cmd_tx = Some(cmd_tx);
2617 handler.book.delta_subs.insert(0);
2618 handler.subscriptions.add_reference("order_book:0");
2619 let (response, mut result) = tokio::sync::oneshot::channel();
2620 let (topic, generation) = mark_subscription_inflight(
2621 &mut handler,
2622 LighterWsChannel::OrderBook(0),
2623 waiting.then_some(response),
2624 );
2625 let initial = CancellationToken::new();
2626 handler.book.initial.insert(
2627 0,
2628 PendingSnapshot {
2629 deadline: None,
2630 cancel: initial.clone(),
2631 gate: SnapshotGate::default(),
2632 },
2633 );
2634
2635 handler.reject_book_subscription(
2636 0,
2637 generation,
2638 LighterWsError::Client("invalid market".into()),
2639 );
2640
2641 let recovering = handler
2642 .book
2643 .recovery
2644 .get(&0)
2645 .is_some_and(BookRecoveryState::is_running);
2646
2647 if waiting {
2648 assert!(
2649 matches!(result.try_recv(), Ok(Err(message)) if message.contains("invalid market"))
2650 );
2651 } else {
2652 assert_eq!(
2653 result.try_recv(),
2654 Err(tokio::sync::oneshot::error::TryRecvError::Closed)
2655 );
2656 }
2657
2658 assert_eq!(recovering, !waiting);
2659 assert_eq!(handler.book.work.len(), usize::from(!waiting));
2660 assert_eq!(handler.subscription_attempts.contains_key(&topic), !waiting);
2661 assert!(!handler.inflight_subs.contains_key(&topic));
2662 assert!(!handler.book.initial.contains_key(&0));
2663 assert!(initial.is_cancelled());
2664 }
2665
2666 #[rstest]
2669 #[case::response_txs(false)]
2670 #[case::pending_response_txs(true)]
2671 #[tokio::test]
2672 async fn book_client_rejection_during_recovery_answers_waiting_subscriber(
2673 #[case] pending: bool,
2674 ) {
2675 let (mut handler, _cmd_tx, mut result) =
2676 recovering_book_with_waiting_subscriber(false).await;
2677 let topic = Ustr::from("order_book:0");
2678
2679 if pending {
2680 let attempt = handler.subscription_attempts.get_mut(&topic).unwrap();
2682 attempt.pending_auth = Some(SecretString::from("token"));
2683 attempt.pending_response_txs = std::mem::take(&mut attempt.response_txs);
2684 }
2685
2686 let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2687 let gate_opened = !recovery.gate.lock().is_closed();
2688 let pending = result.try_recv();
2689
2690 handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2691
2692 assert!(gate_opened);
2693 assert_eq!(
2694 pending,
2695 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2696 );
2697 assert_eq!(
2698 result.try_recv(),
2699 Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string()))
2700 );
2701 assert!(!handler.subscription_attempts.contains_key(&topic));
2702 assert!(!handler.inflight_subs.contains_key(&topic));
2703 assert!(!handler.book.expected.contains_key(&0));
2704 assert!(recovery.is_running());
2705 assert!(recovery.gate.lock().is_closed());
2706 assert!(matches!(
2707 &*recovery.outcome.borrow(),
2708 BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2709 if message == "venue rejected the WebSocket subscribe with code 30012"
2710 ));
2711 }
2712
2713 #[rstest]
2717 #[case::deltas_last_reference(false, false)]
2718 #[case::deltas_other_consumer(false, true)]
2719 #[case::depth_last_reference(true, false)]
2720 #[case::depth_other_consumer(true, true)]
2721 #[tokio::test]
2722 async fn book_rejected_subscriber_release_cancels_orphaned_recovery(
2723 #[case] depth: bool,
2724 #[case] other_consumer: bool,
2725 ) {
2726 let (mut handler, cmd_tx, mut result) =
2727 recovering_book_with_waiting_subscriber(depth).await;
2728 let topic = Ustr::from("order_book:0");
2729
2730 if other_consumer {
2731 let other = if depth {
2732 &mut handler.book.delta_subs
2733 } else {
2734 &mut handler.book.depth_subs
2735 };
2736
2737 other.insert(0);
2738 assert!(!handler.subscriptions.add_reference("order_book:0"));
2739 }
2740
2741 let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2742 handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2743 let answer = result.try_recv();
2744
2745 let released = handler.subscriptions.remove_reference("order_book:0");
2747
2748 let release = if depth {
2749 HandlerCommand::SetDepthSub {
2750 market_index: 0,
2751 subscribed: false,
2752 }
2753 } else {
2754 HandlerCommand::SetBookDeltasSub {
2755 market_index: 0,
2756 subscribed: false,
2757 }
2758 };
2759
2760 cmd_tx.send(release).unwrap();
2761 cmd_tx.send(HandlerCommand::Disconnect).unwrap();
2762 assert!(handler.next().await.is_none());
2763
2764 let frame: LighterWsFrame = serde_json::from_str(include_str!(
2765 "../../test_data/ws_order_book_subscribed.json"
2766 ))
2767 .unwrap();
2768 let late = handler.handle_frame(frame, UnixNanos::from(1));
2769 let (completion, _completed) = tokio::sync::oneshot::channel();
2770 handler.queue_book_replacement(
2771 0,
2772 BookWrite {
2773 cancel: recovery.cancellation.child_token(),
2774 gate: recovery.gate.clone(),
2775 completion,
2776 },
2777 );
2778
2779 let replacement = handler.subscription_attempts.get(&topic).map(|attempt| {
2780 (
2781 attempt.response_txs.len(),
2782 handler.pending_subs.contains(&(topic, attempt.generation)),
2783 )
2784 });
2785
2786 assert_eq!(answer, Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string())));
2787 assert_eq!(released, !other_consumer);
2788 assert_eq!(recovery.is_running(), other_consumer);
2789 assert_eq!(handler.book.recovery.contains_key(&0), other_consumer);
2790 assert_eq!(handler.book.work.len(), usize::from(other_consumer));
2791 assert!(!recovery.is_accepted());
2792 assert!(recovery.gate.lock().is_closed());
2793 assert!(late.is_empty());
2794 assert!(!handler.book.snapshots_seen.contains(&0));
2795 assert_eq!(handler.book.writes.contains_key(&0), other_consumer);
2796 assert_eq!(replacement, other_consumer.then_some((0, true)));
2797 }
2798
2799 async fn recovering_book_with_waiting_subscriber(
2802 depth: bool,
2803 ) -> (
2804 FeedHandler,
2805 tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2806 tokio::sync::oneshot::Receiver<Result<(), String>>,
2807 ) {
2808 let signal = Arc::new(AtomicBool::new(false));
2809 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2810 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
2811 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
2812 let mut handler =
2813 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2814 handler.set_command_sender(cmd_tx.clone());
2815 handler.instruments.insert(0, stub_eth_perp_instrument());
2816
2817 if depth {
2818 handler.book.depth_subs.insert(0);
2819 } else {
2820 handler.book.delta_subs.insert(0);
2821 }
2822
2823 handler.subscriptions.add_reference("order_book:0");
2824 let (response, result) = tokio::sync::oneshot::channel();
2825 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(response));
2826 handle_control_text(
2827 &mut handler,
2828 r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
2829 );
2830
2831 assert!(handler.book.work.next().now_or_never().is_none());
2833
2834 let Ok(HandlerCommand::RecoverBook {
2835 market_index,
2836 cancel,
2837 gate,
2838 completion,
2839 }) = handler.cmd_rx.try_recv()
2840 else {
2841 panic!("expected book replacement request");
2842 };
2843
2844 handler.queue_book_replacement(
2845 market_index,
2846 BookWrite {
2847 cancel,
2848 gate,
2849 completion,
2850 },
2851 );
2852
2853 handler.pump_pending_subscribes().await;
2854
2855 let Some(BookWorkResult::Sent {
2857 market_index,
2858 generation,
2859 cancel,
2860 write,
2861 ..
2862 }) = handler.book.work.next().await
2863 else {
2864 panic!("expected replacement send");
2865 };
2866
2867 handler.complete_book_send(market_index, generation, cancel, write, Ok(0));
2868
2869 (handler, cmd_tx, result)
2870 }
2871
2872 #[rstest]
2874 #[case::rejected(true)]
2875 #[case::send_failed(false)]
2876 #[tokio::test]
2877 async fn book_failed_replacement_releases_subscription_slot(#[case] rejected: bool) {
2878 let mut handler = make_handler_with_account();
2879 handler.book.delta_subs.insert(0);
2880 handler.subscriptions.add_reference("order_book:0");
2881 let (topic, generation) =
2882 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2883
2884 for i in 0..SUBSCRIBE_INFLIGHT_MAX - 1 {
2885 handler
2886 .inflight_subs
2887 .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
2888 }
2889
2890 handler.queue_subscribe(LighterWsChannel::OrderBook(1), None, None);
2891 let other = Ustr::from("order_book:1");
2892 handler.pump_pending_subscribes().await;
2893 let blocked = !handler.inflight_subs.contains_key(&other);
2894 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2895 assert!(recovery.begin_replacement());
2896 recovery.gate.open();
2897 let (completion, completed) = tokio::sync::oneshot::channel();
2898
2899 if rejected {
2900 handler.reject_book_subscription(
2901 0,
2902 generation,
2903 LighterWsError::Client("venue rejection".into()),
2904 );
2905 } else {
2906 handler.complete_book_send(
2907 0,
2908 generation,
2909 recovery.cancellation.child_token(),
2910 Some(BookWrite {
2911 cancel: recovery.cancellation.child_token(),
2912 gate: recovery.gate.clone(),
2913 completion,
2914 }),
2915 Err(LighterWsError::Network("send failed".into())),
2916 );
2917 }
2918
2919 handler.pump_pending_subscribes().await;
2920
2921 assert!(blocked);
2922 assert!(!handler.inflight_subs.contains_key(&topic));
2923 assert!(handler.inflight_subs.contains_key(&other));
2924 assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
2925 assert!(recovery.is_running());
2926
2927 if rejected {
2928 assert!(matches!(
2929 &*recovery.outcome.borrow(),
2930 BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2931 if message == "venue rejection"
2932 ));
2933 } else {
2934 assert!(
2935 matches!(completed.await, Ok(Err(LighterWsError::Network(message))) if message == "send failed")
2936 );
2937 }
2938 }
2939
2940 #[tokio::test]
2941 async fn book_queued_write_skips_after_last_reference_disappears() {
2942 let mut handler = make_handler_with_account();
2943 handler.book.delta_subs.insert(0);
2944 let channel = LighterWsChannel::OrderBook(0);
2945 handler.subscriptions.add_reference(&channel.topic_key());
2946 let (_, generation) = mark_subscription_inflight(&mut handler, channel.clone(), None);
2947 handler.send_book_subscribe(0, generation);
2948 assert!(handler.subscriptions.remove_reference(&channel.topic_key()));
2949
2950 let BookWorkResult::Sent { result, .. } = handler.book.work.next().await.unwrap() else {
2951 panic!("expected send completion");
2952 };
2953
2954 assert_eq!(
2955 result.unwrap_err().to_string(),
2956 "client error: book subscription cancelled"
2957 );
2958 assert!(handler.inner.is_none());
2959 }
2960
2961 #[rstest]
2962 fn book_wrong_epoch_snapshot_preserves_current_completion() {
2963 let mut handler = make_handler_with_account();
2964 let (tx, mut rx) = tokio::sync::oneshot::channel();
2965 let (topic, generation) =
2966 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2967 handler.book.expected.insert(0, (generation, 12));
2968
2969 assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
2970 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
2971 assert_eq!(
2972 rx.try_recv(),
2973 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2974 );
2975 assert!(handler.complete_typed_subscription(topic.as_str(), 12));
2976 assert_eq!(rx.try_recv(), Ok(Ok(())));
2977 assert!(!handler.inflight_subs.contains_key(&topic));
2978 }
2979
2980 #[rstest]
2981 fn book_control_ack_preserves_snapshot_but_not_successor_completion() {
2982 let mut handler = make_handler_with_account();
2983 let (topic, old_generation) =
2984 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2985 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2986 assert!(handler.complete_typed_subscription(topic.as_str(), 0));
2987 assert_eq!(handler.book.expected[&0], (old_generation, 0));
2988
2989 handler.subscriptions.mark_failure(topic.as_str());
2990 let (_, generation) =
2991 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2992 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2993 handler.subscriptions.mark_failure(topic.as_str());
2994 let (tx, mut rx) = tokio::sync::oneshot::channel();
2995 let (_, successor) =
2996 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2997
2998 assert_ne!(generation, successor);
2999 assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3000 assert_eq!(handler.inflight_subs.get(&topic), Some(&successor));
3001 assert_eq!(
3002 rx.try_recv(),
3003 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3004 );
3005 assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3006 assert_eq!(rx.try_recv(), Ok(Ok(())));
3007 }
3008
3009 #[rstest]
3010 fn book_old_epoch_does_not_consume_current_trailing_snapshot() {
3011 let mut handler = make_handler_with_account();
3012 let (topic, generation) =
3013 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3014 handler.book.expected.insert(0, (generation, 12));
3015 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3016
3017 assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
3018 assert_eq!(handler.book.trailing.get(&0), Some(&(generation, 12)));
3019 assert_eq!(
3020 handler.ignored_completions.get(&topic),
3021 Some(&CompletionKind::Typed)
3022 );
3023 assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3024 assert!(!handler.book.trailing.contains_key(&0));
3025 assert!(!handler.ignored_completions.contains_key(&topic));
3026 }
3027
3028 #[rstest]
3029 fn book_ack_before_write_cannot_complete_subscription() {
3030 let mut handler = make_handler_with_account();
3031 let (topic, generation) =
3032 mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3033 handler.book.expected.clear();
3034
3035 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3036 assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3037 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3038 }
3039
3040 #[rstest]
3041 fn book_empty_snapshot_replaces_cached_levels_for_both_consumers() {
3042 let mut handler = make_handler_with_account();
3043 handler.book.delta_subs.insert(0);
3044 handler.book.depth_subs.insert(0);
3045 let frame: LighterWsFrame = serde_json::from_str(include_str!(
3046 "../../test_data/ws_order_book_subscribed.json"
3047 ))
3048 .unwrap();
3049 assert_eq!(
3050 handler
3051 .handle_frame(frame.clone(), UnixNanos::from(1))
3052 .len(),
3053 2
3054 );
3055
3056 let LighterWsFrame::OrderBookSnapshot { order_book, .. } = &frame else {
3057 panic!("expected book snapshot");
3058 };
3059
3060 let mut empty = order_book.clone();
3061 empty.bids.clear();
3062 empty.asks.clear();
3063 empty.nonce += 1;
3064 let messages = handler.handle_order_book(
3065 Ustr::from("order_book:0"),
3066 &empty,
3067 123,
3068 true,
3069 UnixNanos::from(2),
3070 );
3071
3072 assert_eq!(messages.len(), 2);
3073
3074 let NautilusWsMessage::Deltas(deltas) = &messages[0] else {
3075 panic!("expected deltas");
3076 };
3077
3078 assert_eq!(deltas.deltas.len(), 1);
3079 assert_eq!(deltas.deltas[0].action, BookAction::Clear);
3080 assert_eq!(deltas.deltas[0].sequence, empty.nonce as u64);
3081 assert_eq!(
3082 deltas.deltas[0].flags,
3083 RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8
3084 );
3085
3086 let NautilusWsMessage::Depth(depth) = &messages[1] else {
3087 panic!("expected depth");
3088 };
3089
3090 assert!(depth.bids.is_empty());
3091 assert!(depth.asks.is_empty());
3092 assert!(depth.bid_counts.is_empty());
3093 assert!(depth.ask_counts.is_empty());
3094 assert!(handler.book.states[&0].book.bids.is_empty());
3095 assert!(handler.book.states[&0].book.asks.is_empty());
3096 }
3097
3098 #[tokio::test]
3099 async fn book_initial_deadline_is_cancelled_by_replacement_or_unsubscribe() {
3100 let mut handler = make_handler_with_account();
3101 handler.book.delta_subs.insert(0);
3102 let channel = LighterWsChannel::OrderBook(0);
3103 handler.subscriptions.add_reference(&channel.topic_key());
3104 let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3105 let cancel = CancellationToken::new();
3106 handler.book.initial.insert(
3107 0,
3108 PendingSnapshot {
3109 deadline: None,
3110 cancel: cancel.clone(),
3111 gate: SnapshotGate::default(),
3112 },
3113 );
3114
3115 handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3116 handler.book.cancel(0);
3117
3118 let BookWorkResult::Initial {
3119 cancel: completed,
3120 market_index,
3121 } = handler.book.work.next().await.unwrap()
3122 else {
3123 panic!("expected deadline completion");
3124 };
3125
3126 assert_eq!(market_index, 0);
3127 assert!(completed.is_cancelled());
3128 assert!(cancel.is_cancelled());
3129 assert!(handler.book.recovery.is_empty());
3130 assert!(handler.book.expected.is_empty());
3131 }
3132
3133 #[tokio::test]
3134 async fn book_initial_deadline_expires_without_cancellation() {
3135 let mut handler = make_handler_with_account();
3136 handler.book_snapshot_timeout = Duration::from_millis(50);
3137 handler.book.delta_subs.insert(0);
3138 let channel = LighterWsChannel::OrderBook(0);
3139 handler.subscriptions.add_reference(&channel.topic_key());
3140 let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3141 let cancel = CancellationToken::new();
3142 handler.book.initial.insert(
3143 0,
3144 PendingSnapshot {
3145 deadline: None,
3146 cancel: cancel.clone(),
3147 gate: SnapshotGate::default(),
3148 },
3149 );
3150
3151 handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3152
3153 let BookWorkResult::Initial {
3154 market_index,
3155 cancel: completed,
3156 } = handler.book.work.next().await.unwrap()
3157 else {
3158 panic!("expected deadline completion");
3159 };
3160
3161 assert_eq!(market_index, 0);
3162 assert!(!completed.is_cancelled());
3163 assert!(!cancel.is_cancelled());
3164 }
3165
3166 #[tokio::test]
3167 async fn book_reconnect_retains_recovery_owner_and_retires_old_writes() {
3168 let signal = Arc::new(AtomicBool::new(false));
3169 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3170 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
3171 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
3172 let mut handler =
3173 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3174 let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
3175 handler.book.expected.insert(0, (3, 9));
3176 raw_tx
3177 .send((10, Message::Text(RECONNECTED.into())))
3178 .unwrap();
3179 assert!(matches!(
3180 handler.next().await,
3181 Some(NautilusWsMessage::Reconnected {
3182 connection_epoch: 10
3183 })
3184 ));
3185
3186 assert!(Arc::ptr_eq(
3187 handler.book.recovery[&0].current().unwrap(),
3188 &recovery
3189 ));
3190 assert!(!recovery.cancellation.is_cancelled());
3191 assert!(recovery.gate.lock().is_closed());
3192 assert!(matches!(
3193 *recovery.outcome.borrow(),
3194 BookRecoveryOutcome::Rejected(LighterWsError::Network(_))
3195 ));
3196 assert!(handler.book.expected.is_empty());
3197 assert!(handler.book.writes.is_empty());
3198 }
3199
3200 #[rstest]
3201 fn book_recovery_is_scoped_to_handler_and_market() {
3202 let mut first = make_handler_with_account();
3203 let mut second = make_handler_with_account();
3204 let recovery = first.book.recovery.entry(0).or_default().claim().unwrap();
3205 let other_market = first.book.recovery.entry(1).or_default().claim().unwrap();
3206 let other_socket = second.book.recovery.entry(0).or_default().claim().unwrap();
3207 first.book.cancel(0);
3208
3209 assert!(recovery.cancellation.is_cancelled());
3210 assert!(!other_market.cancellation.is_cancelled());
3211 assert!(!other_socket.cancellation.is_cancelled());
3212 }
3213
3214 #[rstest]
3215 #[tokio::test]
3216 async fn test_subscription_logs_omit_payload_body() {
3217 log::set_logger(&OUTBOUND_LOG_CAPTURE).expect("test logger already installed");
3218 log::set_max_level(LevelFilter::Debug);
3219
3220 let mut handler = make_handler_with_account();
3221 let account_index = SECRET_MARKER.parse::<i64>().unwrap();
3222 let channel = LighterWsChannel::AccountAll(account_index);
3223 let payload_len = serde_json::to_string(&LighterWsRequest::unsubscribe(
3224 channel.subscription_channel(),
3225 ))
3226 .unwrap()
3227 .len();
3228 OUTBOUND_LOG_CAPTURE.clear();
3229
3230 handler.dispatch_unsubscribe(channel).await;
3231 handler.handle_control_value(&json!({"code": 30012, "auth": SECRET_MARKER}));
3232 handler.handle_control_value(&json!({
3233 "code": 30012, "channel": "trade/999", "auth": SECRET_MARKER,
3234 }));
3235
3236 let messages = OUTBOUND_LOG_CAPTURE.messages();
3237
3238 assert!(
3239 messages
3240 .iter()
3241 .all(|message| !message.contains(SECRET_MARKER)),
3242 "outbound logs exposed the secret marker: {messages:?}"
3243 );
3244 assert!(
3245 messages.iter().any(|message| {
3246 message == &format!("Sending Lighter unsubscribe ({payload_len} bytes)")
3247 }),
3248 "unsubscribe metadata missing or inaccurate: {messages:?}"
3249 );
3250 assert!(messages.iter().any(|message| {
3251 message == "Lighter subscribe error code=30012 channel=none: retrying inflight subscriptions"
3252 }));
3253 assert!(messages.iter().any(|message| {
3254 message == "Lighter subscribe error code=30012 channel=trade/999: no matching inflight subscription"
3255 }));
3256 }
3257
3258 fn mark_subscription_inflight(
3259 handler: &mut FeedHandler,
3260 channel: LighterWsChannel,
3261 response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3262 ) -> (Ustr, u64) {
3263 handler.queue_subscribe(channel, None, response_tx);
3264 let (topic, generation) = handler
3265 .pending_subs
3266 .pop_front()
3267 .expect("subscription was not queued");
3268 handler.inflight_subs.insert(topic, generation);
3269 if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
3270 handler.book.expected.insert(market_index, (generation, 0));
3271 }
3272
3273 (topic, generation)
3274 }
3275
3276 fn saturate_subscription_gate(handler: &mut FeedHandler) {
3277 for i in 0..SUBSCRIBE_INFLIGHT_MAX {
3278 handler
3279 .inflight_subs
3280 .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
3281 }
3282 }
3283
3284 fn strip_account_marker(mut msgs: Vec<NautilusWsMessage>) -> Vec<NautilusWsMessage> {
3289 if matches!(
3290 msgs.last(),
3291 Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3292 ) {
3293 msgs.pop();
3294 }
3295 msgs
3296 }
3297
3298 #[rstest]
3299 fn handle_frame_routes_account_orders_to_execution_reports() {
3300 let mut handler = make_handler_with_account();
3301 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3302
3303 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3304
3305 assert_eq!(messages.len(), 1);
3306 match &messages[0] {
3307 NautilusWsMessage::ExecutionReports(reports) => {
3308 assert_eq!(reports.len(), 1);
3309 match &reports[0] {
3310 super::ExecutionReport::Order(order) => {
3311 assert_eq!(order.order_id, "281476929510110");
3312 assert_eq!(order.client_order_id, "42");
3313 }
3314 other => panic!("expected order report, was {other:?}"),
3315 }
3316 }
3317 other => panic!("expected execution reports, was {other:?}"),
3318 }
3319 }
3320
3321 #[rstest]
3322 fn handle_frame_routes_account_trades_to_execution_reports() {
3323 let mut handler = make_handler_with_account();
3324 let frame: super::LighterWsFrame =
3325 serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3326
3327 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3328
3329 assert_eq!(messages.len(), 1);
3330 match &messages[0] {
3331 NautilusWsMessage::ExecutionReports(reports) => {
3332 assert_eq!(reports.len(), 1);
3333 match &reports[0] {
3334 super::ExecutionReport::Fill(fill) => {
3335 assert_eq!(fill.bid_id_str.as_deref(), Some("562947905631053"),);
3341 }
3342 other => panic!("expected fill report, was {other:?}"),
3343 }
3344 }
3345 other => panic!("expected execution reports, was {other:?}"),
3346 }
3347 }
3348
3349 #[rstest]
3350 fn handle_frame_routes_account_positions_to_update_without_readiness_marker() {
3351 let mut handler = make_handler_with_account();
3352 let frame: super::LighterWsFrame =
3353 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3354
3355 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3356
3357 assert_eq!(messages.len(), 1);
3358 match &messages[0] {
3359 NautilusWsMessage::PositionUpdate {
3360 reports,
3361 closed_market_ids,
3362 skipped_market_ids,
3363 } => {
3364 assert!(closed_market_ids.is_empty());
3365 assert!(skipped_market_ids.is_empty());
3366 assert_eq!(reports.len(), 1);
3367 assert_eq!(reports[0].quantity, Quantity::from("1.5000"));
3368 }
3369 other => panic!("expected position update, was {other:?}"),
3370 }
3371 }
3372
3373 #[rstest]
3374 fn handle_frame_tolerates_unknown_position_margin_mode() {
3375 let mut handler = make_handler_with_account();
3376 let mut frame_json: serde_json::Value =
3377 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3378 frame_json["positions"]["0"]["margin_mode"] = json!(99);
3379 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3380
3381 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3382
3383 assert_eq!(messages.len(), 1);
3384 match &messages[0] {
3385 NautilusWsMessage::PositionUpdate {
3386 reports,
3387 closed_market_ids,
3388 skipped_market_ids,
3389 } => {
3390 assert!(closed_market_ids.is_empty());
3391 assert!(skipped_market_ids.is_empty());
3392 assert_eq!(reports.len(), 1);
3393 }
3394 other => panic!("expected position update, was {other:?}"),
3395 }
3396 }
3397
3398 #[rstest]
3399 fn handle_frame_routes_empty_account_positions_to_empty_update() {
3400 let mut handler = make_handler_with_account();
3401 let frame_json = serde_json::json!({
3402 "type": "update/account_all_positions",
3403 "channel": "account_all_positions:1234",
3404 "positions": {},
3405 "shares": [],
3406 "last_funding_round": null,
3407 "last_funding_discount": null,
3408 });
3409 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3410
3411 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3412
3413 assert_eq!(messages.len(), 1);
3414 match &messages[0] {
3415 NautilusWsMessage::PositionUpdate {
3416 reports,
3417 closed_market_ids,
3418 skipped_market_ids,
3419 } => {
3420 assert!(closed_market_ids.is_empty());
3421 assert!(skipped_market_ids.is_empty());
3422 assert!(reports.is_empty());
3423 }
3424 other => panic!("expected empty position update, was {other:?}"),
3425 }
3426 }
3427
3428 #[rstest]
3429 fn handle_frame_routes_zero_account_position_to_closed_update() {
3430 let mut handler = make_handler_with_account();
3431 let mut frame_json: serde_json::Value =
3432 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3433 frame_json["positions"]["0"]["position"] = json!("0.0000");
3434 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3435
3436 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3437
3438 assert_eq!(messages.len(), 1);
3439 match &messages[0] {
3440 NautilusWsMessage::PositionUpdate {
3441 reports,
3442 closed_market_ids,
3443 skipped_market_ids,
3444 } => {
3445 assert!(reports.is_empty());
3446 assert!(skipped_market_ids.is_empty());
3447 assert_eq!(closed_market_ids, &[0]);
3448 }
3449 other => panic!("expected closed position update, was {other:?}"),
3450 }
3451 }
3452
3453 #[rstest]
3454 fn handle_frame_marks_account_positions_incomplete_when_position_instrument_uncached() {
3455 let mut handler = make_handler_with_account();
3456 let mut frame_json: serde_json::Value =
3457 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3458 frame_json["type"] = json!("subscribed/account_all_positions");
3459 frame_json["positions"]["0"]["market_id"] = json!(999);
3460 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3461
3462 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3463
3464 assert_eq!(messages.len(), 1);
3465 match &messages[0] {
3466 NautilusWsMessage::PositionSnapshot {
3467 reports,
3468 skipped_market_ids,
3469 } => {
3470 assert_eq!(skipped_market_ids, &[999]);
3471 assert!(reports.is_empty());
3472 }
3473 other => panic!("expected incomplete position snapshot, was {other:?}"),
3474 }
3475 }
3476
3477 #[rstest]
3478 fn handle_frame_marks_account_positions_incomplete_when_position_parse_fails() {
3479 let mut handler = make_handler_with_account();
3480 let mut frame_json: serde_json::Value =
3481 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3482 frame_json["type"] = json!("subscribed/account_all_positions");
3483 frame_json["positions"]["0"]["position"] = json!("-1.5000");
3484 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3485
3486 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3487
3488 assert_eq!(messages.len(), 1);
3489 match &messages[0] {
3490 NautilusWsMessage::PositionSnapshot {
3491 reports,
3492 skipped_market_ids,
3493 } => {
3494 assert_eq!(skipped_market_ids, &[0]);
3495 assert!(reports.is_empty());
3496 }
3497 other => panic!("expected incomplete position snapshot, was {other:?}"),
3498 }
3499 }
3500
3501 #[rstest]
3502 fn handle_frame_marks_position_update_incomplete_when_position_parse_fails() {
3503 let mut handler = make_handler_with_account();
3504 let mut frame_json: serde_json::Value =
3505 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3506 frame_json["positions"]["0"]["position"] = json!("-1.5000");
3507 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3508
3509 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3510
3511 assert_eq!(messages.len(), 1);
3512 match &messages[0] {
3513 NautilusWsMessage::PositionUpdate {
3514 reports,
3515 closed_market_ids,
3516 skipped_market_ids,
3517 } => {
3518 assert!(reports.is_empty());
3519 assert!(closed_market_ids.is_empty());
3520 assert_eq!(skipped_market_ids, &[0]);
3521 }
3522 other => panic!("expected incomplete position update, was {other:?}"),
3523 }
3524 }
3525
3526 #[rstest]
3527 fn handle_frame_routes_subscribed_account_all_positions_snapshot() {
3528 let mut handler = make_handler_with_account();
3529 let frame_json = serde_json::json!({
3530 "type": "subscribed/account_all_positions",
3531 "channel": "account_all_positions:1234",
3532 "positions": {
3533 "0": {
3534 "allocated_margin": "0.000000",
3535 "avg_entry_price": "0.111230",
3536 "initial_margin_fraction": "10.00",
3537 "liquidation_price": "0.100598",
3538 "margin_mode": 0,
3539 "market_id": 0,
3540 "open_order_count": 0,
3541 "pending_order_count": 0,
3542 "position": "100",
3543 "position_tied_order_count": 0,
3544 "position_value": "11.123000",
3545 "realized_pnl": "0.000000",
3546 "sign": 1,
3547 "symbol": "ETH",
3548 "total_discount": "0.000000",
3549 "total_funding_paid_out": "0.000000",
3550 "unrealized_pnl": "0.000000"
3551 }
3552 },
3553 });
3554 let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3555
3556 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3557
3558 assert_eq!(messages.len(), 1);
3559 match &messages[0] {
3560 NautilusWsMessage::PositionSnapshot {
3561 reports,
3562 skipped_market_ids,
3563 } => {
3564 assert!(skipped_market_ids.is_empty());
3565 assert_eq!(reports.len(), 1);
3566 assert_eq!(reports[0].quantity, Quantity::from("100"));
3567 }
3568 other => panic!("expected position snapshot, was {other:?}"),
3569 }
3570 }
3571
3572 #[rstest]
3573 #[case::connected(serde_json::json!({"type": "connected", "session_id": "x"}), true, false)]
3574 #[case::ping(serde_json::json!({"type": "ping"}), true, false)]
3575 #[case::pong(serde_json::json!({"type": "pong"}), true, false)]
3576 #[case::send_tx_ack(
3577 serde_json::json!({"type": "jsonapi/sendtx", "code": 200, "tx_hash": "abc"}),
3578 true,
3579 true,
3580 )]
3581 #[case::send_tx_without_code(
3582 serde_json::json!({"type": "jsonapi/sendtx", "tx_hash": "abc"}),
3583 true,
3584 false,
3585 )]
3586 #[case::send_tx_with_nonnumeric_code(
3587 serde_json::json!({"type": "jsonapi/sendtx", "code": "200", "tx_hash": "abc"}),
3588 true,
3589 false,
3590 )]
3591 #[case::error_frame(
3592 serde_json::json!({"type": "error", "code": 21727, "message": "invalid client order index"}),
3593 true,
3594 true,
3595 )]
3596 #[case::error_frame_integrator_not_approved(
3597 serde_json::json!({"type": "error", "code": 21149, "message": "integrator is not approved"}),
3598 true,
3599 true,
3600 )]
3601 #[case::send_tx_ack_integrator_not_approved(
3602 serde_json::json!({"type": "jsonapi/sendtx", "code": 21149, "message": "integrator is not approved"}),
3603 true,
3604 true,
3605 )]
3606 #[case::wrapped_error_integrator_not_approved(
3607 serde_json::json!({"error": {"code": 21149, "message": "integrator is not approved"}}),
3608 true,
3609 true,
3610 )]
3611 #[case::subscription_error_frame(
3612 serde_json::json!({"type": "error", "code": 30003, "message": "Already Subscribed to : ticker:3"}),
3613 true,
3614 false,
3615 )]
3616 #[case::wrapped_subscription_error(
3617 serde_json::json!({"error": {"code": 30003, "message": "Already Subscribed to : ticker:3"}}),
3618 true,
3619 false,
3620 )]
3621 #[case::codeless_error_frame(
3622 serde_json::json!({"type": "error", "message": "unclassifiable"}),
3623 true,
3624 false,
3625 )]
3626 #[case::unknown_type(
3627 serde_json::json!({"type": "something_unexpected", "payload": "x"}),
3628 false,
3629 false,
3630 )]
3631 #[case::no_type_field(
3632 serde_json::json!({"error": {"code": 21702, "message": "invalid price"}}),
3633 true,
3634 true,
3635 )]
3636 fn handle_control_text_tri_state(
3637 #[case] payload: serde_json::Value,
3638 #[case] expected_matched: bool,
3639 #[case] expected_has_msg: bool,
3640 ) {
3641 let mut handler = make_handler_with_account();
3648 let text = payload.to_string();
3649 let (matched, msg) = handle_control_text(&mut handler, &text);
3650 assert_eq!(matched, expected_matched, "matched flag");
3651 assert_eq!(msg.is_some(), expected_has_msg, "msg presence");
3652 }
3653
3654 #[rstest]
3655 #[case::ack(serde_json::json!({"type":"jsonapi/sendtxbatch","id":"tx-batch:abc","code":200,"tx_hash":["abc","def"]}), 200, "batch rejected", vec!["abc", "def"])]
3656 #[case::error(serde_json::json!({"id":"tx-batch:abc","error":{"code":21104,"message":"invalid nonce"}}), 21104, "invalid nonce", vec![])]
3657 fn batch_response_preserves_correlation_and_epoch(
3658 #[case] value: serde_json::Value,
3659 #[case] expected_code: i64,
3660 #[case] expected_message: &str,
3661 #[case] expected_hashes: Vec<&str>,
3662 ) {
3663 let mut handler = make_handler_with_account();
3664 let (matched, message) = handler.handle_control_value(&value);
3665 assert!(matched);
3666
3667 match message.unwrap().with_connection_epoch(7) {
3668 NautilusWsMessage::SendTxBatchResult {
3669 connection_epoch,
3670 id,
3671 code,
3672 message,
3673 tx_hashes,
3674 } => {
3675 assert_eq!(connection_epoch, 7);
3676 assert_eq!(id, "tx-batch:abc");
3677 assert_eq!(code, expected_code);
3678 assert_eq!(message, expected_message);
3679 assert_eq!(tx_hashes, expected_hashes);
3680 }
3681 message => panic!("expected batch response, was {message:?}"),
3682 }
3683 }
3684
3685 #[rstest]
3686 fn handle_control_text_sendtx_success_emits_typed_ack() {
3687 let mut handler = make_handler_with_account();
3688 let payload = serde_json::json!({
3689 "type": "jsonapi/sendtx",
3690 "code": 200,
3691 "tx_hash": "0000abcd",
3692 })
3693 .to_string();
3694
3695 let (_, msg) = handle_control_text(&mut handler, &payload);
3696
3697 match msg.expect("SendTxAck emitted") {
3698 NautilusWsMessage::SendTxAck { tx_hash, code, .. } => {
3699 assert_eq!(code, 200);
3700 assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3701 }
3702 other => panic!("expected SendTxAck, was {other:?}"),
3703 }
3704 }
3705
3706 #[rstest]
3707 fn handle_control_text_sendtx_failure_emits_ack_sourced_rejection() {
3708 let mut handler = make_handler_with_account();
3709 let payload = serde_json::json!({
3710 "type": "jsonapi/sendtx",
3711 "code": 21727,
3712 "message": "invalid client order index",
3713 })
3714 .to_string();
3715
3716 let (_, msg) = handle_control_text(&mut handler, &payload);
3717
3718 match msg.expect("SendTxRejected emitted") {
3719 NautilusWsMessage::SendTxRejected {
3720 source,
3721 code,
3722 message,
3723 tx_hash,
3724 ..
3725 } => {
3726 assert_eq!(source, SendTxRejectionSource::Ack);
3727 assert_eq!(code, Some(21727));
3728 assert_eq!(message, "invalid client order index");
3729 assert_eq!(tx_hash, None);
3730 }
3731 other => panic!("expected SendTxRejected, was {other:?}"),
3732 }
3733 }
3734
3735 #[rstest]
3736 fn handle_control_text_sendtx_failure_carries_echoed_tx_hash() {
3737 let mut handler = make_handler_with_account();
3738 let payload = serde_json::json!({
3739 "type": "jsonapi/sendtx",
3740 "code": 21727,
3741 "message": "invalid client order index",
3742 "tx_hash": "0000abcd",
3743 })
3744 .to_string();
3745
3746 let (_, msg) = handle_control_text(&mut handler, &payload);
3747
3748 match msg.expect("SendTxRejected emitted") {
3749 NautilusWsMessage::SendTxRejected { tx_hash, .. } => {
3750 assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3751 }
3752 other => panic!("expected SendTxRejected, was {other:?}"),
3753 }
3754 }
3755
3756 #[rstest]
3757 fn handle_control_text_bare_error_frame_emits_bare_error_rejection() {
3758 let mut handler = make_handler_with_account();
3759 let payload = serde_json::json!({
3760 "type": "error",
3761 "code": 21702,
3762 "message": "invalid price",
3763 })
3764 .to_string();
3765
3766 let (_, msg) = handle_control_text(&mut handler, &payload);
3767
3768 match msg.expect("SendTxRejected emitted") {
3769 NautilusWsMessage::SendTxRejected {
3770 source,
3771 code,
3772 message,
3773 tx_hash,
3774 ..
3775 } => {
3776 assert_eq!(source, SendTxRejectionSource::BareError);
3777 assert_eq!(code, Some(21702));
3778 assert_eq!(message, "invalid price");
3779 assert_eq!(tx_hash, None);
3780 }
3781 other => panic!("expected SendTxRejected, was {other:?}"),
3782 }
3783 }
3784
3785 #[rstest]
3786 fn handle_control_text_wrapped_error_emits_bare_error_rejection() {
3787 let mut handler = make_handler_with_account();
3789 let payload = serde_json::json!({
3790 "error": {"code": 21149, "message": "integrator is not approved"},
3791 })
3792 .to_string();
3793
3794 let (_, msg) = handle_control_text(&mut handler, &payload);
3795
3796 match msg.expect("SendTxRejected emitted") {
3797 NautilusWsMessage::SendTxRejected {
3798 source,
3799 code,
3800 message,
3801 tx_hash,
3802 ..
3803 } => {
3804 assert_eq!(source, SendTxRejectionSource::BareError);
3805 assert_eq!(code, Some(21149));
3806 assert_eq!(message, "integrator is not approved");
3807 assert_eq!(tx_hash, None);
3808 }
3809 other => panic!("expected SendTxRejected, was {other:?}"),
3810 }
3811 }
3812
3813 #[rstest]
3814 fn handle_frame_emits_no_account_state_until_both_streams_seen() {
3815 let mut handler = make_handler_with_account();
3819 let assets_only: super::LighterWsFrame =
3820 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3821
3822 let messages = strip_account_marker(handler.handle_frame(assets_only, UnixNanos::from(11)));
3823
3824 assert!(
3825 messages.is_empty(),
3826 "expected no AccountState before user_stats arrives, received {messages:?}"
3827 );
3828 }
3829
3830 #[rstest]
3831 fn handle_frame_routes_account_assets_and_user_stats_to_unified_state() {
3832 let mut handler = make_handler_with_account();
3839 let assets_frame: super::LighterWsFrame =
3840 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3841 let user_stats_frame: super::LighterWsFrame =
3842 serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3843
3844 let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3845 let messages =
3846 strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3847
3848 assert_eq!(messages.len(), 1);
3849 match &messages[0] {
3850 NautilusWsMessage::AccountState(state) => {
3851 let usdc = Currency::get_or_create_crypto("USDC");
3852 assert_eq!(state.account_type, AccountType::Margin);
3853 assert_eq!(state.base_currency, None);
3854 assert_eq!(state.balances.len(), 1);
3855 assert_eq!(state.balances[0].currency, usdc);
3856 assert_eq!(state.balances[0].total, Money::from("50.000000 USDC"));
3859 assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3860 assert_eq!(state.balances[0].free, Money::from("50.000000 USDC"));
3861 assert_eq!(state.margins.len(), 1);
3862 assert_eq!(state.margins[0].currency, usdc);
3863 assert_eq!(state.margins[0].initial, Money::from("0 USDC"));
3864 assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3865 assert!(state.margins[0].instrument_id.is_none());
3866 assert!(state.is_reported);
3867 }
3868 other => panic!("expected account state, was {other:?}"),
3869 }
3870 }
3871
3872 #[rstest]
3873 fn handle_frame_unified_state_reflects_open_position() {
3874 let mut handler = make_handler_with_account();
3885 let assets_frame: super::LighterWsFrame =
3886 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_WITH_POSITION).unwrap();
3887 let user_stats_frame: super::LighterWsFrame =
3888 serde_json::from_str(WS_USER_STATS_WITH_POSITION).unwrap();
3889
3890 let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3891 let messages =
3892 strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3893
3894 assert_eq!(messages.len(), 1);
3895 match &messages[0] {
3896 NautilusWsMessage::AccountState(state) => {
3897 assert_eq!(state.account_type, AccountType::Margin);
3898 assert_eq!(state.base_currency, None);
3899 assert_eq!(state.balances.len(), 1);
3900 assert_eq!(state.balances[0].total, Money::from("49.99536956 USDC"));
3904 assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3905 assert_eq!(state.balances[0].free, Money::from("49.99536956 USDC"));
3906 assert_eq!(state.margins.len(), 1);
3907 assert_eq!(state.margins[0].initial, Money::from("0.82705500 USDC"));
3909 assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3910 assert!(state.margins[0].instrument_id.is_none());
3911 }
3912 other => panic!("expected account state, was {other:?}"),
3913 }
3914 }
3915
3916 #[rstest]
3917 fn handle_frame_emits_account_stream_first_frame_marker_per_variant() {
3918 let mut handler = make_handler_with_account();
3924 let orders_frame: super::LighterWsFrame =
3925 serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3926 let trades_frame: super::LighterWsFrame =
3927 serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3928 let positions_frame: super::LighterWsFrame = serde_json::from_value({
3929 let mut value: serde_json::Value =
3930 serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3931 value["type"] = json!("subscribed/account_all_positions");
3932 value
3933 })
3934 .unwrap();
3935 let assets_frame: super::LighterWsFrame =
3936 serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3937 let user_stats_frame: super::LighterWsFrame =
3938 serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3939
3940 let cases = [
3941 (orders_frame, AccountStream::Orders),
3942 (trades_frame, AccountStream::Trades),
3943 (positions_frame, AccountStream::Positions),
3944 (assets_frame, AccountStream::Assets),
3945 (user_stats_frame, AccountStream::UserStats),
3946 ];
3947
3948 for (frame, expected) in cases {
3949 let msgs = handler.handle_frame(frame, UnixNanos::from(11));
3950 let marker = msgs
3951 .iter()
3952 .find(|m| matches!(m, NautilusWsMessage::AccountStreamFirstFrame(_)))
3953 .unwrap_or_else(|| panic!("missing marker for {expected:?}"));
3954 match marker {
3955 NautilusWsMessage::AccountStreamFirstFrame(stream) => {
3956 assert_eq!(*stream, expected);
3957 }
3958 other => panic!("expected AccountStreamFirstFrame, was {other:?}"),
3959 }
3960 assert!(
3963 matches!(
3964 msgs.last(),
3965 Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3966 ),
3967 "marker must trail typed reports for {expected:?}",
3968 );
3969 }
3970 }
3971
3972 #[rstest]
3973 fn handle_frame_account_orders_without_context_falls_back_to_raw() {
3974 let signal = Arc::new(AtomicBool::new(false));
3975 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3976 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3977 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3978 let mut handler =
3979 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3980 handler.instruments.insert(0, stub_eth_perp_instrument());
3981 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3985 let messages = handler.handle_frame(frame, UnixNanos::from(11));
3986
3987 assert_eq!(messages.len(), 1);
3988 match &messages[0] {
3989 NautilusWsMessage::Raw(value) => {
3990 assert_eq!(value["type"], "update/account_orders");
3991 }
3992 other => panic!("expected raw fallback, was {other:?}"),
3993 }
3994 }
3995
3996 #[rstest]
3997 fn handle_frame_account_orders_skips_unknown_market() {
3998 let signal = Arc::new(AtomicBool::new(false));
4004 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4005 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4006 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4007 let mut handler =
4008 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4009 handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
4010 let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4013 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4014
4015 assert!(messages.is_empty());
4016 }
4017
4018 #[rstest]
4019 fn handle_frame_account_assets_invalid_timestamp_returns_empty() {
4020 let mut handler = make_handler_with_account();
4021 let frame_json = r#"{
4028 "type": "update/account_all_assets",
4029 "channel": "account_all_assets:1234",
4030 "timestamp": 18446744073709551615,
4031 "assets": {
4032 "0": {
4033 "symbol": "USDC",
4034 "asset_id": 0,
4035 "balance": "100.000000",
4036 "locked_balance": "1.000000"
4037 }
4038 }
4039 }"#;
4040 let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4041
4042 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4043
4044 assert!(messages.is_empty());
4045 }
4046
4047 #[rstest]
4048 fn handle_frame_account_all_orders_routes_to_execution_reports() {
4049 let mut handler = make_handler_with_account();
4053 let frame_json = r#"{
4054 "type": "update/account_all_orders",
4055 "channel": "account_all_orders:1234",
4056 "orders": {
4057 "0": [{
4058 "order_index": 281476929510110,
4059 "client_order_index": 42,
4060 "order_id": "281476929510110",
4061 "client_order_id": "42",
4062 "market_index": 0,
4063 "owner_account_index": 1234,
4064 "initial_base_amount": "0.0050",
4065 "price": "2352.74",
4066 "nonce": 9182390020,
4067 "remaining_base_amount": "0.0050",
4068 "is_ask": true,
4069 "base_size": 50,
4070 "base_price": 235274,
4071 "filled_base_amount": "0.0000",
4072 "filled_quote_amount": "0.000000",
4073 "side": "sell",
4074 "type": "limit",
4075 "time_in_force": "good-till-time",
4076 "reduce_only": false,
4077 "trigger_price": "0.00",
4078 "order_expiry": 1780360584479,
4079 "status": "open",
4080 "trigger_status": "na",
4081 "trigger_time": 0,
4082 "parent_order_index": 0,
4083 "parent_order_id": "0",
4084 "to_trigger_order_id_0": "0",
4085 "to_trigger_order_id_1": "0",
4086 "to_cancel_order_id_0": "0",
4087 "integrator_fee_collector_index": "0",
4088 "integrator_taker_fee": "0",
4089 "integrator_maker_fee": "0",
4090 "block_height": 227535532,
4091 "timestamp": 1777941383576,
4092 "created_at": 1777941383576,
4093 "updated_at": 1777941383576,
4094 "transaction_time": 1777941383576735
4095 }]
4096 }
4097 }"#;
4098 let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4099
4100 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4101
4102 assert_eq!(messages.len(), 1);
4103 match &messages[0] {
4104 NautilusWsMessage::ExecutionReports(reports) => {
4105 assert_eq!(reports.len(), 1);
4106 match &reports[0] {
4107 super::ExecutionReport::Order(order) => {
4108 assert_eq!(order.order_id, "281476929510110");
4109 }
4110 other => panic!("expected order report, was {other:?}"),
4111 }
4112 }
4113 other => panic!("expected execution reports, was {other:?}"),
4114 }
4115 }
4116
4117 fn snapshot_trade_frame_json() -> &'static str {
4118 r#"{
4119 "type": "subscribed/account_all_trades",
4120 "channel": "account_all_trades:1234",
4121 "trades": [{
4122 "trade_id": 19209006902,
4123 "trade_id_str": "19209006902",
4124 "tx_hash": "000000128b1ee814",
4125 "type": "trade",
4126 "market_id": 0,
4127 "size": "0.1336",
4128 "price": "2352.73",
4129 "usd_amount": "314.324728",
4130 "ask_id": 281476929510102,
4131 "bid_id": 562947905631053,
4132 "ask_client_id": 0,
4133 "bid_client_id": 7001011966,
4134 "ask_account_id": 91249,
4135 "bid_account_id": 1234,
4136 "is_maker_ask": true,
4137 "block_height": 227535535,
4138 "timestamp": 1777941384181,
4139 "transaction_time": 1777941384181586
4140 }],
4141 "total_volume": "100.0",
4142 "monthly_volume": "100.0",
4143 "weekly_volume": "100.0",
4144 "daily_volume": "100.0"
4145 }"#
4146 }
4147
4148 #[rstest]
4149 fn handle_frame_account_all_trades_snapshot_is_dropped_with_context() {
4150 let mut handler = make_handler_with_account();
4151 let frame: super::LighterWsFrame =
4156 serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4157
4158 let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4159
4160 assert!(messages.is_empty());
4161 }
4162
4163 #[rstest]
4164 fn handle_frame_account_all_trades_snapshot_falls_back_to_raw_without_context() {
4165 let signal = Arc::new(AtomicBool::new(false));
4166 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4167 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4168 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4169 let mut handler =
4170 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4171 handler.instruments.insert(0, stub_eth_perp_instrument());
4172 let frame: super::LighterWsFrame =
4176 serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4177 let messages = handler.handle_frame(frame, UnixNanos::from(11));
4178
4179 assert_eq!(messages.len(), 1);
4180 match &messages[0] {
4181 NautilusWsMessage::Raw(value) => {
4182 assert_eq!(value["type"], "subscribed/account_all_trades");
4183 }
4184 other => panic!("expected raw fallback, was {other:?}"),
4185 }
4186 }
4187
4188 #[rstest]
4189 fn handle_frame_market_stats_emits_mark_index_and_funding_updates() {
4190 let mut handler = make_handler_with_account();
4191 let frame: super::LighterWsFrame =
4192 serde_json::from_str(WS_MARKET_STATS_UPDATE_SINGLE).unwrap();
4193
4194 let messages = handler.handle_frame(frame, UnixNanos::from(11));
4195
4196 assert_eq!(messages.len(), 3);
4197 match &messages[0] {
4198 NautilusWsMessage::MarkPrice(update) => {
4199 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4200 assert_eq!(update.value, Price::from("2064.47"));
4201 assert_eq!(update.ts_event, UnixNanos::from(1_774_883_844_933_000_000));
4202 }
4203 event => panic!("expected mark price update, was {event:?}"),
4204 }
4205
4206 match &messages[1] {
4207 NautilusWsMessage::IndexPrice(update) => {
4208 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4209 assert_eq!(update.value, Price::from("2064.48"));
4210 }
4211 event => panic!("expected index price update, was {event:?}"),
4212 }
4213
4214 match &messages[2] {
4215 NautilusWsMessage::FundingRate(update) => {
4216 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4217 assert_eq!(update.rate, Decimal::new(1, 6));
4218 assert_eq!(update.next_funding_ns, None);
4219 }
4220 event => panic!("expected funding rate update, was {event:?}"),
4221 }
4222 }
4223
4224 #[rstest]
4225 fn handle_frame_market_stats_all_emits_mark_index_and_funding_updates() {
4226 let mut handler = make_handler_with_account();
4227 let frame: super::LighterWsFrame =
4228 serde_json::from_str(WS_MARKET_STATS_UPDATE_ALL).unwrap();
4229
4230 let messages = handler.handle_frame(frame, UnixNanos::from(11));
4231
4232 assert_eq!(messages.len(), 3);
4233 assert!(matches!(&messages[0], NautilusWsMessage::MarkPrice(_)));
4234 assert!(matches!(&messages[1], NautilusWsMessage::IndexPrice(_)));
4235 assert!(matches!(&messages[2], NautilusWsMessage::FundingRate(_)));
4236
4237 match &messages[0] {
4238 NautilusWsMessage::MarkPrice(update) => {
4239 assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4240 assert_eq!(update.value, Price::from("2064.47"));
4241 }
4242 event => panic!("expected mark price update, was {event:?}"),
4243 }
4244
4245 match &messages[2] {
4246 NautilusWsMessage::FundingRate(update) => {
4247 assert_eq!(update.rate, Decimal::new(1, 6));
4248 assert_eq!(update.next_funding_ns, None);
4249 }
4250 event => panic!("expected funding rate update, was {event:?}"),
4251 }
4252 }
4253
4254 #[rstest]
4255 fn handle_frame_spot_market_stats_emits_index_update() {
4256 let signal = Arc::new(AtomicBool::new(false));
4257 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4258 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4259 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4260 let mut handler =
4261 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4262 handler.instruments.insert(2048, stub_eth_spot_instrument());
4263 let frame: super::LighterWsFrame =
4264 serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_SINGLE).unwrap();
4265
4266 let messages = handler.handle_frame(frame, UnixNanos::from(11));
4267
4268 assert_eq!(messages.len(), 1);
4269 match &messages[0] {
4270 NautilusWsMessage::IndexPrice(update) => {
4271 assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4272 assert_eq!(update.value, Price::from("1.00"));
4273 }
4274 event => panic!("expected spot index price update, was {event:?}"),
4275 }
4276 }
4277
4278 #[rstest]
4279 fn handle_frame_spot_market_stats_all_emits_index_update() {
4280 let signal = Arc::new(AtomicBool::new(false));
4281 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4282 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4283 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4284 let mut handler =
4285 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4286 handler.instruments.insert(2048, stub_eth_spot_instrument());
4287 let frame: super::LighterWsFrame =
4288 serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_ALL).unwrap();
4289
4290 let messages = handler.handle_frame(frame, UnixNanos::from(11));
4291
4292 assert_eq!(messages.len(), 1);
4293 match &messages[0] {
4294 NautilusWsMessage::IndexPrice(update) => {
4295 assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4296 assert_eq!(update.value, Price::from("1.00"));
4297 }
4298 event => panic!("expected spot index price update, was {event:?}"),
4299 }
4300 }
4301
4302 #[rstest]
4303 #[case(LighterWsChannel::OrderBook(0), "order_book:0", "order_book/0")]
4304 #[case(LighterWsChannel::Trade(7), "trade:7", "trade/7")]
4305 #[case(LighterWsChannel::Ticker(2), "ticker:2", "ticker/2")]
4306 #[case(LighterWsChannel::Height, "height", "height")]
4307 #[case(
4308 LighterWsChannel::MarketStats(LighterMarketSelection::All),
4309 "market_stats:all",
4310 "market_stats/all"
4311 )]
4312 #[case(
4313 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(2048)),
4314 "spot_market_stats:2048",
4315 "spot_market_stats/2048"
4316 )]
4317 #[case(
4318 LighterWsChannel::OrderBook(4095),
4319 "order_book:4095",
4320 "order_book/4095"
4321 )]
4322 #[case(
4323 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(50_000)),
4324 "spot_market_stats:50000",
4325 "spot_market_stats/50000"
4326 )]
4327 #[case(
4328 LighterWsChannel::Candle {
4329 market_index: 40_000,
4330 resolution: LighterCandleResolution::OneHour,
4331 },
4332 "candle:40000:1h",
4333 "candle/40000/1h"
4334 )]
4335 #[case(
4336 LighterWsChannel::AccountOrders { market_index: 0, account_index: 1234 },
4337 "account_orders:0:1234",
4338 "account_orders/0/1234",
4339 )]
4340 fn topic_and_subscription_round_trip(
4341 #[case] channel: LighterWsChannel,
4342 #[case] expected_topic: &str,
4343 #[case] expected_subscription: &str,
4344 ) {
4345 assert_eq!(channel.topic_key(), expected_topic);
4346 assert_eq!(channel.subscription_channel(), expected_subscription);
4347 }
4348
4349 #[rstest]
4350 #[case("order_book:0", Some(0))]
4351 #[case("trade:42", Some(42))]
4352 #[case("order_book:4095", Some(4095))]
4353 #[case("trade:40000", Some(40_000))]
4354 #[case("market_stats:50000", Some(50_000))]
4355 #[case("height", None)]
4356 #[case("malformed", None)]
4357 fn market_index_extraction(#[case] topic: &str, #[case] expected: Option<i64>) {
4358 assert_eq!(market_index_from_topic(topic), expected);
4359 }
4360
4361 #[rstest]
4362 #[case("order_book:0", Some(0))]
4363 #[case("order_book:42", Some(42))]
4364 #[case("order_book:40000", Some(40_000))]
4365 #[case("trade:42", None)]
4366 #[case("ticker:2", None)]
4367 #[case("market_stats:0", None)]
4368 #[case("height", None)]
4369 #[case("order_book:not-an-int", None)]
4370 fn order_book_market_index_only_matches_order_book_channel(
4371 #[case] topic: &str,
4372 #[case] expected: Option<i64>,
4373 ) {
4374 assert_eq!(order_book_market_index_from_topic(topic), expected);
4375 }
4376
4377 #[rstest]
4378 #[case(LighterWsChannel::AccountAll(1234), true)]
4379 #[case(LighterWsChannel::OrderBook(0), false)]
4380 #[case(LighterWsChannel::AccountAllPositions(1), true)]
4381 #[case(LighterWsChannel::Trade(0), false)]
4382 fn requires_auth_classification(#[case] channel: LighterWsChannel, #[case] expected: bool) {
4383 assert_eq!(channel.requires_auth(), expected);
4384 }
4385
4386 #[rstest]
4387 fn handler_command_subscribe_debug_redacts_auth_token() {
4388 let token = "schnorr-signature-bytes-do-not-leak";
4389 let cmd = HandlerCommand::Subscribe {
4390 channel: LighterWsChannel::AccountAll(1234),
4391 auth: Some(SecretString::from(token)),
4392 response_tx: None,
4393 };
4394
4395 let dbg = format!("{cmd:?}");
4396
4397 assert!(
4398 !dbg.contains(token),
4399 "Debug output must not contain the auth token, found: {dbg}",
4400 );
4401 assert!(dbg.contains(REDACTED));
4402 }
4403
4404 #[tokio::test]
4405 async fn send_tx_command_returns_handler_send_error_without_active_client() {
4406 let signal = Arc::new(AtomicBool::new(false));
4407 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4408 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4409 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4410 let mut handler =
4411 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4412 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4413
4414 cmd_tx
4415 .send(HandlerCommand::SendTx {
4416 tx_type: LighterTxType::CreateOrder as u8,
4417 tx_info: serde_json::value::RawValue::from_string(
4418 r#"{"AccountIndex":12345,"Nonce":42}"#.to_string(),
4419 )
4420 .unwrap(),
4421 connection_epoch: 0,
4422 response_tx,
4423 })
4424 .unwrap();
4425 drop(cmd_tx);
4426 drop(raw_tx);
4427
4428 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4429 .await
4430 .expect("timed out waiting for handler to drain command");
4431 let result = response_rx.await.expect("sendTx response channel closed");
4432
4433 assert!(next.is_none());
4434 let Err(LighterWsError::Client(message)) = result else {
4435 panic!("expected client send error, was {result:?}");
4436 };
4437 assert!(message.contains("no active WebSocket client"));
4438 }
4439
4440 #[tokio::test]
4441 async fn resubscribe_order_book_command_skips_when_reference_removed() {
4442 let signal = Arc::new(AtomicBool::new(false));
4443 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4444 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4445 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4446 let subscriptions = SubscriptionState::new(':');
4447 let topic = LighterWsChannel::OrderBook(0).topic_key();
4448 assert!(subscriptions.add_reference(&topic));
4449 assert!(subscriptions.remove_reference(&topic));
4450
4451 let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions.clone());
4452 handler.book.delta_subs.insert(0);
4453
4454 let (completion, result) = tokio::sync::oneshot::channel();
4455 cmd_tx
4456 .send(HandlerCommand::RecoverBook {
4457 market_index: 0,
4458 cancel: CancellationToken::new(),
4459 gate: SnapshotGate::default(),
4460 completion,
4461 })
4462 .expect("queue resync");
4463 drop(cmd_tx);
4464 drop(raw_tx);
4465
4466 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4467 .await
4468 .expect("timed out waiting for handler to drain command");
4469
4470 assert!(next.is_none());
4471 assert!(subscriptions.pending_subscribe_topics().is_empty());
4472 assert!(subscriptions.pending_unsubscribe_topics().is_empty());
4473 assert!(result.await.is_err());
4474 }
4475
4476 #[tokio::test]
4477 async fn resubscribe_order_book_queues_when_inflight_is_at_cap() {
4478 let signal = Arc::new(AtomicBool::new(false));
4479 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4480 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4481 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4482 let subscriptions = SubscriptionState::new(':');
4483 let topic = LighterWsChannel::OrderBook(0).topic_key();
4484 assert!(subscriptions.add_reference(&topic));
4485
4486 let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions);
4487 handler.book.delta_subs.insert(0);
4488 saturate_subscription_gate(&mut handler);
4489
4490 let (completion, _rx) = tokio::sync::oneshot::channel();
4491 handler.queue_book_replacement(
4492 0,
4493 BookWrite {
4494 cancel: CancellationToken::new(),
4495 gate: SnapshotGate::default(),
4496 completion,
4497 },
4498 );
4499
4500 handler.pump_pending_subscribes().await;
4501
4502 assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4503 assert!(
4504 handler
4505 .pending_subs
4506 .iter()
4507 .any(|(topic, _)| *topic == Ustr::from("order_book:0")),
4508 );
4509 }
4510
4511 fn stub_candle(
4512 t: i64,
4513 open: i64,
4514 high: i64,
4515 low: i64,
4516 close: i64,
4517 volume_ticks: i64,
4518 ) -> LighterWsCandle {
4519 LighterWsCandle {
4520 t,
4521 o: Decimal::new(open, 2),
4522 h: Decimal::new(high, 2),
4523 l: Decimal::new(low, 2),
4524 c: Decimal::new(close, 2),
4525 v: Decimal::new(volume_ticks, 4),
4526 quote_volume: Decimal::ZERO,
4527 i: 0,
4528 }
4529 }
4530
4531 fn candle_frame(channel: &str, candle: LighterWsCandle, is_snapshot: bool) -> LighterWsFrame {
4532 if is_snapshot {
4533 LighterWsFrame::CandleSnapshot {
4534 channel: Ustr::from(channel),
4535 candles: vec![candle],
4536 timestamp: 0,
4537 }
4538 } else {
4539 LighterWsFrame::Candle {
4540 channel: Ustr::from(channel),
4541 candles: vec![candle],
4542 timestamp: 0,
4543 }
4544 }
4545 }
4546
4547 #[rstest]
4548 fn handle_candles_first_observation_caches_without_emit() {
4549 let mut handler = make_handler_with_account();
4550 let frame = candle_frame(
4551 "candle:0:1m",
4552 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
4553 true,
4554 );
4555
4556 let messages = handler.handle_frame(frame, UnixNanos::from(99));
4557
4558 assert!(messages.is_empty(), "first observation must not emit");
4559 let key = (0_i64, LighterCandleResolution::OneMinute);
4560 assert_eq!(handler.last_candles.get(&key).map(|c| c.t), Some(1_000_000));
4561 }
4562
4563 #[rstest]
4564 fn handle_candles_t_advance_emits_bar_for_previous_candle() {
4565 let mut handler = make_handler_with_account();
4566 let prev = stub_candle(1_000_000, 10_000, 11_000, 9_900, 10_500, 10_000);
4567 let next = stub_candle(1_060_000, 10_500, 10_600, 10_450, 10_550, 20_000);
4568 let next_t = next.t;
4569 handler.handle_frame(candle_frame("candle:0:1m", prev, true), UnixNanos::from(1));
4570
4571 let messages =
4572 handler.handle_frame(candle_frame("candle:0:1m", next, false), UnixNanos::from(2));
4573
4574 assert_eq!(messages.len(), 1);
4575 match &messages[0] {
4576 NautilusWsMessage::Bar(bar) => {
4577 assert_eq!(bar.open, Price::from("100.00"));
4579 assert_eq!(bar.high, Price::from("110.00"));
4580 assert_eq!(bar.low, Price::from("99.00"));
4581 assert_eq!(bar.close, Price::from("105.00"));
4582 assert_eq!(bar.volume, Quantity::from("1.0000"));
4583 assert_eq!(bar.ts_event, UnixNanos::from(1_000_000 * 1_000_000));
4584 }
4585 other => panic!("expected Bar message, was {other:?}"),
4586 }
4587 let cached = handler
4588 .last_candles
4589 .get(&(0_i64, LighterCandleResolution::OneMinute))
4590 .expect("cache populated");
4591 assert_eq!(cached.t, next_t);
4592 }
4593
4594 #[rstest]
4595 fn handle_candles_same_t_updates_cache_without_emit() {
4596 let mut handler = make_handler_with_account();
4597 let initial = stub_candle(1_000_000, 10_000, 10_050, 9_950, 10_025, 5_000);
4598 let same_t_updated = stub_candle(1_000_000, 10_000, 10_100, 9_950, 10_075, 7_500);
4599 let same_t_h = same_t_updated.h;
4600 let same_t_c = same_t_updated.c;
4601 handler.handle_frame(
4602 candle_frame("candle:0:1m", initial, true),
4603 UnixNanos::from(1),
4604 );
4605
4606 let messages = handler.handle_frame(
4607 candle_frame("candle:0:1m", same_t_updated, false),
4608 UnixNanos::from(2),
4609 );
4610
4611 assert!(messages.is_empty(), "same-`t` update must not emit");
4612 let cached = handler
4613 .last_candles
4614 .get(&(0_i64, LighterCandleResolution::OneMinute))
4615 .expect("cache populated");
4616 assert_eq!(cached.h, same_t_h);
4617 assert_eq!(cached.c, same_t_c);
4618 }
4619
4620 #[rstest]
4621 fn handle_candles_regressed_t_is_skipped() {
4622 let mut handler = make_handler_with_account();
4623 let initial = stub_candle(2_000_000, 10_000, 10_000, 10_000, 10_000, 5_000);
4624 let regressed = stub_candle(1_000_000, 9_000, 9_000, 9_000, 9_000, 5_000);
4625 let initial_t = initial.t;
4626 handler.handle_frame(
4627 candle_frame("candle:0:1m", initial, true),
4628 UnixNanos::from(1),
4629 );
4630
4631 let messages = handler.handle_frame(
4632 candle_frame("candle:0:1m", regressed, false),
4633 UnixNanos::from(2),
4634 );
4635
4636 assert!(messages.is_empty(), "regressed `t` must not emit");
4637 let cached = handler
4638 .last_candles
4639 .get(&(0_i64, LighterCandleResolution::OneMinute))
4640 .expect("cache populated");
4641 assert_eq!(cached.t, initial_t);
4643 }
4644
4645 #[rstest]
4646 fn handle_candles_unknown_market_returns_empty() {
4647 let mut handler = make_handler_with_account();
4648 let frame = candle_frame(
4649 "candle:99:1m",
4650 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 5_000),
4651 true,
4652 );
4653
4654 let messages = handler.handle_frame(frame, UnixNanos::from(1));
4655
4656 assert!(messages.is_empty());
4657 }
4658
4659 #[rstest]
4660 fn handle_unsubscribe_ack_clears_only_matching_candle_key() {
4661 let mut handler = make_handler_with_account();
4662 handler.last_candles.insert(
4663 (0, LighterCandleResolution::OneMinute),
4664 stub_candle(1, 0, 0, 0, 0, 0),
4665 );
4666 handler.last_candles.insert(
4667 (0, LighterCandleResolution::FiveMinute),
4668 stub_candle(2, 0, 0, 0, 0, 0),
4669 );
4670 handler.subscriptions.mark_unsubscribe("candle:0:1m");
4671
4672 let payload = json!({"type": "unsubscribed", "channel": "candle:0:1m"});
4673 let (matched, _) = handle_control_text(&mut handler, &payload.to_string());
4674
4675 assert!(matched);
4676 assert!(
4677 handler
4678 .last_candles
4679 .get(&(0, LighterCandleResolution::OneMinute))
4680 .is_none(),
4681 );
4682 assert!(
4683 handler
4684 .last_candles
4685 .get(&(0, LighterCandleResolution::FiveMinute))
4686 .is_some(),
4687 );
4688 }
4689
4690 #[rstest]
4691 #[case::well_formed("candle:0:1m", Some((0, LighterCandleResolution::OneMinute)))]
4692 #[case::weekly("candle:3:1w", Some((3, LighterCandleResolution::OneWeek)))]
4693 #[case::widened_id("candle:40000:1h", Some((40_000, LighterCandleResolution::OneHour)))]
4694 #[case::other_kind("order_book:0", None)]
4695 #[case::missing_resolution("candle:0", None)]
4696 #[case::bad_market("candle:notanint:1m", None)]
4697 #[case::bad_resolution("candle:0:bogus", None)]
4698 fn test_candle_market_and_resolution_from_topic(
4699 #[case] topic: &str,
4700 #[case] expected: Option<(i64, LighterCandleResolution)>,
4701 ) {
4702 assert_eq!(candle_market_and_resolution_from_topic(topic), expected);
4703 }
4704
4705 #[rstest]
4706 #[case::network_retries(LighterWsError::Network("disconnected".into()), true)]
4707 #[case::auth_does_not_retry(LighterWsError::Authentication("bad token".into()), false)]
4708 #[case::parse_does_not_retry(LighterWsError::Parse("bad json".into()), false)]
4709 #[case::client_does_not_retry(LighterWsError::Client("no active WebSocket client".into()), false)]
4710 #[case::transport_closed_does_not_retry(LighterWsError::Transport(SendError::Closed), false)]
4711 #[case::transport_invalid_input_does_not_retry(
4712 LighterWsError::Transport(SendError::InvalidInput("pong payload too large".into())),
4713 false
4714 )]
4715 #[case::transport_connection_changed_does_not_retry(
4716 LighterWsError::Transport(SendError::ConnectionChanged),
4717 false
4718 )]
4719 #[case::transport_timeout_retries(LighterWsError::Transport(SendError::Timeout), true)]
4720 #[case::transport_write_timeout_does_not_retry(
4721 LighterWsError::Transport(SendError::WriteTimeout),
4722 false
4723 )]
4724 #[case::transport_broken_pipe_does_not_retry(
4725 LighterWsError::Transport(SendError::BrokenPipe(
4726 "writer closed".into(),
4727 )),
4728 false,
4729 )]
4730 #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), false)]
4731 fn test_should_retry_lighter_ws_error(#[case] error: LighterWsError, #[case] expected: bool) {
4732 assert_eq!(should_retry_lighter_ws_error(&error), expected);
4733 }
4734
4735 #[rstest]
4737 #[case::closed(SendError::Closed)]
4738 #[case::invalid_input(SendError::InvalidInput("pong payload too large".into()))]
4739 #[case::timeout(SendError::Timeout)]
4740 #[case::write_timeout(SendError::WriteTimeout)]
4741 #[case::connection_changed(SendError::ConnectionChanged)]
4742 #[case::broken_pipe(SendError::BrokenPipe("writer dropped".into()))]
4743 fn send_error_converts_into_transport_variant(#[case] send_error: SendError) {
4744 let err: LighterWsError = send_error.into();
4745 assert!(
4746 matches!(err, LighterWsError::Transport(_)),
4747 "expected Transport variant, was {err:?}",
4748 );
4749 }
4750
4751 #[tokio::test]
4752 async fn subscribe_command_parks_in_pending_subs_when_inflight_at_cap() {
4753 let signal = Arc::new(AtomicBool::new(false));
4754 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4755 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4756 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4757 let mut handler =
4758 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4759
4760 saturate_subscription_gate(&mut handler);
4762
4763 cmd_tx
4764 .send(HandlerCommand::Subscribe {
4765 channel: LighterWsChannel::Candle {
4766 market_index: 0,
4767 resolution: LighterCandleResolution::OneMinute,
4768 },
4769 auth: None,
4770 response_tx: None,
4771 })
4772 .expect("queue subscribe");
4773 drop(cmd_tx);
4774 drop(raw_tx);
4775
4776 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4777 .await
4778 .expect("timed out waiting for handler to drain command");
4779
4780 assert!(next.is_none());
4781 assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4782 assert_eq!(handler.pending_subs.len(), 1);
4783 assert_eq!(handler.subscription_attempts.len(), 1);
4784 assert_eq!(handler.pending_subs[0].0, Ustr::from("candle:0:1m"));
4785 }
4786
4787 #[tokio::test]
4788 async fn unsubscribe_drops_queued_subscribe_while_gate_full() {
4789 let signal = Arc::new(AtomicBool::new(false));
4790 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4791 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4792 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4793 let mut handler =
4794 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4795
4796 saturate_subscription_gate(&mut handler);
4798
4799 cmd_tx
4800 .send(HandlerCommand::Subscribe {
4801 channel: LighterWsChannel::Trade(0),
4802 auth: None,
4803 response_tx: None,
4804 })
4805 .expect("queue subscribe");
4806 cmd_tx
4807 .send(HandlerCommand::Unsubscribe {
4808 channel: LighterWsChannel::Trade(0),
4809 })
4810 .expect("queue unsubscribe");
4811 drop(cmd_tx);
4812 drop(raw_tx);
4813
4814 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4815 .await
4816 .expect("timed out waiting for handler to drain commands");
4817
4818 assert!(next.is_none());
4819 assert!(handler.pending_subs.is_empty());
4820 assert!(handler.subscription_attempts.is_empty());
4821 }
4822
4823 #[tokio::test]
4824 async fn reconnect_requeues_attempt_with_new_generation() {
4825 let signal = Arc::new(AtomicBool::new(false));
4826 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4827 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4828 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4829 let mut handler =
4830 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4831
4832 saturate_subscription_gate(&mut handler);
4833 handler.queue_subscribe(LighterWsChannel::Trade(0), None, None);
4834 let old_generation = handler.pending_subs[0].1;
4835
4836 raw_tx
4837 .send((7, Message::Text(RECONNECTED.to_string().into())))
4838 .expect("queue reconnect sentinel");
4839
4840 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4841 .await
4842 .expect("timed out waiting for reconnect");
4843
4844 assert!(matches!(next, Some(NautilusWsMessage::Reconnected { .. })));
4845 assert!(handler.inflight_subs.is_empty());
4846 assert_eq!(handler.pending_subs.len(), 1);
4847 assert_eq!(handler.pending_subs[0].0, Ustr::from("trade:0"));
4848 assert_ne!(handler.pending_subs[0].1, old_generation);
4849 assert_eq!(
4850 handler.subscription_attempts[&Ustr::from("trade:0")].generation,
4851 handler.pending_subs[0].1,
4852 );
4853 }
4854
4855 #[rstest]
4856 #[tokio::test]
4857 async fn pump_releases_inflight_slot_and_schedules_send_failure_retry() {
4858 let signal = Arc::new(AtomicBool::new(false));
4859 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4860 let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4861 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4862 let mut handler =
4863 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4864
4865 for market_index in 0..3 {
4867 handler.queue_subscribe(LighterWsChannel::Trade(market_index), None, None);
4868 }
4869
4870 handler.pump_pending_subscribes().await;
4871
4872 assert!(handler.pending_subs.is_empty());
4873 assert!(handler.inflight_subs.is_empty());
4874 assert_eq!(handler.subscription_attempts.len(), 3);
4875 assert!(
4876 handler
4877 .subscription_attempts
4878 .values()
4879 .all(|attempt| attempt.retries == 1),
4880 );
4881 assert_eq!(handler.subscription_retries.len(), 3);
4882 }
4883
4884 #[rstest]
4885 fn subscribed_control_frame_releases_inflight_slot() {
4886 let mut handler = make_handler_with_account();
4887 mark_subscription_inflight(
4888 &mut handler,
4889 LighterWsChannel::Candle {
4890 market_index: 0,
4891 resolution: LighterCandleResolution::OneMinute,
4892 },
4893 None,
4894 );
4895
4896 let (matched, msg) = handle_control_text(
4897 &mut handler,
4898 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
4899 );
4900
4901 assert!(matched);
4902 assert!(msg.is_none());
4903 assert!(
4904 !handler
4905 .inflight_subs
4906 .contains_key(&Ustr::from("candle:0:1m"))
4907 );
4908 assert!(handler.subscription_attempts.is_empty());
4909 }
4910
4911 #[rstest]
4912 #[case::top_level(
4913 r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_orders:12345"}"#
4914 )]
4915 #[case::nested(
4916 r#"{"error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4917 )]
4918 #[case::nested_typed(
4919 r#"{"type":"error","error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4920 )]
4921 fn already_subscribed_confirms_matching_inflight_topic(#[case] payload: &str) {
4922 let mut handler = make_handler_with_account();
4923 mark_subscription_inflight(
4924 &mut handler,
4925 LighterWsChannel::AccountAllOrders(12345),
4926 None,
4927 );
4928
4929 let (matched, msg) = handle_control_text(&mut handler, payload);
4930
4931 assert!(matched);
4932 assert!(msg.is_none());
4933 assert!(handler.inflight_subs.is_empty());
4934 assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
4935 assert_eq!(handler.subscriptions.len(), 1);
4936 }
4937
4938 #[rstest]
4939 fn already_subscribed_does_not_confirm_unmatched_topic() {
4940 let mut handler = make_handler_with_account();
4941 let inflight = "account_all_orders:12345";
4942 mark_subscription_inflight(
4943 &mut handler,
4944 LighterWsChannel::AccountAllOrders(12345),
4945 None,
4946 );
4947
4948 let (matched, msg) = handle_control_text(
4949 &mut handler,
4950 r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_trades:12345"}"#,
4951 );
4952
4953 assert!(matched);
4954 assert!(msg.is_none());
4955 assert!(handler.inflight_subs.contains_key(&Ustr::from(inflight)));
4956 assert_eq!(
4957 handler.subscriptions.pending_subscribe_topics(),
4958 vec![inflight]
4959 );
4960 assert_eq!(handler.subscriptions.len(), 0);
4961 }
4962
4963 #[tokio::test]
4964 async fn duplicate_queued_and_inflight_topics_share_one_generation() {
4965 let mut handler = make_handler_with_account();
4966 let (response_tx_1, mut response_rx_1) = tokio::sync::oneshot::channel();
4967 let (response_tx_2, mut response_rx_2) = tokio::sync::oneshot::channel();
4968 let (response_tx_3, mut response_rx_3) = tokio::sync::oneshot::channel();
4969 let channel = LighterWsChannel::Trade(7);
4970
4971 handler.queue_subscribe(channel.clone(), None, Some(response_tx_1));
4972 let generation = handler.pending_subs[0].1;
4973 handler.queue_subscribe(channel.clone(), None, Some(response_tx_2));
4974
4975 assert_eq!(handler.pending_subs.len(), 1);
4976 assert_eq!(handler.subscription_attempts.len(), 1);
4977 assert_eq!(
4978 handler.subscription_attempts[&Ustr::from("trade:7")]
4979 .response_txs
4980 .len(),
4981 2,
4982 );
4983
4984 let (topic, queued_generation) = handler.pending_subs.pop_front().unwrap();
4985 handler.inflight_subs.insert(topic, queued_generation);
4986 handler.queue_subscribe(channel, None, Some(response_tx_3));
4987
4988 assert_eq!(queued_generation, generation);
4989 assert!(handler.pending_subs.is_empty());
4990 assert_eq!(handler.inflight_subs.len(), 1);
4991 assert_eq!(
4992 handler.subscription_attempts[&Ustr::from("trade:7")]
4993 .response_txs
4994 .len(),
4995 3,
4996 );
4997 assert!(matches!(
4998 response_rx_1.try_recv(),
4999 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5000 ));
5001 assert!(matches!(
5002 response_rx_2.try_recv(),
5003 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5004 ));
5005 assert!(matches!(
5006 response_rx_3.try_recv(),
5007 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5008 ));
5009
5010 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5011
5012 assert_eq!(response_rx_1.await.unwrap(), Ok(()));
5013 assert_eq!(response_rx_2.await.unwrap(), Ok(()));
5014 assert_eq!(response_rx_3.await.unwrap(), Ok(()));
5015 assert!(handler.inflight_subs.is_empty());
5016 assert!(handler.subscription_attempts.is_empty());
5017 }
5018
5019 #[tokio::test]
5020 async fn changed_inflight_auth_waits_for_serialized_successor() {
5021 let mut handler = make_handler_with_account();
5022 let channel = LighterWsChannel::AccountAllOrders(12345);
5023 let topic = Ustr::from(channel.topic_key().as_str());
5024 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5025 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5026
5027 handler.queue_subscribe(
5028 channel.clone(),
5029 Some(SecretString::from("old-token")),
5030 Some(old_tx),
5031 );
5032 let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5033 handler.inflight_subs.insert(queued_topic, old_generation);
5034 handler.queue_subscribe(
5035 channel,
5036 Some(SecretString::from("fresh-token")),
5037 Some(fresh_tx),
5038 );
5039
5040 let attempt = &handler.subscription_attempts[&topic];
5041 assert_eq!(
5042 attempt.auth.as_ref().map(SecretString::expose_secret),
5043 Some("old-token")
5044 );
5045 assert_eq!(
5046 attempt
5047 .pending_auth
5048 .as_ref()
5049 .map(SecretString::expose_secret),
5050 Some("fresh-token")
5051 );
5052 assert_eq!(attempt.response_txs.len(), 1);
5053 assert_eq!(attempt.pending_response_txs.len(), 1);
5054
5055 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5056 assert_eq!(old_rx.await.unwrap(), Ok(()));
5057 assert!(matches!(
5058 fresh_rx.try_recv(),
5059 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5060 ));
5061 let attempt = &handler.subscription_attempts[&topic];
5062 assert_ne!(attempt.generation, old_generation);
5063 assert_eq!(
5064 attempt.auth.as_ref().map(SecretString::expose_secret),
5065 Some("fresh-token")
5066 );
5067 assert!(attempt.pending_auth.is_none());
5068 assert_eq!(
5069 handler.pending_subs.front(),
5070 Some(&(topic, attempt.generation)),
5071 );
5072
5073 let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5074 handler.inflight_subs.insert(topic, fresh_generation);
5075
5076 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5079 assert!(matches!(
5080 fresh_rx.try_recv(),
5081 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5082 ));
5083 assert_eq!(
5084 handler.subscription_attempts[&topic].generation,
5085 fresh_generation,
5086 );
5087
5088 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5089 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5090 assert!(handler.subscription_attempts.is_empty());
5091 }
5092
5093 #[tokio::test]
5094 async fn stale_typed_frame_does_not_complete_fresh_attempt_after_predecessor_removed() {
5095 let mut handler = make_handler_with_account();
5096 let channel = LighterWsChannel::AccountAllOrders(12345);
5097 let topic = Ustr::from(channel.topic_key().as_str());
5098 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5099 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5100
5101 handler.queue_subscribe(
5104 channel.clone(),
5105 Some(SecretString::from("old-token")),
5106 Some(old_tx),
5107 );
5108 let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5109 handler.inflight_subs.insert(queued_topic, old_generation);
5110 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5111 assert_eq!(old_rx.await.unwrap(), Ok(()));
5112 assert!(handler.subscription_attempts.is_empty());
5113
5114 handler.queue_subscribe(
5116 channel,
5117 Some(SecretString::from("fresh-token")),
5118 Some(fresh_tx),
5119 );
5120 let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5121 handler.inflight_subs.insert(topic, fresh_generation);
5122
5123 assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5125 assert!(matches!(
5126 fresh_rx.try_recv(),
5127 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5128 ));
5129 assert_eq!(
5130 handler.subscription_attempts[&topic].generation,
5131 fresh_generation,
5132 );
5133
5134 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5135 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5136 assert!(handler.subscription_attempts.is_empty());
5137 }
5138
5139 #[tokio::test]
5140 async fn changed_queued_auth_updates_existing_generation_before_dispatch() {
5141 let mut handler = make_handler_with_account();
5142 let channel = LighterWsChannel::AccountAllOrders(12345);
5143 let topic = Ustr::from(channel.topic_key().as_str());
5144 let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5145 let (fresh_tx, fresh_rx) = tokio::sync::oneshot::channel();
5146
5147 handler.queue_subscribe(
5148 channel.clone(),
5149 Some(SecretString::from("old-token")),
5150 Some(old_tx),
5151 );
5152 let generation = handler.pending_subs[0].1;
5153 handler.queue_subscribe(
5154 channel,
5155 Some(SecretString::from("fresh-token")),
5156 Some(fresh_tx),
5157 );
5158
5159 assert_eq!(handler.pending_subs.len(), 1);
5160 assert_eq!(handler.pending_subs[0], (topic, generation));
5161 let attempt = &handler.subscription_attempts[&topic];
5162 assert_eq!(attempt.generation, generation);
5163 assert_eq!(
5164 attempt.auth.as_ref().map(SecretString::expose_secret),
5165 Some("fresh-token")
5166 );
5167 assert!(attempt.pending_auth.is_none());
5168 assert_eq!(attempt.response_txs.len(), 2);
5169 assert!(attempt.pending_response_txs.is_empty());
5170
5171 handler.pending_subs.pop_front();
5172 handler.inflight_subs.insert(topic, generation);
5173 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5174 assert_eq!(old_rx.await.unwrap(), Ok(()));
5175 assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5176 }
5177
5178 #[rstest]
5179 fn second_reconnect_folds_pending_auth_into_requeued_generation() {
5180 let mut handler = make_handler_with_account();
5181 let channel = LighterWsChannel::AccountAllOrders(12345);
5182 let topic = Ustr::from(channel.topic_key().as_str());
5183 let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5184 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5185
5186 handler.queue_subscribe(
5187 channel.clone(),
5188 Some(SecretString::from("old-token")),
5189 Some(old_tx),
5190 );
5191 handler.reset_subscription_attempts_after_reconnect();
5192 let (_, replay_generation) = handler.pending_subs.pop_front().unwrap();
5193 handler.inflight_subs.insert(topic, replay_generation);
5194 handler.queue_subscribe(
5195 channel,
5196 Some(SecretString::from("fresh-token")),
5197 Some(fresh_tx),
5198 );
5199
5200 let attempt = &handler.subscription_attempts[&topic];
5201 assert_eq!(
5202 attempt.auth.as_ref().map(SecretString::expose_secret),
5203 Some("old-token")
5204 );
5205 assert_eq!(
5206 attempt
5207 .pending_auth
5208 .as_ref()
5209 .map(SecretString::expose_secret),
5210 Some("fresh-token")
5211 );
5212 assert_eq!(attempt.response_txs.len(), 1);
5213 assert_eq!(attempt.pending_response_txs.len(), 1);
5214
5215 handler.reset_subscription_attempts_after_reconnect();
5216
5217 let attempt = &handler.subscription_attempts[&topic];
5218 assert_ne!(attempt.generation, replay_generation);
5219 assert_eq!(
5220 attempt.auth.as_ref().map(SecretString::expose_secret),
5221 Some("fresh-token")
5222 );
5223 assert!(attempt.pending_auth.is_none());
5224 assert_eq!(attempt.response_txs.len(), 2);
5225 assert!(attempt.pending_response_txs.is_empty());
5226 assert_eq!(handler.pending_subs.len(), 1);
5227 assert_eq!(handler.pending_subs[0], (topic, attempt.generation));
5228 assert!(handler.inflight_subs.is_empty());
5229 assert!(matches!(
5230 old_rx.try_recv(),
5231 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5232 ));
5233 assert!(matches!(
5234 fresh_rx.try_recv(),
5235 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5236 ));
5237
5238 let (_, generation) = handler.pending_subs.pop_front().unwrap();
5239 handler.inflight_subs.insert(topic, generation);
5240 assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5241
5242 assert_eq!(old_rx.try_recv(), Ok(Ok(())));
5243 assert_eq!(fresh_rx.try_recv(), Ok(Ok(())));
5244 assert!(handler.subscription_attempts.is_empty());
5245 }
5246
5247 #[tokio::test]
5248 async fn retry_folds_pending_auth_without_resetting_retry_budget() {
5249 let mut handler = make_handler_with_account();
5250 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5251 handler.set_command_sender(cmd_tx);
5252 let channel = LighterWsChannel::AccountAllOrders(12345);
5253 let topic = Ustr::from(channel.topic_key().as_str());
5254 let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5255 let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5256
5257 handler.queue_subscribe(
5258 channel.clone(),
5259 Some(SecretString::from("old-token")),
5260 Some(old_tx),
5261 );
5262 let (_, generation) = handler.pending_subs.pop_front().unwrap();
5263 handler.inflight_subs.insert(topic, generation);
5264 handler
5265 .subscription_attempts
5266 .get_mut(&topic)
5267 .unwrap()
5268 .retries = 2;
5269 handler.queue_subscribe(
5270 channel,
5271 Some(SecretString::from("fresh-token")),
5272 Some(fresh_tx),
5273 );
5274
5275 handler.schedule_subscription_retry(topic, generation, "retry");
5276
5277 let attempt = &handler.subscription_attempts[&topic];
5278 assert_eq!(attempt.retries, 3);
5279 assert_ne!(attempt.generation, generation);
5280 assert_eq!(
5281 attempt.auth.as_ref().map(SecretString::expose_secret),
5282 Some("fresh-token")
5283 );
5284 assert!(attempt.pending_auth.is_none());
5285 assert_eq!(attempt.response_txs.len(), 2);
5286 assert!(attempt.pending_response_txs.is_empty());
5287 assert!(handler.pending_subs.is_empty());
5288 assert!(handler.inflight_subs.is_empty());
5289 assert_eq!(handler.subscription_retries.len(), 1);
5290 assert!(matches!(
5291 old_rx.try_recv(),
5292 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5293 ));
5294 assert!(matches!(
5295 fresh_rx.try_recv(),
5296 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5297 ));
5298 }
5299
5300 #[tokio::test]
5301 async fn confirmed_authenticated_topic_opens_new_generation() {
5302 let mut handler = make_handler_with_account();
5303 let channel = LighterWsChannel::AccountAllOrders(12345);
5304 let topic = channel.topic_key();
5305 handler.subscriptions.mark_subscribe(&topic);
5306 handler.subscriptions.confirm_subscribe(&topic);
5307 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5308
5309 handler.queue_subscribe(
5310 channel,
5311 Some(SecretString::from("rotated-auth-token")),
5312 Some(response_tx),
5313 );
5314
5315 assert_eq!(handler.pending_subs.len(), 1);
5316 assert_eq!(handler.subscription_attempts.len(), 1);
5317 assert_eq!(handler.subscriptions.len(), 1);
5318 assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5319
5320 let (topic, generation) = handler.pending_subs.pop_front().unwrap();
5321 handler.inflight_subs.insert(topic, generation);
5322 handle_control_text(
5323 &mut handler,
5324 r#"{"type":"subscribed","channel":"account_all_orders:12345"}"#,
5325 );
5326
5327 assert_eq!(response_rx.await.unwrap(), Ok(()));
5328 assert!(handler.subscription_attempts.is_empty());
5329 assert_eq!(handler.subscriptions.len(), 1);
5330 }
5331
5332 #[tokio::test]
5333 async fn typed_update_before_ack_does_not_complete_subscription_generation() {
5334 let mut handler = make_handler_with_account();
5335 let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
5336 let (topic, generation) = mark_subscription_inflight(
5337 &mut handler,
5338 LighterWsChannel::Candle {
5339 market_index: 0,
5340 resolution: LighterCandleResolution::OneMinute,
5341 },
5342 Some(response_tx),
5343 );
5344
5345 handler.handle_frame(
5346 candle_frame(
5347 "candle:0:1m",
5348 stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
5349 false,
5350 ),
5351 UnixNanos::from(1),
5352 );
5353
5354 assert_eq!(
5355 handler
5356 .last_candles
5357 .get(&(0, LighterCandleResolution::OneMinute))
5358 .map(|candle| candle.t),
5359 Some(1_000_000),
5360 );
5361 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5362 assert_eq!(
5363 handler.subscriptions.pending_subscribe_topics(),
5364 vec!["candle:0:1m"],
5365 );
5366 assert_eq!(handler.subscriptions.len(), 0);
5367 assert!(matches!(
5368 response_rx.try_recv(),
5369 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5370 ));
5371
5372 handle_control_text(
5373 &mut handler,
5374 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5375 );
5376
5377 assert_eq!(response_rx.await.unwrap(), Ok(()));
5378 assert_eq!(handler.subscriptions.len(), 1);
5379 }
5380
5381 #[tokio::test]
5382 async fn typed_subscribed_frame_completes_subscription_generation() {
5383 let signal = Arc::new(AtomicBool::new(false));
5384 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5385 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5386 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5387 let mut handler =
5388 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5389 handler.instruments.insert(0, stub_eth_perp_instrument());
5390 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5391 mark_subscription_inflight(
5392 &mut handler,
5393 LighterWsChannel::Candle {
5394 market_index: 0,
5395 resolution: LighterCandleResolution::OneMinute,
5396 },
5397 Some(response_tx),
5398 );
5399
5400 raw_tx
5401 .send((7, Message::Text(WS_CANDLE_SUBSCRIBED.into())))
5402 .expect("typed subscribed frame");
5403 drop(raw_tx);
5404 drop(cmd_tx);
5405
5406 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5407 .await
5408 .expect("handler did not process typed subscribed frame");
5409
5410 assert!(next.is_none());
5411 assert_eq!(response_rx.await.unwrap(), Ok(()));
5412 assert!(handler.inflight_subs.is_empty());
5413 assert!(handler.subscription_attempts.is_empty());
5414 assert_eq!(handler.subscriptions.len(), 1);
5415 }
5416
5417 #[tokio::test]
5418 async fn unparsable_subscribed_frame_still_completes_subscription() {
5419 let signal = Arc::new(AtomicBool::new(false));
5420 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5421 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5422 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5423 let mut handler =
5424 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5425 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5426 mark_subscription_inflight(
5427 &mut handler,
5428 LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(4098)),
5429 Some(response_tx),
5430 );
5431
5432 raw_tx
5433 .send((7, Message::Text(WS_SPOT_STATS_SUBSCRIBED_BAD_BODY.into())))
5434 .expect("unparsable subscribed frame");
5435 drop(raw_tx);
5436 drop(cmd_tx);
5437
5438 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5439 .await
5440 .expect("handler did not process unparsable subscribed frame");
5441
5442 assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5443 let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5444 .await
5445 .expect("subscription was not completed");
5446 assert_eq!(response.unwrap(), Ok(()));
5447 assert!(handler.inflight_subs.is_empty());
5448 assert!(handler.subscription_attempts.is_empty());
5449 assert_eq!(handler.subscriptions.len(), 1);
5450 }
5451
5452 #[tokio::test]
5453 async fn unparsable_book_subscribed_frame_still_completes_subscription() {
5454 let signal = Arc::new(AtomicBool::new(false));
5455 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5456 let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5457 let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5458 let mut handler =
5459 FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5460 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5461 mark_subscription_inflight(
5462 &mut handler,
5463 LighterWsChannel::OrderBook(4095),
5464 Some(response_tx),
5465 );
5466
5467 raw_tx
5470 .send((0, Message::Text(WS_BOOK_SUBSCRIBED_BAD_BODY.into())))
5471 .expect("unparsable book subscribed frame");
5472 drop(raw_tx);
5473 drop(cmd_tx);
5474
5475 let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5476 .await
5477 .expect("handler did not process unparsable book subscribed frame");
5478
5479 assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5480 let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5481 .await
5482 .expect("book subscription was not completed");
5483 assert_eq!(response.unwrap(), Ok(()));
5484 assert!(handler.inflight_subs.is_empty());
5485 assert!(handler.subscription_attempts.is_empty());
5486 assert_eq!(handler.subscriptions.len(), 1);
5487 }
5488
5489 #[rstest]
5490 #[case::nonbook_top_level(false, false, "market_stats:0")]
5491 #[case::nonbook_wrapped(false, true, "market_stats/0")]
5492 #[case::book_top_level(true, false, "order_book/0")]
5493 #[case::book_wrapped(true, true, "order_book:0")]
5494 #[tokio::test]
5495 async fn subscribe_failure_preserves_distinct_inflight_subscription(
5496 #[case] book: bool,
5497 #[case] wrapped: bool,
5498 #[case] channel: &str,
5499 ) {
5500 let mut handler = make_handler_with_account();
5501 let (failed_tx, mut failed_rx) = tokio::sync::oneshot::channel();
5502 let (valid_tx, mut valid_rx) = tokio::sync::oneshot::channel();
5503
5504 let failed_channel = if book {
5505 LighterWsChannel::OrderBook(0)
5506 } else {
5507 LighterWsChannel::MarketStats(LighterMarketSelection::Market(0))
5508 };
5509
5510 let (failed_topic, _) =
5511 mark_subscription_inflight(&mut handler, failed_channel, Some(failed_tx));
5512 let (valid_topic, generation) =
5513 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(valid_tx));
5514 let error = serde_json::json!({"code": 30012, "channel": channel});
5515
5516 let value = if wrapped {
5517 serde_json::json!({"error": error})
5518 } else {
5519 error
5520 };
5521
5522 let (matched, message) = handler.handle_control_value(&value);
5523 let pending = valid_rx.try_recv();
5524 let inflight = handler.inflight_subs.get(&valid_topic).copied();
5525 let retries = handler.subscription_attempts[&valid_topic].retries;
5526 let rejected = failed_rx.try_recv();
5527 let failure_retained = handler.subscription_attempts.contains_key(&failed_topic);
5528 handler.handle_control_value(&serde_json::json!({
5529 "type": "subscribed", "channel": "trade:7"
5530 }));
5531
5532 let expected_error = if book {
5533 BOOK_SUBSCRIBE_REJECTION.to_string()
5534 } else {
5535 "subscription market_stats:0 failed after 1 attempts: venue rejected the WebSocket \
5536 subscribe with code 30012"
5537 .to_string()
5538 };
5539
5540 assert!(matched);
5541 assert!(message.is_none());
5542 assert_eq!(
5543 pending,
5544 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5545 );
5546 assert_eq!(inflight, Some(generation));
5547 assert_eq!(retries, 0);
5548 assert_eq!(rejected, Ok(Err(expected_error)));
5549 assert!(!failure_retained);
5550 assert_eq!(valid_rx.try_recv(), Ok(Ok(())));
5551 assert!(handler.inflight_subs.is_empty());
5552 assert!(handler.subscription_attempts.is_empty());
5553 assert_eq!(handler.subscriptions.len(), 1);
5554 }
5555
5556 #[rstest]
5557 #[case::unknown("trade:999")]
5558 #[case::confirmed("trade:7")]
5559 #[case::queued("market_stats:0")]
5560 #[tokio::test]
5561 async fn subscribe_failure_ignores_channel_without_inflight_attempt(#[case] channel: &str) {
5562 let mut handler = make_handler_with_account();
5563 let (response, mut result) = tokio::sync::oneshot::channel();
5564 let (topic, generation) =
5565 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(3), Some(response));
5566 handler.subscriptions.mark_subscribe("trade:7");
5567 handler.subscriptions.confirm_subscribe("trade:7");
5568 handler.queue_subscribe(
5569 LighterWsChannel::MarketStats(LighterMarketSelection::Market(0)),
5570 None,
5571 None,
5572 );
5573 let (queued_topic, queued_generation) = *handler.pending_subs.front().unwrap();
5574
5575 handler.handle_control_value(&serde_json::json!({"code": 30012, "channel": channel}));
5576
5577 assert_eq!(
5578 result.try_recv(),
5579 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5580 );
5581 assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5582 assert_eq!(
5583 handler.subscription_attempts[&queued_topic].generation,
5584 queued_generation
5585 );
5586 assert_eq!(handler.subscription_attempts[&queued_topic].retries, 0);
5587 assert_eq!(
5588 handler.pending_subs.iter().copied().collect::<Vec<_>>(),
5589 [(queued_topic, queued_generation)]
5590 );
5591 assert!(handler.subscription_retries.is_empty());
5592 assert_eq!(handler.subscriptions.len(), 1);
5593 }
5594
5595 #[tokio::test]
5596 async fn failed_subscribe_error_fails_every_waiter_for_the_generation() {
5597 let mut handler = make_handler_with_account();
5598 let (response_tx_1, response_rx_1) = tokio::sync::oneshot::channel();
5599 let (response_tx_2, response_rx_2) = tokio::sync::oneshot::channel();
5600 let channel = LighterWsChannel::MarketStats(LighterMarketSelection::Market(0));
5601 mark_subscription_inflight(&mut handler, channel.clone(), Some(response_tx_1));
5602 handler.queue_subscribe(channel, None, Some(response_tx_2));
5603
5604 let (matched, msg) = handle_control_text(
5605 &mut handler,
5606 &serde_json::json!({
5607 "type": "error", "code": 30012, "channel": "market_stats:0"
5608 })
5609 .to_string(),
5610 );
5611
5612 assert!(matched);
5613 assert!(msg.is_none());
5614 let error_1 = response_rx_1.await.unwrap().unwrap_err();
5615 let error_2 = response_rx_2.await.unwrap().unwrap_err();
5616 assert!(error_1.contains("market_stats:0"));
5617 assert!(error_1.contains("30012"));
5618 assert_eq!(error_1, error_2);
5619 assert!(handler.inflight_subs.is_empty());
5620 assert!(handler.subscription_attempts.is_empty());
5621 assert!(handler.subscriptions.is_empty());
5622 }
5623
5624 #[rstest]
5625 #[case::rate_limit(30009)]
5626 #[case::unattributed_failure(30012)]
5627 #[tokio::test]
5628 async fn subscription_retry_exhaustion_fails_waiter_and_clears_intent(#[case] code: u64) {
5629 let mut handler = make_handler_with_account();
5630 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5631 let (topic, _) =
5632 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(response_tx));
5633 handler
5634 .subscription_attempts
5635 .get_mut(&topic)
5636 .expect("subscription attempt")
5637 .retries = SUBSCRIBE_RETRY_MAX;
5638
5639 let (matched, msg) =
5640 handle_control_text(&mut handler, &serde_json::json!({"code": code}).to_string());
5641
5642 assert!(matched);
5643 assert!(msg.is_none());
5644 assert_eq!(
5645 response_rx.await.unwrap(),
5646 Err(format!(
5647 "subscription trade:7 failed after 6 attempts: venue rejected the WebSocket \
5648 subscribe with code {code}"
5649 )),
5650 );
5651 assert!(handler.inflight_subs.is_empty());
5652 assert!(handler.subscription_attempts.is_empty());
5653 assert!(handler.subscriptions.is_empty());
5654 }
5655
5656 #[rstest]
5657 #[case::rate_limit(serde_json::json!({"code": 30009}))]
5658 #[case::unattributed_failure(serde_json::json!({"code": 30012}))]
5659 #[case::wrapped_failure(serde_json::json!({"error": {"code": 30012}}))]
5660 #[case::invalid_channel(serde_json::json!({"code": 30012, "channel": 7}))]
5661 #[case::empty_channel(serde_json::json!({"code": 30012, "channel": ""}))]
5662 #[tokio::test]
5663 async fn subscription_error_retries_each_inflight_topic_with_new_generation(
5664 #[case] value: serde_json::Value,
5665 ) {
5666 let mut handler = make_handler_with_account();
5667 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5668 handler.set_command_sender(cmd_tx);
5669 let (trade_tx, mut trade_rx) = tokio::sync::oneshot::channel();
5670 let (candle_tx, mut candle_rx) = tokio::sync::oneshot::channel();
5671 let (trade_topic, trade_generation) =
5672 mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(trade_tx));
5673 let (candle_topic, candle_generation) = mark_subscription_inflight(
5674 &mut handler,
5675 LighterWsChannel::Candle {
5676 market_index: 0,
5677 resolution: LighterCandleResolution::OneMinute,
5678 },
5679 Some(candle_tx),
5680 );
5681
5682 let (matched, msg) = handler.handle_control_value(&value);
5683
5684 assert!(matched);
5685 assert!(msg.is_none());
5686 assert!(handler.inflight_subs.is_empty());
5687 assert!(handler.pending_subs.is_empty());
5688 let trade_retry = &handler.subscription_attempts[&trade_topic];
5689 let candle_retry = &handler.subscription_attempts[&candle_topic];
5690 assert_eq!(trade_retry.retries, 1);
5691 assert_eq!(candle_retry.retries, 1);
5692 assert_ne!(trade_retry.generation, trade_generation);
5693 assert_ne!(candle_retry.generation, candle_generation);
5694 assert_eq!(handler.subscription_retries.len(), 2);
5695 assert!(matches!(
5696 trade_rx.try_recv(),
5697 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5698 ));
5699 assert!(matches!(
5700 candle_rx.try_recv(),
5701 Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5702 ));
5703
5704 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5705 assert_eq!(handler.subscription_attempts.len(), 2);
5706 assert_eq!(handler.subscriptions.len(), 0);
5707
5708 let retry_generations = [
5709 (
5710 trade_topic,
5711 handler.subscription_attempts[&trade_topic].generation,
5712 ),
5713 (
5714 candle_topic,
5715 handler.subscription_attempts[&candle_topic].generation,
5716 ),
5717 ];
5718
5719 for (topic, generation) in retry_generations {
5720 handler.queue_subscription_retry(topic, generation);
5721 let queued = handler.pending_subs.pop_front().unwrap();
5722 assert_eq!(queued, (topic, generation));
5723 handler.inflight_subs.insert(topic, generation);
5724 }
5725
5726 handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5727 handle_control_text(
5728 &mut handler,
5729 r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5730 );
5731
5732 assert_eq!(trade_rx.await.unwrap(), Ok(()));
5733 assert_eq!(candle_rx.await.unwrap(), Ok(()));
5734 assert!(handler.subscription_attempts.is_empty());
5735 assert_eq!(handler.subscriptions.len(), 2);
5736 }
5737}