Skip to main content

nautilus_network/websocket/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! WebSocket client implementation with automatic reconnection.
17//!
18//! This module contains the core WebSocket client implementation including:
19//! - Connection management with automatic reconnection.
20//! - Split read/write architecture with separate tasks.
21//! - Unbounded channels on latency-sensitive paths.
22//! - Event-driven state notification via `Notify` for immediate wakeup on transitions.
23//! - Heartbeat support.
24//! - Rate limiting integration.
25
26use std::{
27    collections::VecDeque,
28    fmt::Debug,
29    pin::pin,
30    sync::{
31        Arc, OnceLock,
32        atomic::{AtomicBool, AtomicU8, Ordering},
33    },
34    time::Duration,
35};
36
37use futures_util::{SinkExt, StreamExt};
38use http::HeaderName;
39use nautilus_core::CleanDrop;
40use nautilus_cryptography::providers::install_cryptographic_provider;
41#[cfg(any(feature = "turmoil", feature = "transport-sockudo"))]
42use rustls::ClientConfig;
43#[cfg(feature = "transport-sockudo")]
44use sockudo_ws::{
45    Config as SockudoConfig, Http1, Role, Stream as SockudoStream,
46    WebSocketStream as SockudoWebSocketStream,
47};
48#[cfg(feature = "transport-sockudo")]
49use tokio::io::{AsyncRead, AsyncWrite};
50#[cfg(any(feature = "turmoil", feature = "transport-sockudo"))]
51use tokio_rustls::TlsConnector;
52#[cfg(feature = "turmoil")]
53use tokio_tungstenite::MaybeTlsStream;
54#[cfg(feature = "turmoil")]
55use tokio_tungstenite::client_async;
56#[cfg(not(feature = "turmoil"))]
57use tokio_tungstenite::connect_async_with_config;
58use tokio_tungstenite::tungstenite::{client::IntoClientRequest, http::HeaderValue};
59use ustr::Ustr;
60
61#[cfg(not(feature = "turmoil"))]
62use super::proxy::{ProxiedStream, ProxyKind, WsTarget, tunnel_via_proxy};
63use super::{
64    auth::{AuthState, AuthTracker},
65    config::{TransportBackend, WebSocketConfig},
66    consts::{
67        CONNECTION_STATE_CHECK_INTERVAL_MS, GRACEFUL_SHUTDOWN_DELAY_MS,
68        GRACEFUL_SHUTDOWN_TIMEOUT_SECS,
69    },
70    types::{MessageHandler, MessageReader, MessageWriter, PingHandler, WriterCommand},
71};
72#[cfg(feature = "turmoil")]
73use crate::net::TcpConnector;
74#[cfg(feature = "transport-sockudo")]
75use crate::net::TcpStream;
76#[cfg(feature = "transport-sockudo")]
77use crate::transport::sockudo::{
78    PrefixedIo, SockudoTransport, client_handshake_with_headers, validate_extra_headers,
79};
80use crate::{
81    RECONNECTED,
82    backoff::ExponentialBackoff,
83    dst,
84    error::{SendError, is_connection_drop_io_error},
85    logging::{log_task_aborted, log_task_started, log_task_stopped},
86    mode::ConnectionMode,
87    ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota},
88    transport::{BoxedWsTransport, Message, TransportError, tungstenite::TungsteniteTransport},
89};
90
91/// `WebSocketClient` connects to a websocket server to read and send messages.
92///
93/// The client is opinionated about how messages are read and written. It
94/// assumes that data can only have one reader but multiple writers.
95///
96/// The client splits the connection into read and write halves. It moves
97/// the read half into a tokio task which keeps receiving messages from the
98/// server and calls a handler - a Python function that takes the data
99/// as its parameter. It stores the write half in the struct wrapped
100/// with an Arc Mutex. This way the client struct can be used to write
101/// data to the server from multiple scopes/tasks.
102///
103/// The client also maintains a heartbeat if given a duration in seconds.
104/// It's preferable to set the duration slightly lower - heartbeat more
105/// frequently - than the required amount.
106pub struct WebSocketClientInner {
107    config: WebSocketConfig,
108    /// The function to handle incoming messages (stored separately from config).
109    message_handler: Option<MessageHandler>,
110    /// The handler for incoming pings (stored separately from config).
111    ping_handler: Option<PingHandler>,
112    read_task: Option<tokio::task::JoinHandle<()>>,
113    write_task: tokio::task::JoinHandle<()>,
114    writer_tx: tokio::sync::mpsc::UnboundedSender<WriterCommand>,
115    heartbeat_task: Option<tokio::task::JoinHandle<()>>,
116    connection_mode: Arc<AtomicU8>,
117    state_notify: Arc<tokio::sync::Notify>,
118    reconnect_timeout: Duration,
119    backoff: ExponentialBackoff,
120    /// True if this is a stream-based client (created via `connect_stream`).
121    /// Stream-based clients disable auto-reconnect because the reader is
122    /// owned by the caller and cannot be replaced during reconnection.
123    is_stream_mode: bool,
124    /// Maximum number of reconnection attempts before giving up (None = unlimited).
125    reconnect_max_attempts: Option<u32>,
126    /// Current count of consecutive reconnection attempts.
127    reconnection_attempt_count: u32,
128    /// Shared auth tracker invalidated on connection drops.
129    auth_tracker: Arc<OnceLock<AuthTracker>>,
130    /// Controls whether buffered replay waits for the next authenticated session.
131    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
132}
133
134enum ReconnectBufferAction {
135    Drain,
136    Wait,
137    Discard,
138}
139
140impl WebSocketClientInner {
141    /// Create an inner websocket client with an existing writer.
142    ///
143    /// This is used for stream mode where the reader is owned by the caller.
144    ///
145    /// # Errors
146    ///
147    /// Returns an error if the exponential backoff configuration is invalid.
148    #[expect(
149        clippy::unused_async,
150        reason = "async signature for consistency with connect-based constructors"
151    )]
152    pub async fn new_with_writer(
153        config: WebSocketConfig,
154        writer: MessageWriter,
155    ) -> Result<Self, TransportError> {
156        install_cryptographic_provider();
157
158        if config.heartbeat == Some(0) {
159            return Err(TransportError::Io(std::io::Error::new(
160                std::io::ErrorKind::InvalidInput,
161                "Heartbeat interval cannot be zero",
162            )));
163        }
164
165        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
166        let state_notify = Arc::new(tokio::sync::Notify::new());
167
168        // Note: We don't spawn a read task here since the reader is handled externally
169        let read_task = None;
170
171        // Stream mode ignores reconnect settings, use harmless defaults
172        let backoff = ExponentialBackoff::new(
173            Duration::from_secs(2),
174            Duration::from_secs(30),
175            1.5,
176            100,
177            true,
178        )
179        .map_err(|e| {
180            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
181        })?;
182
183        let auth_tracker = Arc::new(OnceLock::new());
184        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
185
186        let (writer_tx, writer_rx) = tokio::sync::mpsc::unbounded_channel::<WriterCommand>();
187        let write_task = Self::spawn_write_task(
188            connection_mode.clone(),
189            state_notify.clone(),
190            writer,
191            writer_rx,
192            Arc::clone(&auth_tracker),
193            Arc::clone(&reconnect_buffer_waits_for_auth),
194        );
195
196        let heartbeat_task = if let Some(heartbeat_interval) = config.heartbeat {
197            Some(Self::spawn_heartbeat_task(
198                connection_mode.clone(),
199                heartbeat_interval,
200                config.heartbeat_msg.clone(),
201                writer_tx.clone(),
202            ))
203        } else {
204            None
205        };
206
207        let reconnect_max_attempts = None; // Stream mode does not reconnect
208        let reconnect_timeout = Duration::from_secs(10);
209
210        Ok(Self {
211            config,
212            message_handler: None, // Stream mode has no handler
213            ping_handler: None,
214            writer_tx,
215            connection_mode,
216            state_notify,
217            reconnect_timeout,
218            heartbeat_task,
219            read_task,
220            write_task,
221            backoff,
222            is_stream_mode: true,
223            reconnect_max_attempts,
224            reconnection_attempt_count: 0,
225            auth_tracker,
226            reconnect_buffer_waits_for_auth,
227        })
228    }
229
230    /// Create an inner websocket client.
231    ///
232    /// # Errors
233    ///
234    /// Returns an error if:
235    /// - The connection to the server fails.
236    /// - The exponential backoff configuration is invalid.
237    pub async fn connect_url(
238        config: WebSocketConfig,
239        message_handler: Option<MessageHandler>,
240        ping_handler: Option<PingHandler>,
241    ) -> Result<Self, TransportError> {
242        install_cryptographic_provider();
243
244        if config.heartbeat == Some(0) {
245            return Err(TransportError::Io(std::io::Error::new(
246                std::io::ErrorKind::InvalidInput,
247                "Heartbeat interval cannot be zero",
248            )));
249        }
250
251        if config.idle_timeout_ms == Some(0) {
252            return Err(TransportError::Io(std::io::Error::new(
253                std::io::ErrorKind::InvalidInput,
254                "Idle timeout cannot be zero",
255            )));
256        }
257
258        // Capture whether we're in stream mode before moving config
259        let is_stream_mode = message_handler.is_none();
260        let reconnect_max_attempts = config.reconnect_max_attempts;
261
262        if !is_stream_mode && config.reconnect_timeout_ms == Some(0) {
263            return Err(TransportError::Io(std::io::Error::new(
264                std::io::ErrorKind::InvalidInput,
265                "Reconnect timeout cannot be zero",
266            )));
267        }
268
269        // Stream mode documents reconnect_* fields as ignored (callers may pass Some(0))
270        let reconnect_timeout = if is_stream_mode {
271            Duration::from_secs(10)
272        } else {
273            Duration::from_millis(config.reconnect_timeout_ms.unwrap_or(10_000))
274        };
275
276        // Bound the dial: a server that accepts TCP but never upgrades must not hang the caller
277        let (writer, reader) = dst::time::timeout(
278            reconnect_timeout,
279            Box::pin(Self::connect_with_server(
280                &config.url,
281                config.headers.clone(),
282                config.backend,
283                config.proxy_url.as_deref(),
284            )),
285        )
286        .await
287        .map_err(|_| {
288            TransportError::Io(std::io::Error::new(
289                std::io::ErrorKind::TimedOut,
290                format!(
291                    "connection timed out after {}s",
292                    reconnect_timeout.as_secs_f64()
293                ),
294            ))
295        })??;
296
297        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
298        let state_notify = Arc::new(tokio::sync::Notify::new());
299
300        let read_task = if message_handler.is_some() {
301            Some(Self::spawn_message_handler_task(
302                connection_mode.clone(),
303                state_notify.clone(),
304                reader,
305                message_handler.as_ref(),
306                ping_handler.as_ref(),
307                config.idle_timeout_ms,
308            ))
309        } else {
310            None
311        };
312
313        let auth_tracker = Arc::new(OnceLock::new());
314        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
315
316        let (writer_tx, writer_rx) = tokio::sync::mpsc::unbounded_channel::<WriterCommand>();
317        let write_task = Self::spawn_write_task(
318            connection_mode.clone(),
319            state_notify.clone(),
320            writer,
321            writer_rx,
322            Arc::clone(&auth_tracker),
323            Arc::clone(&reconnect_buffer_waits_for_auth),
324        );
325
326        // Optionally spawn a heartbeat task to periodically ping server
327        let heartbeat_task = config.heartbeat.map(|heartbeat_secs| {
328            Self::spawn_heartbeat_task(
329                connection_mode.clone(),
330                heartbeat_secs,
331                config.heartbeat_msg.clone(),
332                writer_tx.clone(),
333            )
334        });
335
336        let backoff = ExponentialBackoff::new(
337            Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
338            Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
339            config.reconnect_backoff_factor.unwrap_or(1.5),
340            config.reconnect_jitter_ms.unwrap_or(100),
341            true, // immediate-first
342        )
343        .map_err(|e| {
344            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
345        })?;
346
347        Ok(Self {
348            config,
349            message_handler,
350            ping_handler,
351            read_task,
352            write_task,
353            writer_tx,
354            heartbeat_task,
355            connection_mode,
356            state_notify,
357            reconnect_timeout,
358            backoff,
359            // Set stream mode when no message handler (reader not managed by client)
360            is_stream_mode,
361            reconnect_max_attempts,
362            reconnection_attempt_count: 0,
363            auth_tracker,
364            reconnect_buffer_waits_for_auth,
365        })
366    }
367
368    /// Connect to the server and return the split halves of the active transport.
369    ///
370    /// Dispatches on `backend` to the matching backend helper. The
371    /// [`TransportBackend::Tungstenite`] backend is always available; the
372    /// [`TransportBackend::Sockudo`] backend requires the `transport-sockudo`
373    /// Cargo feature (enabled by default) and uses a custom HTTP/1.1 handshake
374    /// path for upgrade headers.
375    ///
376    /// When `proxy_url` is `Some`, the Tungstenite backend establishes an HTTP
377    /// `CONNECT` tunnel through the proxy before performing the WebSocket
378    /// handshake. The Sockudo backend does not yet support proxying; when it
379    /// is selected together with a proxy URL, this method logs a warning and
380    /// transparently falls back to Tungstenite so omitted-backend Python
381    /// configurations keep working.
382    ///
383    /// # Errors
384    ///
385    /// Returns a [`TransportError`] if the URL is invalid, headers fail to
386    /// parse, the TCP / TLS layer cannot be established, the proxy refuses
387    /// the tunnel, or the WebSocket handshake is rejected by the peer. When
388    /// the Sockudo backend is selected without the `transport-sockudo`
389    /// feature, returns [`TransportError::Other`].
390    #[inline]
391    pub async fn connect_with_server(
392        url: &str,
393        headers: Vec<(String, String)>,
394        backend: TransportBackend,
395        proxy_url: Option<&str>,
396    ) -> Result<(MessageWriter, MessageReader), TransportError> {
397        // Sockudo does not yet support proxy tunnels. When a proxy URL is supplied,
398        // route through Tungstenite so configurations that rely on the runtime
399        // default keep working (notably the Python `WebSocketConfig` binding,
400        // which exposes `proxy_url` but no `backend` selector).
401        if matches!(backend, TransportBackend::Sockudo)
402            && let Some(proxy) = proxy_url
403        {
404            log::warn!("Sockudo backend does not support proxy_url; falling back to Tungstenite");
405            return Box::pin(Self::connect_tungstenite_via_proxy(url, headers, proxy)).await;
406        }
407
408        match backend {
409            TransportBackend::Tungstenite => match proxy_url {
410                Some(proxy) => {
411                    Box::pin(Self::connect_tungstenite_via_proxy(url, headers, proxy)).await
412                }
413                None => Self::connect_tungstenite(url, headers).await,
414            },
415            TransportBackend::Sockudo => {
416                #[cfg(feature = "transport-sockudo")]
417                {
418                    Self::connect_sockudo(url, headers).await
419                }
420                #[cfg(not(feature = "transport-sockudo"))]
421                {
422                    Err(TransportError::Other(
423                        "sockudo backend selected but the transport-sockudo \
424                         Cargo feature is not enabled"
425                            .to_string(),
426                    ))
427                }
428            }
429        }
430    }
431
432    /// Connects with the server creating a tokio-tungstenite websocket stream.
433    /// Production version that uses `connect_async_with_config` convenience helper.
434    #[inline]
435    #[cfg(not(feature = "turmoil"))]
436    async fn connect_tungstenite(
437        url: &str,
438        headers: Vec<(String, String)>,
439    ) -> Result<(MessageWriter, MessageReader), TransportError> {
440        let mut request = url.into_client_request().map_err(TransportError::from)?;
441        let req_headers = request.headers_mut();
442
443        for (key, val) in headers {
444            let header_value = HeaderValue::from_str(&val)
445                .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
446            let header_name: HeaderName = key
447                .parse()
448                .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
449            req_headers.insert(header_name, header_value);
450        }
451
452        let (stream, _resp) = connect_async_with_config(request, None, true)
453            .await
454            .map_err(TransportError::from)?;
455        let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
456        Ok(transport.split())
457    }
458
459    /// Connects via an HTTP `CONNECT` proxy and performs the WebSocket
460    /// handshake over the resulting tunnel.
461    ///
462    /// Recognised but unsupported proxy schemes (currently SOCKS) log a
463    /// warning and fall back to a direct connection so existing REST proxy
464    /// configs remain usable. Only available in production builds; the
465    /// turmoil simulator does not model arbitrary outbound TCP via a proxy.
466    #[inline]
467    #[cfg(not(feature = "turmoil"))]
468    async fn connect_tungstenite_via_proxy(
469        url: &str,
470        headers: Vec<(String, String)>,
471        proxy_url: &str,
472    ) -> Result<(MessageWriter, MessageReader), TransportError> {
473        let proxy = match ProxyKind::parse(proxy_url)? {
474            ProxyKind::Http(target) => target,
475            ProxyKind::Unsupported { scheme } => {
476                log::warn!(
477                    "WebSocket proxy_url scheme '{scheme}' is not yet supported; \
478                     connecting without a WebSocket proxy"
479                );
480                return Self::connect_tungstenite(url, headers).await;
481            }
482        };
483
484        let mut request = url.into_client_request().map_err(TransportError::from)?;
485        let req_headers = request.headers_mut();
486
487        for (key, val) in headers {
488            let header_value = HeaderValue::from_str(&val)
489                .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
490            let header_name: HeaderName = key
491                .parse()
492                .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
493            req_headers.insert(header_name, header_value);
494        }
495
496        let target = WsTarget::parse(url)?;
497        let stream = tunnel_via_proxy(&target, &proxy).await?;
498
499        // Each ProxiedStream variant carries a distinct concrete stream type,
500        // so we monomorphize the handshake through `proxied_ws_handshake`
501        // rather than duplicating the body four times. The futures are boxed
502        // because `client_async` produces a large state machine.
503        let transport: BoxedWsTransport = match stream {
504            ProxiedStream::Plain(tcp) => Box::pin(proxied_ws_handshake(request, tcp)).await?,
505            ProxiedStream::PlainOverTlsProxy(s) => {
506                Box::pin(proxied_ws_handshake(request, *s)).await?
507            }
508            ProxiedStream::Tls(s) => Box::pin(proxied_ws_handshake(request, *s)).await?,
509            ProxiedStream::TlsOverTlsProxy(s) => {
510                Box::pin(proxied_ws_handshake(request, *s)).await?
511            }
512        };
513
514        Ok(transport.split())
515    }
516
517    /// Turmoil simulator variant: HTTP `CONNECT` tunneling is not supported
518    /// under the simulator so any proxy URL is rejected up front.
519    #[inline]
520    #[cfg(feature = "turmoil")]
521    #[expect(
522        clippy::unused_async,
523        reason = "signature mirrors the production variant; both are awaited in the dispatcher"
524    )]
525    async fn connect_tungstenite_via_proxy(
526        _url: &str,
527        _headers: Vec<(String, String)>,
528        _proxy_url: &str,
529    ) -> Result<(MessageWriter, MessageReader), TransportError> {
530        Err(TransportError::Other(
531            "proxy_url is not supported under the turmoil simulator".to_string(),
532        ))
533    }
534
535    /// Connects with the server creating a tokio-tungstenite websocket stream.
536    /// Turmoil version that uses the lower-level `client_async` API with injected stream.
537    #[inline]
538    #[cfg(feature = "turmoil")]
539    async fn connect_tungstenite(
540        url: &str,
541        headers: Vec<(String, String)>,
542    ) -> Result<(MessageWriter, MessageReader), TransportError> {
543        let mut request = url.into_client_request().map_err(TransportError::from)?;
544        let req_headers = request.headers_mut();
545
546        for (key, val) in headers {
547            let header_value = HeaderValue::from_str(&val)
548                .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
549            let header_name: HeaderName = key
550                .parse()
551                .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
552            req_headers.insert(header_name, header_value);
553        }
554
555        let uri = request.uri();
556        let scheme = uri.scheme_str().unwrap_or("ws");
557        let host = uri
558            .host()
559            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
560
561        // Determine port: use explicit port if specified, otherwise default based on scheme
562        let port = uri
563            .port_u16()
564            .unwrap_or_else(|| if scheme == "wss" { 443 } else { 80 });
565
566        let addr = format!("{host}:{port}");
567
568        // Use the connector to get a turmoil-compatible stream
569        let connector = crate::net::RealTcpConnector;
570        let tcp_stream = connector.connect(&addr).await?;
571        if let Err(e) = tcp_stream.set_nodelay(true) {
572            log::warn!("Failed to enable TCP_NODELAY for socket client: {e:?}");
573        }
574
575        // Wrap stream appropriately based on scheme
576        let maybe_tls_stream = if scheme == "wss" {
577            // Build TLS config with webpki roots
578            let mut root_store = rustls::RootCertStore::empty();
579            root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
580
581            let config = ClientConfig::builder()
582                .with_root_certificates(root_store)
583                .with_no_client_auth();
584
585            let tls_connector = TlsConnector::from(std::sync::Arc::new(config));
586            let domain = rustls::pki_types::ServerName::try_from(host.to_string())
587                .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
588
589            let tls_stream = tls_connector
590                .connect(domain, tcp_stream)
591                .await
592                .map_err(TransportError::Io)?;
593            MaybeTlsStream::Rustls(tls_stream)
594        } else {
595            MaybeTlsStream::Plain(tcp_stream)
596        };
597
598        // Use client_async with the stream (plain or TLS)
599        let (stream, _resp) = client_async(request, maybe_tls_stream)
600            .await
601            .map_err(TransportError::from)?;
602        let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
603        Ok(transport.split())
604    }
605
606    /// Connects with the server using the sockudo-ws backend.
607    ///
608    /// Uses a local HTTP/1.1 handshake helper so error logging and stream
609    /// construction stay in our hands regardless of header count.
610    ///
611    /// Under the turmoil simulator, only plaintext `ws://` is supported (the
612    /// simulator does not model TLS), so a `wss://` URL returns
613    /// [`TransportError::Tls`] up front.
614    #[inline]
615    #[cfg(feature = "transport-sockudo")]
616    async fn connect_sockudo(
617        url: &str,
618        headers: Vec<(String, String)>,
619    ) -> Result<(MessageWriter, MessageReader), TransportError> {
620        let target = SockudoTarget::parse(url)?;
621        validate_extra_headers(&headers).map_err(TransportError::from)?;
622
623        #[cfg(feature = "turmoil")]
624        if target.is_tls {
625            return Err(TransportError::Tls(
626                "wss:// is not supported under the turmoil simulator; use ws://".to_string(),
627            ));
628        }
629
630        let tcp_stream = TcpStream::connect((target.host.as_str(), target.port))
631            .await
632            .map_err(TransportError::Io)?;
633
634        if let Err(e) = tcp_stream.set_nodelay(true) {
635            log::warn!("Failed to enable TCP_NODELAY for sockudo client: {e:?}");
636        }
637
638        #[cfg(not(feature = "turmoil"))]
639        if target.is_tls {
640            let mut root_store = rustls::RootCertStore::empty();
641            root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
642            let config = ClientConfig::builder()
643                .with_root_certificates(root_store)
644                .with_no_client_auth();
645            let connector = TlsConnector::from(std::sync::Arc::new(config));
646            let domain = rustls::pki_types::ServerName::try_from(target.host.clone())
647                .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
648            let tls_stream = connector
649                .connect(domain, tcp_stream)
650                .await
651                .map_err(TransportError::Io)?;
652            return Self::finish_sockudo_handshake(tls_stream, &target, &headers).await;
653        }
654
655        Self::finish_sockudo_handshake(tcp_stream, &target, &headers).await
656    }
657
658    #[cfg(feature = "transport-sockudo")]
659    async fn finish_sockudo_handshake<S>(
660        mut stream: S,
661        target: &SockudoTarget,
662        headers: &[(String, String)],
663    ) -> Result<(MessageWriter, MessageReader), TransportError>
664    where
665        S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
666    {
667        // Use our helper for both paths: uniform error logging, and we own
668        // stream construction since sockudo's high-level client drops the
669        // handshake leftover.
670        let handshake = client_handshake_with_headers(
671            &mut stream,
672            &target.host_header,
673            &target.path,
674            None,
675            headers,
676        )
677        .await
678        .map_err(TransportError::from)?;
679
680        // Reading the HTTP 101 may also read the first WebSocket frame prefix;
681        // replay it only when present so the ordinary path stays unwrapped.
682        let stream = match handshake.leftover {
683            Some(prefix) => SockudoStream::<Http1>::new(PrefixedIo::new(stream, prefix)),
684            None => SockudoStream::<Http1>::new(stream),
685        };
686        let ws = SockudoWebSocketStream::from_raw(stream, Role::Client, SockudoConfig::default());
687        let transport: BoxedWsTransport = Box::pin(SockudoTransport::new(ws));
688        Ok(transport.split())
689    }
690}
691
692fn is_connection_drop_transport_error(err: &TransportError) -> bool {
693    err.is_closed() || matches!(err, TransportError::Io(e) if is_connection_drop_io_error(e))
694}
695
696// Debug when we asked to disconnect (Disconnect/Closed), else Warn for a peer close
697fn read_termination_log_level(connection_state: &AtomicU8) -> log::Level {
698    let mode = ConnectionMode::from_atomic(connection_state);
699    if mode.is_disconnect() || mode.is_closed() {
700        log::Level::Debug
701    } else {
702        log::Level::Warn
703    }
704}
705
706#[cfg(test)]
707mod connection_error_tests {
708    use std::io;
709
710    use rstest::rstest;
711
712    use super::*;
713    use crate::transport::CloseFrame;
714
715    #[rstest]
716    #[case(TransportError::ConnectionClosed, true)]
717    #[case(TransportError::ConnectionReset, true)]
718    #[case(TransportError::ClosedByPeer(Some(CloseFrame::new(1000, "bye"))), true)]
719    #[case(TransportError::ClosedByPeer(None), true)]
720    #[case(TransportError::Io(io::Error::from(io::ErrorKind::BrokenPipe)), true)]
721    #[case(
722        TransportError::Io(io::Error::from(io::ErrorKind::ConnectionReset)),
723        true
724    )]
725    #[case(TransportError::Io(io::Error::from(io::ErrorKind::TimedOut)), true)]
726    #[case(
727        TransportError::Io(io::Error::from(io::ErrorKind::UnexpectedEof)),
728        true
729    )]
730    #[case(
731        TransportError::Io(io::Error::from(io::ErrorKind::InvalidInput)),
732        false
733    )]
734    #[case(TransportError::InvalidUrl("http://example.com".into()), false)]
735    #[case(TransportError::Handshake("bad".into()), false)]
736    #[case(TransportError::Protocol("bad opcode".into()), false)]
737    #[case(TransportError::Tls("bad certificate".into()), false)]
738    #[case(TransportError::MessageTooLarge, false)]
739    #[case(TransportError::FrameTooLarge, false)]
740    #[case(TransportError::InvalidUtf8, false)]
741    #[case(TransportError::Other("backend protocol mismatch".into()), false)]
742    fn connection_drop_transport_error_classification(
743        #[case] err: TransportError,
744        #[case] expected: bool,
745    ) {
746        assert_eq!(is_connection_drop_transport_error(&err), expected);
747    }
748}
749
750/// Complete the WebSocket handshake over a stream that has already been
751/// tunneled through an HTTP `CONNECT` proxy. Generic over the concrete
752/// stream type so the four [`super::proxy::ProxiedStream`] variants share
753/// a single body.
754#[cfg(not(feature = "turmoil"))]
755async fn proxied_ws_handshake<S>(
756    request: tokio_tungstenite::tungstenite::handshake::client::Request,
757    stream: S,
758) -> Result<BoxedWsTransport, TransportError>
759where
760    S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
761{
762    let (ws, _resp) = tokio_tungstenite::client_async(request, stream)
763        .await
764        .map_err(TransportError::from)?;
765    Ok(Box::pin(TungsteniteTransport::new(ws)))
766}
767
768/// Parsed components of a `ws://` / `wss://` URL needed by the sockudo backend.
769///
770/// Sockudo's HTTP/1.1 client passes the `host` argument verbatim as the
771/// HTTP `Host:` header, so it must include the explicit port when one is
772/// present in the URL (RFC 7230 section 5.4). The DNS / SNI lookup uses the bare
773/// host without the port.
774#[cfg(feature = "transport-sockudo")]
775#[derive(Debug, PartialEq, Eq)]
776struct SockudoTarget {
777    host: String,
778    /// Value to send as the HTTP `Host:` header. Includes `:port` only when
779    /// the URL specifies a non-default port explicitly.
780    host_header: String,
781    port: u16,
782    path: String,
783    is_tls: bool,
784}
785
786#[cfg(feature = "transport-sockudo")]
787impl SockudoTarget {
788    fn parse(url: &str) -> Result<Self, TransportError> {
789        let parsed =
790            url::Url::parse(url).map_err(|e| TransportError::InvalidUrl(format!("{url}: {e}")))?;
791
792        let scheme = parsed.scheme();
793        let is_tls = match scheme {
794            "ws" => false,
795            "wss" => true,
796            other => {
797                return Err(TransportError::InvalidUrl(format!(
798                    "expected ws:// or wss:// scheme, was {other}"
799                )));
800            }
801        };
802
803        let raw_host = parsed
804            .host_str()
805            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
806
807        // url::Url stores IPv6 hosts in their bracketed form (e.g. `[::1]`).
808        // Brackets are correct for the HTTP `Host:` header but invalid for
809        // DNS/TCP and TLS SNI, so we keep two representations: a bracketed
810        // `host_header` for the upgrade, and a bare `host` for socket dialing.
811        let is_bracketed = raw_host.starts_with('[') && raw_host.ends_with(']');
812        let host = if is_bracketed {
813            raw_host[1..raw_host.len() - 1].to_string()
814        } else {
815            raw_host.to_string()
816        };
817
818        let explicit_port = parsed.port();
819        let port = explicit_port.unwrap_or(if is_tls { 443 } else { 80 });
820        let host_header = match explicit_port {
821            Some(p) => format!("{raw_host}:{p}"),
822            None => raw_host.to_string(),
823        };
824
825        let path = if parsed.path().is_empty() {
826            "/".to_string()
827        } else {
828            let mut p = parsed.path().to_string();
829            if let Some(query) = parsed.query() {
830                p.push('?');
831                p.push_str(query);
832            }
833            p
834        };
835
836        Ok(Self {
837            host,
838            host_header,
839            port,
840            path,
841            is_tls,
842        })
843    }
844}
845
846impl WebSocketClientInner {
847    /// Reconnect with server.
848    ///
849    /// Make a new connection with server. Use the new read and write halves
850    /// to update self writer and read and heartbeat tasks.
851    ///
852    /// For stream-based clients (created via `connect_stream`), reconnection is disabled
853    /// because the reader is owned by the caller and cannot be replaced. Stream users
854    /// should handle disconnections by creating a new connection.
855    ///
856    /// The reconnect timeout bounds only connection establishment. Once the
857    /// new writer is handed to the writer task the swap runs to completion,
858    /// so buffered messages can never drain into a connection that lost its
859    /// reader to a timeout; the post-connect steps are individually bounded
860    /// by the writer task's graceful-shutdown timeout.
861    ///
862    /// # Errors
863    ///
864    /// Returns an error if:
865    /// - The reconnection attempt times out.
866    /// - The connection to the server fails.
867    pub async fn reconnect(&mut self) -> Result<(), TransportError> {
868        log::debug!("Reconnecting");
869
870        if self.is_stream_mode {
871            log::warn!(
872                "Auto-reconnect disabled for stream-based WebSocket client; \
873                stream users must manually reconnect by creating a new connection"
874            );
875            // Transition to CLOSED state to stop reconnection attempts
876            self.connection_mode
877                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
878            fail_registered_auth(
879                self.auth_tracker.as_ref(),
880                "WebSocket stream mode cannot reconnect",
881            );
882            return Ok(());
883        }
884
885        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
886            log::debug!("Reconnect aborted due to disconnect state");
887            return Ok(());
888        }
889
890        // Bound only connection establishment; the swap below must run to completion
891        let (new_writer, reader) = dst::time::timeout(
892            self.reconnect_timeout,
893            Self::connect_with_server(
894                &self.config.url,
895                self.config.headers.clone(),
896                self.config.backend,
897                self.config.proxy_url.as_deref(),
898            ),
899        )
900        .await
901        .map_err(|_| {
902            TransportError::Io(std::io::Error::new(
903                std::io::ErrorKind::TimedOut,
904                format!(
905                    "reconnection timed out after {}s",
906                    self.reconnect_timeout.as_secs_f64()
907                ),
908            ))
909        })??;
910
911        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
912            log::debug!("Reconnect aborted mid-flight (after connect)");
913            return Ok(());
914        }
915
916        // Use a oneshot channel to synchronize the writer swap before transitioning
917        // back to ACTIVE. Buffered messages stay in the writer task and replay later.
918        let (tx, rx) = tokio::sync::oneshot::channel();
919        if let Err(e) = self.writer_tx.send(WriterCommand::Update(new_writer, tx)) {
920            log::error!("{e}");
921            return Err(TransportError::Io(std::io::Error::new(
922                std::io::ErrorKind::BrokenPipe,
923                format!("Failed to send update command: {e}"),
924            )));
925        }
926
927        // Wait for writer to confirm it accepted the new socket
928        match rx.await {
929            Ok(true) => log::debug!("Writer confirmed socket update"),
930            Ok(false) => {
931                log::warn!("Writer rejected socket update, aborting reconnect");
932                return Err(TransportError::Io(std::io::Error::other(
933                    "Failed to update reconnection writer",
934                )));
935            }
936            Err(e) => {
937                log::error!("Writer dropped update channel: {e}");
938                return Err(TransportError::Io(std::io::Error::new(
939                    std::io::ErrorKind::BrokenPipe,
940                    "Writer task dropped response channel",
941                )));
942            }
943        }
944
945        // Delay before closing connection
946        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
947
948        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
949            log::debug!("Reconnect aborted mid-flight (after delay)");
950            return Ok(());
951        }
952
953        if let Some(ref read_task) = self.read_task.take()
954            && !read_task.is_finished()
955        {
956            read_task.abort();
957            log_task_aborted("read");
958        }
959
960        // Atomically transition from Reconnect to Active
961        // This prevents race condition where disconnect could be requested between check and store
962        if self
963            .connection_mode
964            .compare_exchange(
965                ConnectionMode::Reconnect.as_u8(),
966                ConnectionMode::Active.as_u8(),
967                Ordering::SeqCst,
968                Ordering::SeqCst,
969            )
970            .is_err()
971        {
972            log::debug!("Reconnect aborted (state changed during reconnect)");
973            return Ok(());
974        }
975
976        self.read_task = if self.message_handler.is_some() {
977            Some(Self::spawn_message_handler_task(
978                self.connection_mode.clone(),
979                self.state_notify.clone(),
980                reader,
981                self.message_handler.as_ref(),
982                self.ping_handler.as_ref(),
983                self.config.idle_timeout_ms,
984            ))
985        } else {
986            None
987        };
988
989        log::debug!("Reconnect succeeded");
990        Ok(())
991    }
992
993    /// Check if the client is still alive.
994    ///
995    /// Returns `true` if both the read and write tasks are still running.
996    /// There may be some delay between the connection closing and the
997    /// client detecting it.
998    #[inline]
999    #[must_use]
1000    pub fn is_alive(&self) -> bool {
1001        match &self.read_task {
1002            Some(read_task) => !read_task.is_finished() && !self.write_task.is_finished(),
1003            None => !self.write_task.is_finished(),
1004        }
1005    }
1006
1007    fn spawn_message_handler_task(
1008        connection_state: Arc<AtomicU8>,
1009        state_notify: Arc<tokio::sync::Notify>,
1010        mut reader: MessageReader,
1011        message_handler: Option<&MessageHandler>,
1012        ping_handler: Option<&PingHandler>,
1013        idle_timeout_ms: Option<u64>,
1014    ) -> tokio::task::JoinHandle<()> {
1015        log::debug!("Started message handler task 'read'");
1016
1017        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1018        let idle_timeout = idle_timeout_ms.map(Duration::from_millis);
1019
1020        // Clone Arc handlers for the async task
1021        let message_handler = message_handler.cloned();
1022        let ping_handler = ping_handler.cloned();
1023
1024        tokio::task::spawn(async move {
1025            let mut last_data_time = dst::time::Instant::now();
1026
1027            loop {
1028                if !ConnectionMode::from_atomic(&connection_state).is_active() {
1029                    break;
1030                }
1031
1032                match dst::time::timeout(check_interval, reader.next()).await {
1033                    Ok(Some(Ok(Message::Binary(data)))) => {
1034                        log::trace!("Received message <binary> {} bytes", data.len());
1035                        last_data_time = dst::time::Instant::now();
1036
1037                        if let Some(ref handler) = message_handler {
1038                            handler(Message::Binary(data));
1039                        }
1040                    }
1041                    Ok(Some(Ok(Message::Text(data)))) => {
1042                        log::trace!("Received message: {data:?}");
1043                        last_data_time = dst::time::Instant::now();
1044
1045                        if let Some(ref handler) = message_handler {
1046                            handler(Message::Text(data));
1047                        }
1048                    }
1049                    Ok(Some(Ok(Message::Ping(ping_data)))) => {
1050                        log::trace!("Received ping: {ping_data:?}");
1051                        // Do not reset last_data_time: pings are keep-alive frames, not application
1052                        // data, so a peer that emits only pings must still trip the idle timeout.
1053                        // Checked here too: a ping flood faster than the check interval starves the timeout branch
1054
1055                        if let Some(ref handler) = ping_handler {
1056                            handler(ping_data.to_vec());
1057                        }
1058
1059                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1060                            break;
1061                        }
1062                    }
1063                    Ok(Some(Ok(Message::Pong(_)))) => {
1064                        log::trace!("Received pong");
1065                        // Do not reset last_data_time: pongs are keep-alive replies (not data)
1066
1067                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1068                            break;
1069                        }
1070                    }
1071                    Ok(Some(Ok(Message::Close(Some(frame))))) => {
1072                        log::log!(
1073                            read_termination_log_level(&connection_state),
1074                            "Received close frame, terminating: code={}, reason='{}'",
1075                            frame.code,
1076                            frame.reason
1077                        );
1078                        break;
1079                    }
1080                    Ok(Some(Ok(Message::Close(None)))) => {
1081                        log::log!(
1082                            read_termination_log_level(&connection_state),
1083                            "Received close frame with no code or reason, terminating"
1084                        );
1085                        break;
1086                    }
1087                    Ok(Some(Err(e))) => {
1088                        if is_connection_drop_transport_error(&e) {
1089                            log::warn!("Received connection error, terminating: {e}");
1090                        } else {
1091                            log::error!("Received transport error, terminating: {e}");
1092                        }
1093                        break;
1094                    }
1095                    Ok(None) => {
1096                        log::log!(
1097                            read_termination_log_level(&connection_state),
1098                            "Connection closed by peer (no close frame), terminating"
1099                        );
1100                        break;
1101                    }
1102                    Err(_) => {
1103                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1104                            break;
1105                        }
1106                    }
1107                }
1108            }
1109
1110            // Wake the controller immediately so it detects the dead read task
1111            state_notify.notify_one();
1112        })
1113    }
1114
1115    /// Attempts to send all buffered messages after reconnection.
1116    ///
1117    /// Returns `true` if a send error occurred (caller should trigger reconnection).
1118    /// Messages remain in buffer if send fails, preserving them for the next reconnection attempt.
1119    async fn drain_reconnect_buffer(
1120        buffer: &mut VecDeque<Message>,
1121        writer: &mut MessageWriter,
1122    ) -> bool {
1123        if buffer.is_empty() {
1124            return false;
1125        }
1126
1127        let initial_buffer_len = buffer.len();
1128        log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
1129
1130        let mut send_error_occurred = false;
1131
1132        while let Some(buffered_msg) = buffer.front() {
1133            // Clone message before attempting send (to keep in buffer if send fails)
1134            let msg_to_send = buffered_msg.clone();
1135
1136            if let Err(e) = writer.send(msg_to_send).await {
1137                if is_connection_drop_transport_error(&e) {
1138                    log::warn!(
1139                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1140                        buffer.len()
1141                    );
1142                } else {
1143                    log::error!(
1144                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1145                        buffer.len()
1146                    );
1147                }
1148                send_error_occurred = true;
1149                break; // Stop processing buffer, remaining messages preserved for next reconnection
1150            }
1151
1152            // Only remove from buffer after successful send
1153            buffer.pop_front();
1154        }
1155
1156        if buffer.is_empty() {
1157            log::info!("Successfully sent all {initial_buffer_len} buffered messages");
1158        }
1159
1160        send_error_occurred
1161    }
1162
1163    fn can_drain_reconnect_buffer(
1164        reconnect_buffer_waits_for_auth: &AtomicBool,
1165        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1166    ) -> ReconnectBufferAction {
1167        if !reconnect_buffer_waits_for_auth.load(Ordering::Acquire) {
1168            return ReconnectBufferAction::Drain;
1169        }
1170
1171        match auth_tracker.get().map(AuthTracker::auth_state) {
1172            Some(AuthState::Authenticated) => ReconnectBufferAction::Drain,
1173            Some(AuthState::Failed) => ReconnectBufferAction::Discard,
1174            Some(AuthState::Unauthenticated) | None => ReconnectBufferAction::Wait,
1175        }
1176    }
1177
1178    fn spawn_write_task(
1179        connection_state: Arc<AtomicU8>,
1180        state_notify: Arc<tokio::sync::Notify>,
1181        writer: MessageWriter,
1182        mut writer_rx: tokio::sync::mpsc::UnboundedReceiver<WriterCommand>,
1183        auth_tracker: Arc<OnceLock<AuthTracker>>,
1184        reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1185    ) -> tokio::task::JoinHandle<()> {
1186        log_task_started("write");
1187
1188        // Interval between checking the connection mode
1189        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1190
1191        tokio::task::spawn(async move {
1192            let mut active_writer = writer;
1193            // Buffer for messages received during reconnection
1194            // VecDeque for efficient pop_front() operations
1195            let mut reconnect_buffer: VecDeque<Message> = VecDeque::new();
1196
1197            loop {
1198                let mode = ConnectionMode::from_atomic(&connection_state);
1199
1200                match mode {
1201                    ConnectionMode::Disconnect => {
1202                        // Log any buffered messages that will be lost
1203                        if !reconnect_buffer.is_empty() {
1204                            log::warn!(
1205                                "Discarding {} buffered messages due to disconnect",
1206                                reconnect_buffer.len()
1207                            );
1208                            reconnect_buffer.clear();
1209                        }
1210
1211                        // Attempt to close the writer gracefully before exiting,
1212                        // we ignore any error as the writer may already be closed.
1213                        _ = dst::time::timeout(
1214                            Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
1215                            active_writer.close(),
1216                        )
1217                        .await;
1218                        break;
1219                    }
1220                    ConnectionMode::Closed => {
1221                        // Log any buffered messages that will be lost
1222                        if !reconnect_buffer.is_empty() {
1223                            log::warn!(
1224                                "Discarding {} buffered messages due to closed connection",
1225                                reconnect_buffer.len()
1226                            );
1227                            reconnect_buffer.clear();
1228                        }
1229                        break;
1230                    }
1231                    _ => {}
1232                }
1233
1234                if mode.is_active() && !reconnect_buffer.is_empty() {
1235                    match Self::can_drain_reconnect_buffer(
1236                        reconnect_buffer_waits_for_auth.as_ref(),
1237                        &auth_tracker,
1238                    ) {
1239                        ReconnectBufferAction::Drain => {
1240                            let send_error = Self::drain_reconnect_buffer(
1241                                &mut reconnect_buffer,
1242                                &mut active_writer,
1243                            )
1244                            .await;
1245
1246                            // CAS: a disconnect landing mid-drain must not be overwritten
1247                            if send_error && ConnectionMode::request_reconnect(&connection_state) {
1248                                if let Some(tracker) = auth_tracker.get() {
1249                                    tracker.invalidate();
1250                                }
1251                                state_notify.notify_one();
1252                            }
1253
1254                            continue;
1255                        }
1256                        ReconnectBufferAction::Discard => {
1257                            log::warn!(
1258                                "Discarding {} buffered messages after authentication failed",
1259                                reconnect_buffer.len()
1260                            );
1261                            reconnect_buffer.clear();
1262                            continue;
1263                        }
1264                        ReconnectBufferAction::Wait => {}
1265                    }
1266                }
1267
1268                match dst::time::timeout(check_interval, writer_rx.recv()).await {
1269                    Ok(Some(msg)) => {
1270                        // Re-check connection mode after receiving a message
1271                        let mode = ConnectionMode::from_atomic(&connection_state);
1272                        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1273                            break;
1274                        }
1275
1276                        match msg {
1277                            WriterCommand::Update(new_writer, tx) => {
1278                                log::debug!("Received new writer");
1279
1280                                // Delay before closing connection
1281                                dst::time::sleep(Duration::from_millis(100)).await;
1282
1283                                // Attempt to close the writer gracefully on update,
1284                                // we ignore any error as the writer may already be closed.
1285                                _ = dst::time::timeout(
1286                                    Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
1287                                    active_writer.close(),
1288                                )
1289                                .await;
1290
1291                                active_writer = new_writer;
1292                                log::debug!("Updated writer");
1293
1294                                if let Err(e) = tx.send(true) {
1295                                    log::error!(
1296                                        "Failed to report writer update to controller: {e:?}"
1297                                    );
1298                                }
1299                            }
1300                            WriterCommand::Send(msg) if mode.is_reconnect() => {
1301                                // Buffer messages during reconnection instead of dropping them
1302                                log::debug!(
1303                                    "Buffering message during reconnection (buffer size: {})",
1304                                    reconnect_buffer.len() + 1
1305                                );
1306                                reconnect_buffer.push_back(msg);
1307                            }
1308                            WriterCommand::Send(msg) => {
1309                                if let Err(e) = active_writer.send(msg.clone()).await {
1310                                    if is_connection_drop_transport_error(&e) {
1311                                        log::warn!("Failed to send message: {e}");
1312                                    } else {
1313                                        log::error!("Failed to send message: {e}");
1314                                    }
1315                                    reconnect_buffer.push_back(msg);
1316
1317                                    // CAS: a disconnect landing mid-send must not be overwritten
1318                                    if ConnectionMode::request_reconnect(&connection_state) {
1319                                        log::warn!("Writer triggering reconnect");
1320
1321                                        if let Some(tracker) = auth_tracker.get() {
1322                                            tracker.invalidate();
1323                                        }
1324                                        state_notify.notify_one();
1325                                    }
1326                                }
1327                            }
1328                        }
1329                    }
1330                    Ok(None) => {
1331                        // Channel closed - writer task should terminate
1332                        log::debug!("Writer channel closed, terminating writer task");
1333                        break;
1334                    }
1335                    Err(_) => {
1336                        // Timeout - just continue the loop
1337                    }
1338                }
1339            }
1340
1341            // Attempt to close the writer gracefully before exiting,
1342            // we ignore any error as the writer may already be closed.
1343            _ = dst::time::timeout(
1344                Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
1345                active_writer.close(),
1346            )
1347            .await;
1348
1349            log_task_stopped("write");
1350        })
1351    }
1352
1353    fn spawn_heartbeat_task(
1354        connection_state: Arc<AtomicU8>,
1355        heartbeat_secs: u64,
1356        message: Option<String>,
1357        writer_tx: tokio::sync::mpsc::UnboundedSender<WriterCommand>,
1358    ) -> tokio::task::JoinHandle<()> {
1359        log_task_started("heartbeat");
1360
1361        tokio::task::spawn(async move {
1362            let interval = Duration::from_secs(heartbeat_secs);
1363
1364            loop {
1365                dst::time::sleep(interval).await;
1366
1367                match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
1368                    ConnectionMode::Active => {
1369                        let msg = match &message {
1370                            Some(text) => WriterCommand::Send(Message::Text(text.clone().into())),
1371                            None => WriterCommand::Send(Message::Ping(vec![].into())),
1372                        };
1373
1374                        match writer_tx.send(msg) {
1375                            Ok(()) => log::trace!("Sent heartbeat to writer task"),
1376                            Err(e) => {
1377                                log::error!("Failed to send heartbeat to writer task: {e}");
1378                            }
1379                        }
1380                    }
1381                    ConnectionMode::Reconnect => {}
1382                    ConnectionMode::Disconnect | ConnectionMode::Closed => break,
1383                }
1384            }
1385
1386            log_task_stopped("heartbeat");
1387        })
1388    }
1389}
1390
1391fn idle_timeout_exceeded(
1392    last_data_time: dst::time::Instant,
1393    idle_timeout: Option<Duration>,
1394) -> bool {
1395    if let Some(timeout) = idle_timeout {
1396        let idle_duration = last_data_time.elapsed();
1397        if idle_duration >= timeout {
1398            log::warn!(
1399                "Read idle timeout: no data received for {:.1}s",
1400                idle_duration.as_secs_f64()
1401            );
1402            return true;
1403        }
1404    }
1405    false
1406}
1407
1408impl Drop for WebSocketClientInner {
1409    fn drop(&mut self) {
1410        // Delegate to explicit cleanup handler
1411        self.clean_drop();
1412    }
1413}
1414
1415/// Cleanup on drop: aborts background tasks and clears handlers to break reference cycles.
1416impl CleanDrop for WebSocketClientInner {
1417    fn clean_drop(&mut self) {
1418        if let Some(ref read_task) = self.read_task.take()
1419            && !read_task.is_finished()
1420        {
1421            read_task.abort();
1422            log_task_aborted("read");
1423        }
1424
1425        if !self.write_task.is_finished() {
1426            self.write_task.abort();
1427            log_task_aborted("write");
1428        }
1429
1430        if let Some(ref handle) = self.heartbeat_task.take()
1431            && !handle.is_finished()
1432        {
1433            handle.abort();
1434            log_task_aborted("heartbeat");
1435        }
1436
1437        // Clear handlers to break potential reference cycles
1438        self.message_handler = None;
1439        self.ping_handler = None;
1440    }
1441}
1442
1443#[expect(
1444    clippy::missing_fields_in_debug,
1445    reason = "handler closures and internal task handles are intentionally omitted"
1446)]
1447impl Debug for WebSocketClientInner {
1448    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1449        f.debug_struct(stringify!(WebSocketClientInner))
1450            .field("config", &self.config)
1451            .field(
1452                "connection_mode",
1453                &ConnectionMode::from_atomic(&self.connection_mode),
1454            )
1455            .field("reconnect_timeout", &self.reconnect_timeout)
1456            .field("is_stream_mode", &self.is_stream_mode)
1457            .finish()
1458    }
1459}
1460
1461/// WebSocket client with automatic reconnection.
1462///
1463/// Handles connection state, callbacks, and rate limiting.
1464/// See module docs for architecture details.
1465#[cfg_attr(
1466    feature = "python",
1467    pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.network")
1468)]
1469#[cfg_attr(
1470    feature = "python",
1471    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.network")
1472)]
1473pub struct WebSocketClient {
1474    pub(crate) controller_task: tokio::task::JoinHandle<()>,
1475    pub(crate) connection_mode: Arc<AtomicU8>,
1476    pub(crate) state_notify: Arc<tokio::sync::Notify>,
1477    pub(crate) reconnect_timeout: Duration,
1478    pub(crate) rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
1479    pub(crate) writer_tx: tokio::sync::mpsc::UnboundedSender<WriterCommand>,
1480    auth_tracker: Arc<OnceLock<AuthTracker>>,
1481    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1482}
1483
1484impl Debug for WebSocketClient {
1485    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1486        f.debug_struct(stringify!(WebSocketClient)).finish()
1487    }
1488}
1489
1490impl WebSocketClient {
1491    /// Creates a websocket client in **stream mode** that returns a [`MessageReader`].
1492    ///
1493    /// Returns a stream that the caller owns and reads from directly. Automatic reconnection
1494    /// is **disabled** because the reader cannot be replaced internally. On disconnection, the
1495    /// client transitions to CLOSED state and the caller must manually reconnect by calling
1496    /// `connect_stream` again.
1497    ///
1498    /// Use stream mode when you need custom reconnection logic, direct control over message
1499    /// reading, or fine-grained backpressure handling.
1500    ///
1501    /// See [`WebSocketConfig`] documentation for comparison with handler mode.
1502    ///
1503    /// # Errors
1504    ///
1505    /// Returns an error if the connection cannot be established.
1506    pub async fn connect_stream(
1507        config: WebSocketConfig,
1508        keyed_quotas: Vec<(String, Quota)>,
1509        default_quota: Option<Quota>,
1510        post_reconnect: Option<Arc<dyn Fn() + Send + Sync>>,
1511    ) -> Result<(MessageReader, Self), TransportError> {
1512        install_cryptographic_provider();
1513
1514        // Create a single connection and split it, respecting configured headers.
1515        // The dial bound is a fixed default: stream mode documents reconnect_* fields as ignored
1516        let connect_timeout = Duration::from_secs(10);
1517        let (writer, reader) = dst::time::timeout(
1518            connect_timeout,
1519            WebSocketClientInner::connect_with_server(
1520                &config.url,
1521                config.headers.clone(),
1522                config.backend,
1523                config.proxy_url.as_deref(),
1524            ),
1525        )
1526        .await
1527        .map_err(|_| {
1528            TransportError::Io(std::io::Error::new(
1529                std::io::ErrorKind::TimedOut,
1530                format!(
1531                    "connection timed out after {}s",
1532                    connect_timeout.as_secs_f64()
1533                ),
1534            ))
1535        })??;
1536
1537        // Create inner without connecting (we'll provide the writer)
1538        let inner = WebSocketClientInner::new_with_writer(config, writer).await?;
1539
1540        let connection_mode = inner.connection_mode.clone();
1541        let state_notify = inner.state_notify.clone();
1542        let reconnect_timeout = inner.reconnect_timeout;
1543        let auth_tracker = Arc::clone(&inner.auth_tracker);
1544        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
1545        let keyed_quotas = keyed_quotas
1546            .into_iter()
1547            .map(|(key, quota)| (Ustr::from(&key), quota))
1548            .collect();
1549        let rate_limiter = Arc::new(RateLimiter::new_with_quota(default_quota, keyed_quotas));
1550        let writer_tx = inner.writer_tx.clone();
1551
1552        let controller_task = Self::spawn_controller_task(
1553            inner,
1554            connection_mode.clone(),
1555            state_notify.clone(),
1556            post_reconnect,
1557            Arc::clone(&auth_tracker),
1558        );
1559
1560        Ok((
1561            reader,
1562            Self {
1563                controller_task,
1564                connection_mode,
1565                state_notify,
1566                reconnect_timeout,
1567                rate_limiter,
1568                writer_tx,
1569                auth_tracker,
1570                reconnect_buffer_waits_for_auth,
1571            },
1572        ))
1573    }
1574
1575    /// Creates a websocket client in **handler mode** with automatic reconnection.
1576    ///
1577    /// The handler is called for each incoming message on an internal task.
1578    /// Automatic reconnection is **enabled** with exponential backoff. On disconnection,
1579    /// the client automatically attempts to reconnect and replaces the internal reader
1580    /// (the handler continues working seamlessly).
1581    ///
1582    /// Use handler mode for simplified connection management, automatic reconnection, Python
1583    /// bindings, or callback-based message handling.
1584    ///
1585    /// See [`WebSocketConfig`] documentation for comparison with stream mode.
1586    ///
1587    /// # Errors
1588    ///
1589    /// Returns an error if:
1590    /// - The connection cannot be established.
1591    /// - `message_handler` is `None` (use `connect_stream` instead).
1592    pub async fn connect(
1593        config: WebSocketConfig,
1594        message_handler: Option<MessageHandler>,
1595        ping_handler: Option<PingHandler>,
1596        post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1597        keyed_quotas: Vec<(String, Quota)>,
1598        default_quota: Option<Quota>,
1599    ) -> Result<Self, TransportError> {
1600        let keyed_quotas = keyed_quotas
1601            .into_iter()
1602            .map(|(key, quota)| (Ustr::from(&key), quota))
1603            .collect();
1604        let rate_limiter = Arc::new(RateLimiter::new_with_quota(default_quota, keyed_quotas));
1605        Self::connect_with_rate_limiter(
1606            config,
1607            message_handler,
1608            ping_handler,
1609            post_reconnection,
1610            rate_limiter,
1611        )
1612        .await
1613    }
1614
1615    /// Creates a websocket client in **handler mode** sharing an externally-owned rate limiter.
1616    ///
1617    /// Use this constructor to share a single [`RateLimiter`] across multiple
1618    /// [`WebSocketClient`] instances (for example, the WebSocket clients owned
1619    /// by an exchange adapter's data and execution clients). All quota state
1620    /// lives inside the limiter, so passing the same `Arc` produces a single
1621    /// shared bucket — the only way to honour a venue's per-IP / per-account
1622    /// WS message cap when more than one connection is opened in-process.
1623    ///
1624    /// Behavior otherwise matches [`Self::connect`].
1625    ///
1626    /// # Errors
1627    ///
1628    /// Returns an error if:
1629    /// - The connection cannot be established.
1630    /// - `message_handler` is `None` (use `connect_stream` instead).
1631    pub async fn connect_with_rate_limiter(
1632        config: WebSocketConfig,
1633        message_handler: Option<MessageHandler>,
1634        ping_handler: Option<PingHandler>,
1635        post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1636        rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
1637    ) -> Result<Self, TransportError> {
1638        if message_handler.is_none() {
1639            return Err(TransportError::Io(std::io::Error::new(
1640                std::io::ErrorKind::InvalidInput,
1641                "Handler mode requires message_handler to be set. Use connect_stream() for stream mode without a handler.",
1642            )));
1643        }
1644
1645        log::debug!("Connecting");
1646        let inner =
1647            WebSocketClientInner::connect_url(config, message_handler, ping_handler).await?;
1648        let connection_mode = inner.connection_mode.clone();
1649        let state_notify = inner.state_notify.clone();
1650        let writer_tx = inner.writer_tx.clone();
1651        let reconnect_timeout = inner.reconnect_timeout;
1652        let auth_tracker = Arc::clone(&inner.auth_tracker);
1653        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
1654
1655        let controller_task = Self::spawn_controller_task(
1656            inner,
1657            connection_mode.clone(),
1658            state_notify.clone(),
1659            post_reconnection,
1660            Arc::clone(&auth_tracker),
1661        );
1662
1663        Ok(Self {
1664            controller_task,
1665            connection_mode,
1666            state_notify,
1667            reconnect_timeout,
1668            rate_limiter,
1669            writer_tx,
1670            auth_tracker,
1671            reconnect_buffer_waits_for_auth,
1672        })
1673    }
1674
1675    /// Returns the current connection mode.
1676    #[must_use]
1677    pub fn connection_mode(&self) -> ConnectionMode {
1678        ConnectionMode::from_atomic(&self.connection_mode)
1679    }
1680
1681    /// Returns a clone of the connection mode atomic for external state tracking.
1682    ///
1683    /// This allows adapter clients to track connection state across reconnections
1684    /// without message-passing delays.
1685    #[must_use]
1686    pub fn connection_mode_atomic(&self) -> Arc<AtomicU8> {
1687        Arc::clone(&self.connection_mode)
1688    }
1689
1690    /// Check if the client connection is active.
1691    ///
1692    /// Returns `true` if the client is connected and has not been signalled to disconnect.
1693    /// The client will automatically retry connection based on its configuration.
1694    #[inline]
1695    #[must_use]
1696    pub fn is_active(&self) -> bool {
1697        self.connection_mode().is_active()
1698    }
1699
1700    /// Check if the client is disconnected.
1701    #[must_use]
1702    pub fn is_disconnected(&self) -> bool {
1703        self.controller_task.is_finished()
1704    }
1705
1706    /// Check if the client is reconnecting.
1707    ///
1708    /// Returns `true` if the client lost connection and is attempting to reestablish it.
1709    /// The client will automatically retry connection based on its configuration.
1710    #[inline]
1711    #[must_use]
1712    pub fn is_reconnecting(&self) -> bool {
1713        self.connection_mode().is_reconnect()
1714    }
1715
1716    /// Registers an [`AuthTracker`] with the client.
1717    ///
1718    /// When the controller detects a dead connection and transitions to
1719    /// `Reconnect`, it calls `invalidate()` on the tracker so that any
1720    /// pending authenticated sends see the state change immediately. Terminal
1721    /// transitions fail the tracker so pending auth waits can terminate.
1722    /// Set `reconnect_buffer_waits_for_auth` for clients that must not replay
1723    /// buffered messages until the next session authenticates.
1724    ///
1725    /// Call this once after construction, before any authenticated sends.
1726    pub fn set_auth_tracker(&self, tracker: AuthTracker, reconnect_buffer_waits_for_auth: bool) {
1727        let _ = self.auth_tracker.set(tracker);
1728        self.reconnect_buffer_waits_for_auth
1729            .store(reconnect_buffer_waits_for_auth, Ordering::Release);
1730    }
1731
1732    /// Check if the client is disconnecting.
1733    ///
1734    /// Returns `true` if the client is in disconnect mode.
1735    #[inline]
1736    #[must_use]
1737    pub fn is_disconnecting(&self) -> bool {
1738        self.connection_mode().is_disconnect()
1739    }
1740
1741    /// Check if the client is closed.
1742    ///
1743    /// Returns `true` if the client has been explicitly disconnected or reached
1744    /// maximum reconnection attempts. In this state, the client cannot be reused
1745    /// and a new client must be created for further connections.
1746    #[inline]
1747    #[must_use]
1748    pub fn is_closed(&self) -> bool {
1749        self.connection_mode().is_closed()
1750    }
1751
1752    /// Checks whether the connection is in a terminal state (disconnecting or closed).
1753    ///
1754    /// Single atomic load to fail fast before rate limiting or waiting.
1755    #[inline]
1756    fn check_not_terminal(&self) -> Result<(), SendError> {
1757        match self.connection_mode() {
1758            ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
1759            _ => Ok(()),
1760        }
1761    }
1762
1763    /// Waits for rate limiter quota, aborting early if connection enters a terminal state.
1764    async fn await_rate_limit_or_closed(&self, keys: Option<&[Ustr]>) -> Result<(), SendError> {
1765        const CHECK_INTERVAL_MS: u64 = 100;
1766
1767        tokio::select! {
1768            biased;
1769            () = self.rate_limiter.await_keys_ready(keys) => Ok(()),
1770            () = async {
1771                loop {
1772                    // Enable before the state check: an unpolled Notified is unregistered and misses notifies
1773                    let mut notified = pin!(self.state_notify.notified());
1774                    notified.as_mut().enable();
1775
1776                    if matches!(self.connection_mode(), ConnectionMode::Disconnect | ConnectionMode::Closed) {
1777                        break;
1778                    }
1779                    tokio::select! {
1780                        biased;
1781                        () = notified => {}
1782                        () = dst::time::sleep(Duration::from_millis(CHECK_INTERVAL_MS)) => {}
1783                    }
1784                }
1785            } => Err(SendError::Closed),
1786        }
1787    }
1788
1789    /// Waits for the client to become active before sending.
1790    ///
1791    /// Uses `state_notify` for event-driven wakeup so sends resume immediately
1792    /// after reconnection completes. A fallback interval guards against missed
1793    /// notifications.
1794    async fn wait_for_active(&self) -> Result<(), SendError> {
1795        const FALLBACK_INTERVAL_MS: u64 = 100;
1796
1797        let mode = self.connection_mode();
1798        if mode.is_active() {
1799            return Ok(());
1800        }
1801
1802        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1803            return Err(SendError::Closed);
1804        }
1805
1806        log::debug!("Waiting for client to become ACTIVE before sending...");
1807
1808        let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
1809
1810        dst::time::timeout(self.reconnect_timeout, async {
1811            loop {
1812                // Enable before the state check: an unpolled Notified is unregistered and misses notifies
1813                let mut notified = pin!(self.state_notify.notified());
1814                notified.as_mut().enable();
1815
1816                let mode = self.connection_mode();
1817                if mode.is_active() {
1818                    return Ok(());
1819                }
1820
1821                if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1822                    return Err(());
1823                }
1824
1825                tokio::select! {
1826                    biased;
1827                    () = notified => {}
1828                    () = dst::time::sleep(fallback_interval) => {}
1829                }
1830            }
1831        })
1832        .await
1833        .map_err(|_| SendError::Timeout)?
1834        .map_err(|()| SendError::Closed)
1835    }
1836
1837    /// Signals that the caller's reader has observed EOF or a fatal error.
1838    ///
1839    /// In stream mode the controller has no visibility into the caller-owned reader.
1840    /// Call this method when `reader.next().await` returns `None` or an unrecoverable
1841    /// error so the controller transitions to `Closed` and dependent tasks shut down.
1842    ///
1843    /// For peer-initiated close frames (`Message::Close`), use [`disconnect`](Self::disconnect)
1844    /// instead so the writer can send the close reply before shutting down.
1845    ///
1846    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
1847    ///
1848    /// This is a no-op if the connection is already closed or disconnecting.
1849    pub fn notify_closed(&self) {
1850        let mode = self.connection_mode();
1851        if mode.is_disconnect() || mode.is_closed() {
1852            return;
1853        }
1854
1855        log::debug!("Stream reader signalled EOF, transitioning to CLOSED");
1856
1857        self.connection_mode
1858            .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1859        fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
1860        self.state_notify.notify_waiters();
1861    }
1862
1863    /// Set disconnect mode to true.
1864    ///
1865    /// Controller task will periodically check the disconnect mode
1866    /// and shutdown the client if it is alive
1867    ///
1868    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
1869    pub async fn disconnect(&self) {
1870        log::debug!("Disconnecting");
1871
1872        // A CLOSED client keeps its terminal state; its tracker is already failed
1873        if ConnectionMode::request_disconnect(&self.connection_mode)
1874            && let Some(tracker) = self.auth_tracker.get()
1875        {
1876            tracker.fail("WebSocket client disconnected");
1877        }
1878        self.state_notify.notify_waiters();
1879
1880        if dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
1881            while !self.is_disconnected() {
1882                dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
1883            }
1884
1885            if !self.controller_task.is_finished() {
1886                self.controller_task.abort();
1887                log_task_aborted("controller");
1888            }
1889        })
1890        .await
1891            == Ok(())
1892        {
1893            log::debug!("Controller task finished");
1894        } else {
1895            log::warn!("Timeout waiting for controller task to finish");
1896
1897            if !self.controller_task.is_finished() {
1898                self.controller_task.abort();
1899                log_task_aborted("controller");
1900            }
1901            self.connection_mode
1902                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1903        }
1904    }
1905
1906    /// Sends the given text `data` to the server.
1907    ///
1908    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does NOT
1909    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
1910    /// message. During reconnection, messages are buffered and replayed on the new connection.
1911    ///
1912    /// # Errors
1913    ///
1914    /// Returns a websocket error if unable to send.
1915    #[allow(unused_variables)]
1916    pub async fn send_text(&self, data: String, keys: Option<&[Ustr]>) -> Result<(), SendError> {
1917        self.check_not_terminal()?;
1918
1919        self.await_rate_limit_or_closed(keys).await?;
1920        self.wait_for_active().await?;
1921
1922        log::trace!("Sending text: {data:?}");
1923
1924        let msg = Message::Text(data.into());
1925        self.writer_tx
1926            .send(WriterCommand::Send(msg))
1927            .map_err(|e| SendError::BrokenPipe(e.to_string()))
1928    }
1929
1930    /// Sends a pong frame back to the server.
1931    ///
1932    /// # Errors
1933    ///
1934    /// Returns a websocket error if unable to send.
1935    pub async fn send_pong(&self, data: Vec<u8>) -> Result<(), SendError> {
1936        self.wait_for_active().await?;
1937
1938        log::trace!("Sending pong frame ({} bytes)", data.len());
1939
1940        let msg = Message::Pong(data.into());
1941        self.writer_tx
1942            .send(WriterCommand::Send(msg))
1943            .map_err(|e| SendError::BrokenPipe(e.to_string()))
1944    }
1945
1946    /// Sends the given bytes `data` to the server.
1947    ///
1948    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does NOT
1949    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
1950    /// message. During reconnection, messages are buffered and replayed on the new connection.
1951    ///
1952    /// # Errors
1953    ///
1954    /// Returns a websocket error if unable to send.
1955    #[allow(unused_variables)]
1956    pub async fn send_bytes(&self, data: Vec<u8>, keys: Option<&[Ustr]>) -> Result<(), SendError> {
1957        self.check_not_terminal()?;
1958
1959        self.await_rate_limit_or_closed(keys).await?;
1960        self.wait_for_active().await?;
1961
1962        log::trace!("Sending bytes: {data:?}");
1963
1964        let msg = Message::Binary(data.into());
1965        self.writer_tx
1966            .send(WriterCommand::Send(msg))
1967            .map_err(|e| SendError::BrokenPipe(e.to_string()))
1968    }
1969
1970    /// Sends a close message to the server.
1971    ///
1972    /// # Errors
1973    ///
1974    /// Returns a websocket error if unable to send.
1975    pub async fn send_close_message(&self) -> Result<(), SendError> {
1976        self.wait_for_active().await?;
1977
1978        let msg = Message::Close(None);
1979        self.writer_tx
1980            .send(WriterCommand::Send(msg))
1981            .map_err(|e| SendError::BrokenPipe(e.to_string()))
1982    }
1983
1984    fn spawn_controller_task(
1985        mut inner: WebSocketClientInner,
1986        connection_mode: Arc<AtomicU8>,
1987        state_notify: Arc<tokio::sync::Notify>,
1988        post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1989        auth_tracker: Arc<OnceLock<AuthTracker>>,
1990    ) -> tokio::task::JoinHandle<()> {
1991        const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
1992
1993        tokio::task::spawn(async move {
1994            log_task_started("controller");
1995
1996            let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1997
1998            loop {
1999                tokio::select! {
2000                    biased;
2001                    () = state_notify.notified() => {}
2002                    () = dst::time::sleep(fallback_interval) => {}
2003                }
2004
2005                let mut mode = ConnectionMode::from_atomic(&connection_mode);
2006
2007                if mode.is_disconnect() {
2008                    log::debug!("Disconnecting");
2009
2010                    let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
2011                    if dst::time::timeout(timeout, async {
2012                        // Delay awaiting graceful shutdown
2013                        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
2014
2015                        if let Some(task) = &inner.read_task
2016                            && !task.is_finished()
2017                        {
2018                            task.abort();
2019                            log_task_aborted("read");
2020                        }
2021
2022                        if let Some(task) = &inner.heartbeat_task
2023                            && !task.is_finished()
2024                        {
2025                            task.abort();
2026                            log_task_aborted("heartbeat");
2027                        }
2028                    })
2029                    .await
2030                    .is_err()
2031                    {
2032                        log::warn!("Shutdown timed out after {}s", timeout.as_secs());
2033                    }
2034
2035                    log::debug!("Closed");
2036                    break; // Controller finished
2037                }
2038
2039                if mode.is_closed() {
2040                    log::debug!("Connection closed");
2041                    break;
2042                }
2043
2044                if mode.is_active() && !inner.is_alive() {
2045                    let target = if inner.is_stream_mode {
2046                        ConnectionMode::Closed
2047                    } else {
2048                        ConnectionMode::Reconnect
2049                    };
2050
2051                    if connection_mode
2052                        .compare_exchange(
2053                            ConnectionMode::Active.as_u8(),
2054                            target.as_u8(),
2055                            Ordering::SeqCst,
2056                            Ordering::SeqCst,
2057                        )
2058                        .is_ok()
2059                    {
2060                        if target.is_closed() {
2061                            fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
2062                        } else if let Some(tracker) = auth_tracker.get() {
2063                            tracker.invalidate();
2064                        }
2065                        log::debug!("Detected dead connection, transitioning to {target:?}");
2066                    }
2067                    mode = ConnectionMode::from_atomic(&connection_mode);
2068                }
2069
2070                if mode.is_reconnect() {
2071                    // Check if max reconnection attempts exceeded
2072                    if let Some(max_attempts) = inner.reconnect_max_attempts
2073                        && inner.reconnection_attempt_count >= max_attempts
2074                    {
2075                        log::error!(
2076                            "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
2077                        );
2078                        connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
2079                        fail_registered_auth(
2080                            auth_tracker.as_ref(),
2081                            "WebSocket reconnect attempts exhausted",
2082                        );
2083                        state_notify.notify_waiters();
2084                        break;
2085                    }
2086
2087                    inner.reconnection_attempt_count += 1;
2088                    log::debug!(
2089                        "Reconnection attempt {} of {}",
2090                        inner.reconnection_attempt_count,
2091                        inner
2092                            .reconnect_max_attempts
2093                            .map_or_else(|| "unlimited".to_string(), |m| m.to_string())
2094                    );
2095
2096                    // Race reconnect against disconnect notification
2097                    let reconnect_result = tokio::select! {
2098                        biased;
2099                        result = inner.reconnect() => Some(result),
2100                        () = async {
2101                            loop {
2102                                // Enable before the check so a disconnect notify between iterations is not missed
2103                                let mut notified = pin!(state_notify.notified());
2104                                notified.as_mut().enable();
2105
2106                                if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
2107                                    break;
2108                                }
2109                                notified.await;
2110                            }
2111                        } => None,
2112                    };
2113
2114                    match reconnect_result {
2115                        None => {
2116                            log::debug!("Reconnect interrupted by disconnect");
2117                        }
2118                        Some(Ok(())) => {
2119                            inner.backoff.reset();
2120                            inner.reconnection_attempt_count = 0;
2121
2122                            state_notify.notify_waiters();
2123
2124                            if ConnectionMode::from_atomic(&connection_mode).is_active() {
2125                                if let Some(ref handler) = inner.message_handler {
2126                                    let reconnected_msg =
2127                                        Message::Text(RECONNECTED.to_string().into());
2128                                    handler(reconnected_msg);
2129                                    log::debug!("Sent reconnected message to handler");
2130                                }
2131
2132                                // TODO: Retain this legacy callback for use from Python
2133                                if let Some(ref callback) = post_reconnection {
2134                                    callback();
2135                                    log::debug!("Called `post_reconnection` handler");
2136                                }
2137
2138                                log::debug!("Reconnected successfully");
2139                            } else {
2140                                log::debug!(
2141                                    "Skipping post_reconnection handlers due to disconnect state"
2142                                );
2143                            }
2144                        }
2145                        Some(Err(e)) => {
2146                            let duration = inner.backoff.next_duration();
2147                            log::warn!(
2148                                "Reconnect attempt {} failed: {e}",
2149                                inner.reconnection_attempt_count
2150                            );
2151
2152                            if !duration.is_zero() {
2153                                log::warn!("Backing off for {}s...", duration.as_secs_f64());
2154                                // Race backoff sleep against disconnect
2155                                tokio::select! {
2156                                    biased;
2157                                    () = dst::time::sleep(duration) => {}
2158                                    () = async {
2159                                        loop {
2160                                            // Enable before the check so a disconnect notify between iterations is not missed
2161                                            let mut notified = pin!(state_notify.notified());
2162                                            notified.as_mut().enable();
2163
2164                                            if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
2165                                                break;
2166                                            }
2167                                            notified.await;
2168                                        }
2169                                    } => {
2170                                        log::debug!("Backoff interrupted by disconnect");
2171                                    }
2172                                }
2173                            }
2174                        }
2175                    }
2176                }
2177            }
2178            inner
2179                .connection_mode
2180                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
2181
2182            log_task_stopped("controller");
2183        })
2184    }
2185}
2186
2187fn fail_registered_auth(auth_tracker: &OnceLock<AuthTracker>, reason: &str) {
2188    if let Some(tracker) = auth_tracker.get() {
2189        tracker.fail(reason);
2190    }
2191}
2192
2193// Abort controller task on drop to clean up background tasks
2194impl Drop for WebSocketClient {
2195    fn drop(&mut self) {
2196        if !self.controller_task.is_finished() {
2197            self.controller_task.abort();
2198            log_task_aborted("controller");
2199        }
2200    }
2201}
2202
2203#[cfg(test)]
2204#[cfg(not(feature = "turmoil"))]
2205#[cfg(not(all(feature = "simulation", madsim)))] // transport-layer I/O not simulated
2206#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
2207mod tests {
2208    use std::{num::NonZeroU32, sync::Arc};
2209
2210    use futures_util::{SinkExt, StreamExt};
2211    use tokio::{
2212        net::TcpListener,
2213        task::{self, JoinHandle},
2214    };
2215    use tokio_tungstenite::{
2216        accept_hdr_async,
2217        tungstenite::{
2218            Message as WsMessage,
2219            handshake::server::{self, Callback},
2220            http::HeaderValue,
2221        },
2222    };
2223
2224    use crate::{
2225        ratelimiter::quota::Quota,
2226        websocket::{TransportBackend, WebSocketClient, WebSocketConfig},
2227    };
2228
2229    struct TestServer {
2230        task: JoinHandle<()>,
2231        port: u16,
2232    }
2233
2234    #[derive(Debug, Clone)]
2235    struct TestCallback {
2236        key: String,
2237        value: HeaderValue,
2238    }
2239
2240    impl Callback for TestCallback {
2241        #[expect(clippy::panic_in_result_fn)]
2242        fn on_request(
2243            self,
2244            request: &server::Request,
2245            response: server::Response,
2246        ) -> Result<server::Response, server::ErrorResponse> {
2247            let _ = response;
2248            let value = request.headers().get(&self.key);
2249            assert!(value.is_some());
2250
2251            if let Some(value) = request.headers().get(&self.key) {
2252                assert_eq!(value, self.value);
2253            }
2254
2255            Ok(response)
2256        }
2257    }
2258
2259    impl TestServer {
2260        async fn setup() -> Self {
2261            let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
2262            let port = TcpListener::local_addr(&server).unwrap().port();
2263
2264            let header_key = "test".to_string();
2265            let header_value = "test".to_string();
2266
2267            let test_call_back = TestCallback {
2268                key: header_key,
2269                value: HeaderValue::from_str(&header_value).unwrap(),
2270            };
2271
2272            let task = task::spawn(async move {
2273                // Keep accepting connections
2274                loop {
2275                    let (conn, _) = server.accept().await.unwrap();
2276                    let mut websocket = accept_hdr_async(conn, test_call_back.clone())
2277                        .await
2278                        .unwrap();
2279
2280                    task::spawn(async move {
2281                        while let Some(Ok(msg)) = websocket.next().await {
2282                            match msg {
2283                                WsMessage::Text(txt) if txt == "close-now" => {
2284                                    log::debug!("Forcibly closing from server side");
2285                                    // This sends a close frame, then stops reading
2286                                    let _ = websocket.close(None).await;
2287                                    break;
2288                                }
2289                                // Echo text/binary frames
2290                                WsMessage::Text(_) | WsMessage::Binary(_) => {
2291                                    if websocket.send(msg).await.is_err() {
2292                                        break;
2293                                    }
2294                                }
2295                                // If the client closes, we also break
2296                                WsMessage::Close(_frame) => {
2297                                    let _ = websocket.close(None).await;
2298                                    break;
2299                                }
2300                                // Ignore pings/pongs
2301                                _ => {}
2302                            }
2303                        }
2304                    });
2305                }
2306            });
2307
2308            Self { task, port }
2309        }
2310    }
2311
2312    impl Drop for TestServer {
2313        fn drop(&mut self) {
2314            self.task.abort();
2315        }
2316    }
2317
2318    async fn setup_test_client(port: u16) -> WebSocketClient {
2319        let config = WebSocketConfig {
2320            url: format!("ws://127.0.0.1:{port}"),
2321            headers: vec![("test".into(), "test".into())],
2322            heartbeat: None,
2323            heartbeat_msg: None,
2324            reconnect_timeout_ms: None,
2325            reconnect_delay_initial_ms: None,
2326            reconnect_backoff_factor: None,
2327            reconnect_delay_max_ms: None,
2328            reconnect_jitter_ms: None,
2329            reconnect_max_attempts: None,
2330            idle_timeout_ms: None,
2331            backend: TransportBackend::Tungstenite,
2332            proxy_url: None,
2333        };
2334        WebSocketClient::connect(config, Some(Arc::new(|_| {})), None, None, vec![], None)
2335            .await
2336            .expect("Failed to connect")
2337    }
2338
2339    #[tokio::test]
2340    async fn test_websocket_basic() {
2341        let server = TestServer::setup().await;
2342        let client = setup_test_client(server.port).await;
2343
2344        assert!(!client.is_disconnected());
2345
2346        client.disconnect().await;
2347        assert!(client.is_disconnected());
2348    }
2349
2350    #[tokio::test]
2351    async fn test_websocket_heartbeat() {
2352        let server = TestServer::setup().await;
2353        let client = setup_test_client(server.port).await;
2354
2355        // Wait ~3s => server should see multiple "ping"
2356        tokio::time::sleep(std::time::Duration::from_secs(3)).await;
2357
2358        // Cleanup
2359        client.disconnect().await;
2360        assert!(client.is_disconnected());
2361    }
2362
2363    #[tokio::test]
2364    async fn test_websocket_reconnect_exhausted() {
2365        let config = WebSocketConfig {
2366            url: "ws://127.0.0.1:9997".into(), // <-- No server
2367            headers: vec![],
2368            heartbeat: None,
2369            heartbeat_msg: None,
2370            reconnect_timeout_ms: None,
2371            reconnect_delay_initial_ms: None,
2372            reconnect_backoff_factor: None,
2373            reconnect_delay_max_ms: None,
2374            reconnect_jitter_ms: None,
2375            reconnect_max_attempts: None,
2376            idle_timeout_ms: None,
2377            backend: TransportBackend::Tungstenite,
2378            proxy_url: None,
2379        };
2380        let res =
2381            WebSocketClient::connect(config, Some(Arc::new(|_| {})), None, None, vec![], None)
2382                .await;
2383        assert!(res.is_err(), "Should fail quickly with no server");
2384    }
2385
2386    #[tokio::test]
2387    async fn test_websocket_forced_close_reconnect() {
2388        let server = TestServer::setup().await;
2389        let client = setup_test_client(server.port).await;
2390
2391        // 1) Send normal message
2392        client.send_text("Hello".into(), None).await.unwrap();
2393
2394        // 2) Trigger forced close from server
2395        client.send_text("close-now".into(), None).await.unwrap();
2396
2397        // 3) Wait a bit => read loop sees close => reconnect
2398        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
2399
2400        // Confirm not disconnected
2401        assert!(!client.is_disconnected());
2402
2403        // Cleanup
2404        client.disconnect().await;
2405        assert!(client.is_disconnected());
2406    }
2407
2408    #[tokio::test]
2409    async fn test_rate_limiter() {
2410        let server = TestServer::setup().await;
2411        let quota = Quota::per_second(NonZeroU32::new(2).unwrap()).unwrap();
2412
2413        let config = WebSocketConfig {
2414            url: format!("ws://127.0.0.1:{}", server.port),
2415            headers: vec![("test".into(), "test".into())],
2416            heartbeat: None,
2417            heartbeat_msg: None,
2418            reconnect_timeout_ms: None,
2419            reconnect_delay_initial_ms: None,
2420            reconnect_backoff_factor: None,
2421            reconnect_delay_max_ms: None,
2422            reconnect_jitter_ms: None,
2423            reconnect_max_attempts: None,
2424            idle_timeout_ms: None,
2425            backend: TransportBackend::Tungstenite,
2426            proxy_url: None,
2427        };
2428
2429        let client = WebSocketClient::connect(
2430            config,
2431            Some(Arc::new(|_| {})),
2432            None,
2433            None,
2434            vec![("default".into(), quota)],
2435            None,
2436        )
2437        .await
2438        .unwrap();
2439
2440        // Burst of 2 passes immediately; the third send must wait for the
2441        // ~500ms replenish interval (keys=None would bypass the limiter)
2442        let keys: [ustr::Ustr; 1] = [ustr::Ustr::from("default")];
2443        let start = std::time::Instant::now();
2444        client
2445            .send_text("test1".into(), Some(keys.as_slice()))
2446            .await
2447            .unwrap();
2448        client
2449            .send_text("test2".into(), Some(keys.as_slice()))
2450            .await
2451            .unwrap();
2452        let after_burst = start.elapsed();
2453        client
2454            .send_text("test3".into(), Some(keys.as_slice()))
2455            .await
2456            .unwrap();
2457        let after_third = start.elapsed();
2458
2459        assert!(
2460            after_burst < std::time::Duration::from_millis(300),
2461            "Burst sends should not be rate limited, took {after_burst:?}"
2462        );
2463        assert!(
2464            after_third >= std::time::Duration::from_millis(400),
2465            "Third send should wait for quota replenishment, took {after_third:?}"
2466        );
2467
2468        // Cleanup
2469        client.disconnect().await;
2470        assert!(client.is_disconnected());
2471    }
2472
2473    #[tokio::test]
2474    async fn test_concurrent_writers() {
2475        let server = TestServer::setup().await;
2476        let client = Arc::new(setup_test_client(server.port).await);
2477
2478        let mut handles = vec![];
2479
2480        for i in 0..10 {
2481            let client = client.clone();
2482            handles.push(task::spawn(async move {
2483                client.send_text(format!("test{i}"), None).await.unwrap();
2484            }));
2485        }
2486
2487        for handle in handles {
2488            handle.await.unwrap();
2489        }
2490
2491        // Cleanup
2492        client.disconnect().await;
2493        assert!(client.is_disconnected());
2494    }
2495}
2496
2497#[cfg(test)]
2498#[cfg(not(feature = "turmoil"))]
2499#[cfg(not(all(feature = "simulation", madsim)))] // transport-layer I/O not simulated
2500mod rust_tests {
2501    use std::sync::{
2502        Arc, OnceLock,
2503        atomic::{AtomicBool, AtomicU8, Ordering},
2504    };
2505
2506    use futures_util::{SinkExt, StreamExt};
2507    use nautilus_common::testing::wait_until_async;
2508    use rstest::rstest;
2509    #[cfg(feature = "transport-sockudo")]
2510    use sockudo_ws::handshake as sockudo_handshake;
2511    #[cfg(feature = "transport-sockudo")]
2512    use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
2513    use tokio::{
2514        net::TcpListener,
2515        task::{self, JoinHandle},
2516        time::{Duration, sleep},
2517    };
2518    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
2519    #[cfg(feature = "transport-sockudo")]
2520    use tokio_tungstenite::{
2521        accept_hdr_async,
2522        tungstenite::{
2523            handshake::server::{self, Callback},
2524            http::HeaderValue,
2525        },
2526    };
2527
2528    use super::*;
2529    use crate::websocket::types::channel_message_handler;
2530
2531    struct RecordingServer {
2532        task: JoinHandle<()>,
2533        port: u16,
2534        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
2535    }
2536
2537    #[cfg(feature = "transport-sockudo")]
2538    async fn read_http_request<S>(stream: &mut S) -> Vec<u8>
2539    where
2540        S: AsyncRead + Unpin,
2541    {
2542        let mut buf = Vec::new();
2543        let mut chunk = [0u8; 256];
2544
2545        loop {
2546            let n = stream.read(&mut chunk).await.unwrap();
2547            assert!(n > 0, "HTTP request closed before headers completed");
2548            buf.extend_from_slice(&chunk[..n]);
2549            if buf.windows(4).any(|window| window == b"\r\n\r\n") {
2550                return buf;
2551            }
2552        }
2553    }
2554
2555    #[cfg(feature = "transport-sockudo")]
2556    fn extract_header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
2557        request.lines().find_map(|line| {
2558            let (header_name, header_value) = line.split_once(':')?;
2559            if header_name.eq_ignore_ascii_case(name) {
2560                Some(header_value.trim())
2561            } else {
2562                None
2563            }
2564        })
2565    }
2566
2567    #[cfg(feature = "transport-sockudo")]
2568    #[derive(Debug, Clone)]
2569    struct HeaderAssertCallback {
2570        key: String,
2571        value: HeaderValue,
2572    }
2573
2574    #[cfg(feature = "transport-sockudo")]
2575    impl Callback for HeaderAssertCallback {
2576        #[expect(
2577            clippy::panic_in_result_fn,
2578            reason = "assertion failures should fail the test"
2579        )]
2580        fn on_request(
2581            self,
2582            request: &server::Request,
2583            response: server::Response,
2584        ) -> Result<server::Response, server::ErrorResponse> {
2585            assert_eq!(request.headers().get(&self.key), Some(&self.value));
2586            Ok(response)
2587        }
2588    }
2589
2590    impl RecordingServer {
2591        async fn setup() -> Self {
2592            let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2593            let port = listener.local_addr().unwrap().port();
2594            let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
2595            let messages_clone = Arc::clone(&messages);
2596
2597            let task = task::spawn(async move {
2598                loop {
2599                    let (stream, _) = listener.accept().await.unwrap();
2600                    let mut websocket = accept_async(stream).await.unwrap();
2601                    let messages = Arc::clone(&messages_clone);
2602
2603                    task::spawn(async move {
2604                        while let Some(Ok(msg)) = websocket.next().await {
2605                            match msg {
2606                                WsMessage::Text(text) => {
2607                                    messages.lock().await.push(text.to_string());
2608                                }
2609                                WsMessage::Close(_) => {
2610                                    let _ = websocket.close(None).await;
2611                                    break;
2612                                }
2613                                _ => {}
2614                            }
2615                        }
2616                    });
2617                }
2618            });
2619
2620            Self {
2621                task,
2622                port,
2623                messages,
2624            }
2625        }
2626
2627        async fn messages(&self) -> Vec<String> {
2628            self.messages.lock().await.clone()
2629        }
2630    }
2631
2632    impl Drop for RecordingServer {
2633        fn drop(&mut self) {
2634            self.task.abort();
2635        }
2636    }
2637
2638    #[rstest]
2639    #[tokio::test]
2640    async fn test_reconnect_then_disconnect() {
2641        // Bind an ephemeral port
2642        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2643        let port = listener.local_addr().unwrap().port();
2644
2645        // Server task: accept one ws connection then close it
2646        let server = task::spawn(async move {
2647            let (stream, _) = listener.accept().await.unwrap();
2648            let ws = accept_async(stream).await.unwrap();
2649            drop(ws);
2650            // Keep alive briefly
2651            sleep(Duration::from_secs(1)).await;
2652        });
2653
2654        // Build a channel-based message handler for incoming messages (unused here)
2655        let (handler, _rx) = channel_message_handler();
2656
2657        // Configure client with short reconnect backoff
2658        let config = WebSocketConfig {
2659            url: format!("ws://127.0.0.1:{port}"),
2660            headers: vec![],
2661            heartbeat: None,
2662            heartbeat_msg: None,
2663            reconnect_timeout_ms: Some(1_000),
2664            reconnect_delay_initial_ms: Some(50),
2665            reconnect_delay_max_ms: Some(100),
2666            reconnect_backoff_factor: Some(1.0),
2667            reconnect_jitter_ms: Some(0),
2668            reconnect_max_attempts: None,
2669            idle_timeout_ms: None,
2670            backend: TransportBackend::Tungstenite,
2671            proxy_url: None,
2672        };
2673
2674        // Connect the client
2675        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
2676            .await
2677            .unwrap();
2678
2679        // Allow server to drop connection and client to detect
2680        sleep(Duration::from_millis(100)).await;
2681        // Now immediately disconnect the client
2682        client.disconnect().await;
2683        assert!(client.is_disconnected());
2684        server.abort();
2685    }
2686
2687    #[rstest]
2688    #[tokio::test]
2689    async fn test_reconnect_state_flips_when_reader_stops() {
2690        // Bind an ephemeral port and accept a single websocket connection which we drop.
2691        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2692        let port = listener.local_addr().unwrap().port();
2693
2694        let server = task::spawn(async move {
2695            if let Ok((stream, _)) = listener.accept().await
2696                && let Ok(ws) = accept_async(stream).await
2697            {
2698                drop(ws);
2699            }
2700            sleep(Duration::from_millis(50)).await;
2701        });
2702
2703        let (handler, _rx) = channel_message_handler();
2704
2705        let config = WebSocketConfig {
2706            url: format!("ws://127.0.0.1:{port}"),
2707            headers: vec![],
2708            heartbeat: None,
2709            heartbeat_msg: None,
2710            reconnect_timeout_ms: Some(1_000),
2711            reconnect_delay_initial_ms: Some(50),
2712            reconnect_delay_max_ms: Some(100),
2713            reconnect_backoff_factor: Some(1.0),
2714            reconnect_jitter_ms: Some(0),
2715            reconnect_max_attempts: None,
2716            idle_timeout_ms: None,
2717            backend: TransportBackend::Tungstenite,
2718            proxy_url: None,
2719        };
2720
2721        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
2722            .await
2723            .unwrap();
2724
2725        tokio::time::timeout(Duration::from_secs(2), async {
2726            loop {
2727                if client.is_reconnecting() {
2728                    break;
2729                }
2730                tokio::time::sleep(Duration::from_millis(10)).await;
2731            }
2732        })
2733        .await
2734        .expect("client did not enter RECONNECT state");
2735
2736        client.disconnect().await;
2737        server.abort();
2738    }
2739
2740    #[rstest]
2741    #[tokio::test]
2742    async fn test_stream_mode_disables_auto_reconnect() {
2743        // Test that stream-based clients (created via connect_stream) set is_stream_mode flag
2744        // and that reconnect() transitions to CLOSED state for stream mode
2745        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2746        let port = listener.local_addr().unwrap().port();
2747
2748        let server = task::spawn(async move {
2749            if let Ok((stream, _)) = listener.accept().await
2750                && let Ok(_ws) = accept_async(stream).await
2751            {
2752                // Keep connection alive briefly
2753                sleep(Duration::from_millis(100)).await;
2754            }
2755        });
2756
2757        let config = WebSocketConfig {
2758            url: format!("ws://127.0.0.1:{port}"),
2759            headers: vec![],
2760            heartbeat: None,
2761            heartbeat_msg: None,
2762            reconnect_timeout_ms: Some(1_000),
2763            reconnect_delay_initial_ms: Some(50),
2764            reconnect_delay_max_ms: Some(100),
2765            reconnect_backoff_factor: Some(1.0),
2766            reconnect_jitter_ms: Some(0),
2767            reconnect_max_attempts: None,
2768            idle_timeout_ms: None,
2769            backend: TransportBackend::Tungstenite,
2770            proxy_url: None,
2771        };
2772
2773        let (_reader, _client) = WebSocketClient::connect_stream(config, vec![], None, None)
2774            .await
2775            .unwrap();
2776
2777        // Note: We can't easily test the reconnect behavior from the outside since
2778        // the inner client is private. The key fix is that WebSocketClientInner
2779        // now has is_stream_mode=true for connect_stream, and reconnect() will
2780        // transition to CLOSED state instead of creating a new reader that gets dropped.
2781        // This is tested implicitly by the fact that stream users won't get stuck
2782        // in an infinite reconnect loop.
2783
2784        server.abort();
2785    }
2786
2787    #[rstest]
2788    #[tokio::test]
2789    async fn test_message_handler_mode_allows_auto_reconnect() {
2790        // Test that regular clients (with message handler) can auto-reconnect
2791        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2792        let port = listener.local_addr().unwrap().port();
2793
2794        let server = task::spawn(async move {
2795            // Accept first connection and close it
2796            if let Ok((stream, _)) = listener.accept().await
2797                && let Ok(ws) = accept_async(stream).await
2798            {
2799                drop(ws);
2800            }
2801            sleep(Duration::from_millis(50)).await;
2802        });
2803
2804        let (handler, _rx) = channel_message_handler();
2805
2806        let config = WebSocketConfig {
2807            url: format!("ws://127.0.0.1:{port}"),
2808            headers: vec![],
2809            heartbeat: None,
2810            heartbeat_msg: None,
2811            reconnect_timeout_ms: Some(1_000),
2812            reconnect_delay_initial_ms: Some(50),
2813            reconnect_delay_max_ms: Some(100),
2814            reconnect_backoff_factor: Some(1.0),
2815            reconnect_jitter_ms: Some(0),
2816            reconnect_max_attempts: None,
2817            idle_timeout_ms: None,
2818            backend: TransportBackend::Tungstenite,
2819            proxy_url: None,
2820        };
2821
2822        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
2823            .await
2824            .unwrap();
2825
2826        // Wait for the connection to be dropped and reconnection to be attempted
2827        tokio::time::timeout(Duration::from_secs(2), async {
2828            loop {
2829                if client.is_reconnecting() || client.is_closed() {
2830                    break;
2831                }
2832                tokio::time::sleep(Duration::from_millis(10)).await;
2833            }
2834        })
2835        .await
2836        .expect("client should attempt reconnection or close");
2837
2838        // Should either be reconnecting or closed (depending on timing)
2839        // The important thing is it's not staying active forever
2840        assert!(
2841            client.is_reconnecting() || client.is_closed(),
2842            "Client with message handler should attempt reconnection"
2843        );
2844
2845        client.disconnect().await;
2846        server.abort();
2847    }
2848
2849    #[rstest]
2850    #[tokio::test]
2851    async fn test_handler_mode_reconnect_with_new_connection() {
2852        // Test that handler mode successfully reconnects and messages continue flowing
2853        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2854        let port = listener.local_addr().unwrap().port();
2855
2856        let server = task::spawn(async move {
2857            // First connection - accept and immediately close
2858            if let Ok((stream, _)) = listener.accept().await
2859                && let Ok(ws) = accept_async(stream).await
2860            {
2861                drop(ws);
2862            }
2863
2864            // Small delay to let client detect disconnection
2865            sleep(Duration::from_millis(100)).await;
2866
2867            // Second connection - accept, send a message, then keep alive
2868            if let Ok((stream, _)) = listener.accept().await
2869                && let Ok(mut ws) = accept_async(stream).await
2870            {
2871                use futures_util::SinkExt;
2872                let _ = ws
2873                    .send(WsMessage::Text("reconnected".to_string().into()))
2874                    .await;
2875                sleep(Duration::from_secs(1)).await;
2876            }
2877        });
2878
2879        let (handler, mut rx) = channel_message_handler();
2880
2881        let config = WebSocketConfig {
2882            url: format!("ws://127.0.0.1:{port}"),
2883            headers: vec![],
2884            heartbeat: None,
2885            heartbeat_msg: None,
2886            reconnect_timeout_ms: Some(2_000),
2887            reconnect_delay_initial_ms: Some(50),
2888            reconnect_delay_max_ms: Some(200),
2889            reconnect_backoff_factor: Some(1.5),
2890            reconnect_jitter_ms: Some(10),
2891            reconnect_max_attempts: None,
2892            idle_timeout_ms: None,
2893            backend: TransportBackend::Tungstenite,
2894            proxy_url: None,
2895        };
2896
2897        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
2898            .await
2899            .unwrap();
2900
2901        // Wait for reconnection to happen and message to arrive
2902        let result = tokio::time::timeout(Duration::from_secs(5), async {
2903            loop {
2904                if let Ok(msg) = rx.try_recv()
2905                    && matches!(msg, WsMessage::Text(ref text) if AsRef::<str>::as_ref(text) == "reconnected")
2906                {
2907                    return true;
2908                }
2909                tokio::time::sleep(Duration::from_millis(10)).await;
2910            }
2911        })
2912        .await;
2913
2914        assert!(
2915            result.is_ok(),
2916            "Should receive message after reconnection within timeout"
2917        );
2918
2919        client.disconnect().await;
2920        server.abort();
2921    }
2922
2923    #[rstest]
2924    #[tokio::test]
2925    async fn test_stream_mode_no_auto_reconnect() {
2926        // Test that stream mode does not automatically reconnect when connection is lost
2927        // The caller owns the reader and is responsible for detecting disconnection
2928        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2929        let port = listener.local_addr().unwrap().port();
2930
2931        let server = task::spawn(async move {
2932            // Accept connection and send one message, then close
2933            if let Ok((stream, _)) = listener.accept().await
2934                && let Ok(mut ws) = accept_async(stream).await
2935            {
2936                use futures_util::SinkExt;
2937                let _ = ws.send(WsMessage::Text("hello".to_string().into())).await;
2938                sleep(Duration::from_millis(50)).await;
2939                // Connection closes when ws is dropped
2940            }
2941        });
2942
2943        let config = WebSocketConfig {
2944            url: format!("ws://127.0.0.1:{port}"),
2945            headers: vec![],
2946            heartbeat: None,
2947            heartbeat_msg: None,
2948            reconnect_timeout_ms: Some(1_000),
2949            reconnect_delay_initial_ms: Some(50),
2950            reconnect_delay_max_ms: Some(100),
2951            reconnect_backoff_factor: Some(1.0),
2952            reconnect_jitter_ms: Some(0),
2953            reconnect_max_attempts: None,
2954            idle_timeout_ms: None,
2955            backend: TransportBackend::Tungstenite,
2956            proxy_url: None,
2957        };
2958
2959        let (mut reader, client) = WebSocketClient::connect_stream(config, vec![], None, None)
2960            .await
2961            .unwrap();
2962
2963        // Initially active
2964        assert!(client.is_active(), "Client should start as active");
2965
2966        // Read the hello message
2967        let msg = reader.next().await;
2968        assert!(
2969            matches!(&msg, Some(Ok(Message::Text(bytes))) if bytes.as_ref() == b"hello"),
2970            "Should receive initial message"
2971        );
2972
2973        // Read until connection closes (reader will return None or error)
2974        while let Some(msg) = reader.next().await {
2975            if msg.is_err() || matches!(msg, Ok(Message::Close(_))) {
2976                break;
2977            }
2978        }
2979
2980        // Controller cannot detect reader EOF (reader is owned by caller),
2981        // so the client stays ACTIVE until the caller signals.
2982        sleep(Duration::from_millis(200)).await;
2983        assert!(
2984            client.is_active(),
2985            "Stream mode client stays ACTIVE before notify_closed()"
2986        );
2987
2988        // Caller signals EOF via notify_closed()
2989        client.notify_closed();
2990
2991        assert!(
2992            client.is_closed(),
2993            "Stream mode client should be CLOSED after notify_closed()"
2994        );
2995        assert!(
2996            !client.is_reconnecting(),
2997            "Stream mode client should never attempt reconnection"
2998        );
2999
3000        client.disconnect().await;
3001        server.abort();
3002    }
3003
3004    #[rstest]
3005    #[tokio::test]
3006    async fn test_send_timeout_uses_configured_reconnect_timeout() {
3007        // Test that send operations respect the configured reconnect_timeout.
3008        // When a client is stuck in RECONNECT longer than the timeout, sends should fail with Timeout.
3009        use nautilus_common::testing::wait_until_async;
3010
3011        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3012        let port = listener.local_addr().unwrap().port();
3013
3014        let server = task::spawn(async move {
3015            // Accept first connection and immediately close it
3016            if let Ok((stream, _)) = listener.accept().await
3017                && let Ok(ws) = accept_async(stream).await
3018            {
3019                drop(ws);
3020            }
3021            // Don't accept second connection - client will be stuck in RECONNECT
3022            sleep(Duration::from_mins(1)).await;
3023        });
3024
3025        let (handler, _rx) = channel_message_handler();
3026
3027        // Configure with SHORT 2s reconnect timeout
3028        let config = WebSocketConfig {
3029            url: format!("ws://127.0.0.1:{port}"),
3030            headers: vec![],
3031            heartbeat: None,
3032            heartbeat_msg: None,
3033            reconnect_timeout_ms: Some(2_000), // 2s timeout
3034            reconnect_delay_initial_ms: Some(50),
3035            reconnect_delay_max_ms: Some(100),
3036            reconnect_backoff_factor: Some(1.0),
3037            reconnect_jitter_ms: Some(0),
3038            reconnect_max_attempts: None,
3039            idle_timeout_ms: None,
3040            backend: TransportBackend::Tungstenite,
3041            proxy_url: None,
3042        };
3043
3044        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3045            .await
3046            .unwrap();
3047
3048        // Wait for client to enter RECONNECT state
3049        wait_until_async(
3050            || async { client.is_reconnecting() },
3051            Duration::from_secs(3),
3052        )
3053        .await;
3054
3055        // Attempt send while stuck in RECONNECT - should timeout after 2s (configured timeout)
3056        let start = std::time::Instant::now();
3057        let send_result = client.send_text("test".to_string(), None).await;
3058        let elapsed = start.elapsed();
3059
3060        assert!(
3061            send_result.is_err(),
3062            "Send should fail when client stuck in RECONNECT"
3063        );
3064        assert!(
3065            matches!(send_result, Err(crate::error::SendError::Timeout)),
3066            "Send should return Timeout error, was: {send_result:?}"
3067        );
3068        // Verify timeout respects configured value (2s), but don't check upper bound
3069        // as CI scheduler jitter can cause legitimate delays beyond the timeout
3070        assert!(
3071            elapsed >= Duration::from_millis(1800),
3072            "Send should timeout after at least 2s (configured timeout), took {elapsed:?}"
3073        );
3074
3075        client.disconnect().await;
3076        server.abort();
3077    }
3078
3079    #[rstest]
3080    #[tokio::test]
3081    async fn test_send_waits_during_reconnection() {
3082        // Test that send operations wait for reconnection to complete (up to timeout)
3083        use nautilus_common::testing::wait_until_async;
3084
3085        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3086        let port = listener.local_addr().unwrap().port();
3087
3088        let server = task::spawn(async move {
3089            // First connection - accept and immediately close
3090            if let Ok((stream, _)) = listener.accept().await
3091                && let Ok(ws) = accept_async(stream).await
3092            {
3093                drop(ws);
3094            }
3095
3096            // Wait a bit before accepting second connection
3097            sleep(Duration::from_millis(500)).await;
3098
3099            // Second connection - accept and keep alive
3100            if let Ok((stream, _)) = listener.accept().await
3101                && let Ok(mut ws) = accept_async(stream).await
3102            {
3103                // Echo messages
3104                while let Some(Ok(msg)) = ws.next().await {
3105                    if ws.send(msg).await.is_err() {
3106                        break;
3107                    }
3108                }
3109            }
3110        });
3111
3112        let (handler, _rx) = channel_message_handler();
3113
3114        let config = WebSocketConfig {
3115            url: format!("ws://127.0.0.1:{port}"),
3116            headers: vec![],
3117            heartbeat: None,
3118            heartbeat_msg: None,
3119            reconnect_timeout_ms: Some(5_000), // 5s timeout - enough for reconnect
3120            reconnect_delay_initial_ms: Some(100),
3121            reconnect_delay_max_ms: Some(200),
3122            reconnect_backoff_factor: Some(1.0),
3123            reconnect_jitter_ms: Some(0),
3124            reconnect_max_attempts: None,
3125            idle_timeout_ms: None,
3126            backend: TransportBackend::Tungstenite,
3127            proxy_url: None,
3128        };
3129
3130        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3131            .await
3132            .unwrap();
3133
3134        // Wait for reconnection to trigger
3135        wait_until_async(
3136            || async { client.is_reconnecting() },
3137            Duration::from_secs(2),
3138        )
3139        .await;
3140
3141        // Try to send while reconnecting - should wait and succeed after reconnect
3142        let send_result = tokio::time::timeout(
3143            Duration::from_secs(3),
3144            client.send_text("test_message".to_string(), None),
3145        )
3146        .await;
3147
3148        assert!(
3149            send_result.is_ok() && send_result.unwrap().is_ok(),
3150            "Send should succeed after waiting for reconnection"
3151        );
3152
3153        client.disconnect().await;
3154        server.abort();
3155    }
3156
3157    #[rstest]
3158    #[tokio::test]
3159    async fn test_rate_limiter_before_active_wait() {
3160        // Test that rate limiting happens BEFORE active state check.
3161        // This prevents race conditions where connection state changes during rate limit wait.
3162        // We verify this by: (1) exhausting rate limit, (2) ensuring client is RECONNECTING,
3163        // (3) sending again and confirming it waits for rate limit THEN reconnection.
3164        use std::{num::NonZeroU32, sync::Arc};
3165
3166        use nautilus_common::testing::wait_until_async;
3167
3168        use crate::ratelimiter::quota::Quota;
3169
3170        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3171        let port = listener.local_addr().unwrap().port();
3172
3173        let server = task::spawn(async move {
3174            // First connection - accept and close after receiving one message
3175            if let Ok((stream, _)) = listener.accept().await
3176                && let Ok(mut ws) = accept_async(stream).await
3177            {
3178                // Receive first message then close
3179                if let Some(Ok(_)) = ws.next().await {
3180                    drop(ws);
3181                }
3182            }
3183
3184            // Wait before accepting reconnection
3185            sleep(Duration::from_millis(500)).await;
3186
3187            // Second connection - accept and keep alive
3188            if let Ok((stream, _)) = listener.accept().await
3189                && let Ok(mut ws) = accept_async(stream).await
3190            {
3191                while let Some(Ok(msg)) = ws.next().await {
3192                    if ws.send(msg).await.is_err() {
3193                        break;
3194                    }
3195                }
3196            }
3197        });
3198
3199        let (handler, _rx) = channel_message_handler();
3200
3201        let config = WebSocketConfig {
3202            url: format!("ws://127.0.0.1:{port}"),
3203            headers: vec![],
3204            heartbeat: None,
3205            heartbeat_msg: None,
3206            reconnect_timeout_ms: Some(5_000),
3207            reconnect_delay_initial_ms: Some(50),
3208            reconnect_delay_max_ms: Some(100),
3209            reconnect_backoff_factor: Some(1.0),
3210            reconnect_jitter_ms: Some(0),
3211            reconnect_max_attempts: None,
3212            idle_timeout_ms: None,
3213            backend: TransportBackend::Tungstenite,
3214            proxy_url: None,
3215        };
3216
3217        // Very restrictive rate limit: 1 request per second, burst of 1
3218        let quota = Quota::per_second(NonZeroU32::new(1).unwrap())
3219            .unwrap()
3220            .allow_burst(NonZeroU32::new(1).unwrap());
3221
3222        let client = Arc::new(
3223            WebSocketClient::connect(
3224                config,
3225                Some(handler),
3226                None,
3227                None,
3228                vec![("test_key".to_string(), quota)],
3229                None,
3230            )
3231            .await
3232            .unwrap(),
3233        );
3234
3235        // First send exhausts burst capacity and triggers connection close
3236        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
3237        client
3238            .send_text("msg1".to_string(), Some(test_key.as_slice()))
3239            .await
3240            .unwrap();
3241
3242        // Wait for client to enter RECONNECT state
3243        wait_until_async(
3244            || async { client.is_reconnecting() },
3245            Duration::from_secs(2),
3246        )
3247        .await;
3248
3249        // Second send: will hit rate limit (~1s) THEN wait for reconnection (~0.5s)
3250        let start = std::time::Instant::now();
3251        let send_result = client
3252            .send_text("msg2".to_string(), Some(test_key.as_slice()))
3253            .await;
3254        let elapsed = start.elapsed();
3255
3256        // Should succeed after both rate limit AND reconnection
3257        assert!(
3258            send_result.is_ok(),
3259            "Send should succeed after rate limit + reconnection, was: {send_result:?}"
3260        );
3261        // Total wait should be at least rate limit time (~1s)
3262        // The reconnection completes while rate limiting or after
3263        // Use 850ms threshold to account for timing jitter in CI
3264        assert!(
3265            elapsed >= Duration::from_millis(850),
3266            "Should wait for rate limit (~1s), waited {elapsed:?}"
3267        );
3268
3269        client.disconnect().await;
3270        server.abort();
3271    }
3272
3273    #[rstest]
3274    #[tokio::test]
3275    async fn test_disconnect_during_reconnect_exits_cleanly() {
3276        // Test CAS race condition: disconnect called during reconnection
3277        // Should exit cleanly without spawning new tasks
3278        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3279        let port = listener.local_addr().unwrap().port();
3280
3281        let server = task::spawn(async move {
3282            // Accept first connection and immediately close
3283            if let Ok((stream, _)) = listener.accept().await
3284                && let Ok(ws) = accept_async(stream).await
3285            {
3286                drop(ws);
3287            }
3288            // Don't accept second connection - let reconnect hang
3289            sleep(Duration::from_mins(1)).await;
3290        });
3291
3292        let (handler, _rx) = channel_message_handler();
3293
3294        let config = WebSocketConfig {
3295            url: format!("ws://127.0.0.1:{port}"),
3296            headers: vec![],
3297            heartbeat: None,
3298            heartbeat_msg: None,
3299            reconnect_timeout_ms: Some(2_000), // 2s timeout - shorter than disconnect timeout
3300            reconnect_delay_initial_ms: Some(100),
3301            reconnect_delay_max_ms: Some(200),
3302            reconnect_backoff_factor: Some(1.0),
3303            reconnect_jitter_ms: Some(0),
3304            reconnect_max_attempts: None,
3305            idle_timeout_ms: None,
3306            backend: TransportBackend::Tungstenite,
3307            proxy_url: None,
3308        };
3309
3310        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3311            .await
3312            .unwrap();
3313
3314        // Wait for reconnection to start
3315        tokio::time::timeout(Duration::from_secs(2), async {
3316            while !client.is_reconnecting() {
3317                sleep(Duration::from_millis(10)).await;
3318            }
3319        })
3320        .await
3321        .expect("Client should enter RECONNECT state");
3322
3323        // Disconnect while reconnecting
3324        client.disconnect().await;
3325
3326        // Should be cleanly closed
3327        assert!(
3328            client.is_disconnected(),
3329            "Client should be cleanly disconnected"
3330        );
3331
3332        server.abort();
3333    }
3334
3335    #[rstest]
3336    #[tokio::test]
3337    async fn test_send_fails_fast_when_closed_before_rate_limit() {
3338        // Test that send operations check connection state BEFORE rate limiting,
3339        // preventing unnecessary delays when the connection is already closed.
3340        use std::{num::NonZeroU32, sync::Arc};
3341
3342        use nautilus_common::testing::wait_until_async;
3343
3344        use crate::ratelimiter::quota::Quota;
3345
3346        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3347        let port = listener.local_addr().unwrap().port();
3348
3349        let server = task::spawn(async move {
3350            // Accept connection and immediately close
3351            if let Ok((stream, _)) = listener.accept().await
3352                && let Ok(ws) = accept_async(stream).await
3353            {
3354                drop(ws);
3355            }
3356            sleep(Duration::from_mins(1)).await;
3357        });
3358
3359        let (handler, _rx) = channel_message_handler();
3360
3361        let config = WebSocketConfig {
3362            url: format!("ws://127.0.0.1:{port}"),
3363            headers: vec![],
3364            heartbeat: None,
3365            heartbeat_msg: None,
3366            reconnect_timeout_ms: Some(5_000),
3367            reconnect_delay_initial_ms: Some(50),
3368            reconnect_delay_max_ms: Some(100),
3369            reconnect_backoff_factor: Some(1.0),
3370            reconnect_jitter_ms: Some(0),
3371            reconnect_max_attempts: None,
3372            idle_timeout_ms: None,
3373            backend: TransportBackend::Tungstenite,
3374            proxy_url: None,
3375        };
3376
3377        // Very restrictive rate limit: 1 request per 10 seconds
3378        // This ensures that if we wait for rate limit, the test will timeout
3379        let quota = Quota::with_period(Duration::from_secs(10))
3380            .unwrap()
3381            .allow_burst(NonZeroU32::new(1).unwrap());
3382
3383        let client = Arc::new(
3384            WebSocketClient::connect(
3385                config,
3386                Some(handler),
3387                None,
3388                None,
3389                vec![("test_key".to_string(), quota)],
3390                None,
3391            )
3392            .await
3393            .unwrap(),
3394        );
3395
3396        // Wait for disconnection
3397        wait_until_async(
3398            || async { client.is_reconnecting() || client.is_closed() },
3399            Duration::from_secs(2),
3400        )
3401        .await;
3402
3403        // Explicitly disconnect to move away from ACTIVE state
3404        client.disconnect().await;
3405        assert!(
3406            !client.is_active(),
3407            "Client should not be active after disconnect"
3408        );
3409
3410        // Attempt send - should fail IMMEDIATELY without waiting for rate limit
3411        let start = std::time::Instant::now();
3412        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
3413        let result = client
3414            .send_text("test".to_string(), Some(test_key.as_slice()))
3415            .await;
3416        let elapsed = start.elapsed();
3417
3418        // Should fail with Closed error
3419        assert!(result.is_err(), "Send should fail when client is closed");
3420        assert!(
3421            matches!(result, Err(crate::error::SendError::Closed)),
3422            "Send should return Closed error, was: {result:?}"
3423        );
3424
3425        // Should fail FAST (< 100ms) without waiting for rate limit (10s)
3426        assert!(
3427            elapsed < Duration::from_millis(100),
3428            "Send should fail fast without rate limiting, took {elapsed:?}"
3429        );
3430
3431        server.abort();
3432    }
3433
3434    #[rstest]
3435    #[tokio::test]
3436    async fn test_connect_rejects_none_message_handler() {
3437        // Test that connect() properly rejects None message_handler
3438        // to prevent zombie connections that appear alive but never detect disconnections
3439
3440        let config = WebSocketConfig {
3441            url: "ws://127.0.0.1:9999".to_string(),
3442            headers: vec![],
3443            heartbeat: None,
3444            heartbeat_msg: None,
3445            reconnect_timeout_ms: Some(1_000),
3446            reconnect_delay_initial_ms: Some(100),
3447            reconnect_delay_max_ms: Some(500),
3448            reconnect_backoff_factor: Some(1.5),
3449            reconnect_jitter_ms: Some(0),
3450            reconnect_max_attempts: None,
3451            idle_timeout_ms: None,
3452            backend: TransportBackend::Tungstenite,
3453            proxy_url: None,
3454        };
3455
3456        // Pass None for message_handler - should be rejected
3457        let result = WebSocketClient::connect(config, None, None, None, vec![], None).await;
3458
3459        assert!(
3460            result.is_err(),
3461            "connect() should reject None message_handler"
3462        );
3463
3464        let err = result.unwrap_err();
3465        let err_msg = err.to_string();
3466        assert!(
3467            err_msg.contains("Handler mode requires message_handler"),
3468            "Error should mention missing message_handler, was: {err_msg}"
3469        );
3470    }
3471
3472    #[rstest]
3473    #[tokio::test]
3474    async fn test_connect_url_rejects_invalid_reconnect_timing_before_connect() {
3475        let (handler, _rx) = channel_message_handler();
3476
3477        let config = WebSocketConfig {
3478            url: "ws://127.0.0.1:1".to_string(),
3479            headers: vec![],
3480            heartbeat: None,
3481            heartbeat_msg: None,
3482            reconnect_timeout_ms: Some(0),
3483            reconnect_delay_initial_ms: Some(100),
3484            reconnect_delay_max_ms: Some(500),
3485            reconnect_backoff_factor: Some(1.5),
3486            reconnect_jitter_ms: Some(0),
3487            reconnect_max_attempts: None,
3488            idle_timeout_ms: None,
3489            backend: TransportBackend::Tungstenite,
3490            proxy_url: None,
3491        };
3492
3493        let err = WebSocketClientInner::connect_url(config, Some(handler), None)
3494            .await
3495            .expect_err("invalid reconnect timing should be rejected");
3496
3497        match err {
3498            TransportError::Io(error) => {
3499                assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
3500                assert!(
3501                    error
3502                        .to_string()
3503                        .contains("Reconnect timeout cannot be zero"),
3504                    "error should mention zero reconnect timeout, was: {error}"
3505                );
3506            }
3507            other => panic!("expected InvalidInput IO error, was: {other:?}"),
3508        }
3509    }
3510
3511    #[rstest]
3512    #[tokio::test]
3513    async fn test_client_without_handler_sets_stream_mode() {
3514        // Test that if a client is created without a handler via connect_url,
3515        // it properly sets is_stream_mode=true to prevent zombie connections
3516
3517        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3518        let port = listener.local_addr().unwrap().port();
3519
3520        let server = task::spawn(async move {
3521            // Accept and immediately close to simulate server disconnect
3522            if let Ok((stream, _)) = listener.accept().await
3523                && let Ok(ws) = accept_async(stream).await
3524            {
3525                drop(ws); // Drop connection immediately
3526            }
3527        });
3528
3529        let config = WebSocketConfig {
3530            url: format!("ws://127.0.0.1:{port}"),
3531            headers: vec![],
3532            heartbeat: None,
3533            heartbeat_msg: None,
3534            reconnect_timeout_ms: Some(1_000),
3535            reconnect_delay_initial_ms: Some(100),
3536            reconnect_delay_max_ms: Some(500),
3537            reconnect_backoff_factor: Some(1.5),
3538            reconnect_jitter_ms: Some(0),
3539            reconnect_max_attempts: None,
3540            idle_timeout_ms: None,
3541            backend: TransportBackend::Tungstenite,
3542            proxy_url: None,
3543        };
3544
3545        // Create client directly via connect_url with no handler (stream mode)
3546        let inner = WebSocketClientInner::connect_url(config, None, None)
3547            .await
3548            .unwrap();
3549
3550        // Verify is_stream_mode is true when no handler
3551        assert!(
3552            inner.is_stream_mode,
3553            "Client without handler should have is_stream_mode=true"
3554        );
3555
3556        // Verify that when stream mode is enabled, reconnection is disabled
3557        // (documented behavior - stream mode clients close instead of reconnecting)
3558
3559        server.abort();
3560    }
3561
3562    #[rstest]
3563    #[tokio::test]
3564    async fn test_idle_timeout_triggers_reconnect() {
3565        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3566        let port = listener.local_addr().unwrap().port();
3567
3568        // Server accepts WS connection but sends nothing (simulates silent death)
3569        let server = task::spawn(async move {
3570            let (stream, _) = listener.accept().await.unwrap();
3571            let _ws = accept_async(stream).await.unwrap();
3572            // Hold connection open but send nothing
3573            sleep(Duration::from_secs(5)).await;
3574        });
3575
3576        let (handler, _rx) = channel_message_handler();
3577
3578        let config = WebSocketConfig {
3579            url: format!("ws://127.0.0.1:{port}"),
3580            headers: vec![],
3581            heartbeat: None,
3582            heartbeat_msg: None,
3583            reconnect_timeout_ms: Some(2_000),
3584            reconnect_delay_initial_ms: Some(50),
3585            reconnect_delay_max_ms: Some(100),
3586            reconnect_backoff_factor: Some(1.0),
3587            reconnect_jitter_ms: Some(0),
3588            reconnect_max_attempts: Some(1),
3589            idle_timeout_ms: Some(500),
3590            backend: TransportBackend::Tungstenite,
3591            proxy_url: None,
3592        };
3593
3594        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3595            .await
3596            .unwrap();
3597
3598        assert!(client.is_active());
3599
3600        // Wait for idle timeout to fire and client to enter reconnect/closed
3601        wait_until_async(
3602            || async { client.is_reconnecting() || client.is_disconnected() },
3603            Duration::from_secs(3),
3604        )
3605        .await;
3606
3607        assert!(
3608            !client.is_active(),
3609            "Client should not be active after idle timeout"
3610        );
3611
3612        client.disconnect().await;
3613        server.abort();
3614    }
3615
3616    #[rstest]
3617    #[tokio::test]
3618    async fn test_idle_timeout_resets_on_data() {
3619        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3620        let port = listener.local_addr().unwrap().port();
3621
3622        // Server sends a message every 200ms (well within 1s idle timeout)
3623        let server = task::spawn(async move {
3624            let (stream, _) = listener.accept().await.unwrap();
3625            let mut ws = accept_async(stream).await.unwrap();
3626
3627            for _ in 0..10 {
3628                sleep(Duration::from_millis(200)).await;
3629
3630                if ws.send(WsMessage::Text("ping".into())).await.is_err() {
3631                    break;
3632                }
3633            }
3634        });
3635
3636        let (handler, _rx) = channel_message_handler();
3637
3638        let config = WebSocketConfig {
3639            url: format!("ws://127.0.0.1:{port}"),
3640            headers: vec![],
3641            heartbeat: None,
3642            heartbeat_msg: None,
3643            reconnect_timeout_ms: Some(2_000),
3644            reconnect_delay_initial_ms: Some(50),
3645            reconnect_delay_max_ms: Some(100),
3646            reconnect_backoff_factor: Some(1.0),
3647            reconnect_jitter_ms: Some(0),
3648            reconnect_max_attempts: Some(1),
3649            idle_timeout_ms: Some(1_000),
3650            backend: TransportBackend::Tungstenite,
3651            proxy_url: None,
3652        };
3653
3654        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3655            .await
3656            .unwrap();
3657
3658        assert!(client.is_active());
3659
3660        // Wait 1.5s - data arrives every 200ms so idle timeout (1s) should NOT fire
3661        sleep(Duration::from_millis(1_500)).await;
3662
3663        assert!(
3664            client.is_active(),
3665            "Client should remain active when data is flowing"
3666        );
3667
3668        client.disconnect().await;
3669        server.abort();
3670    }
3671
3672    #[rstest]
3673    #[tokio::test]
3674    async fn test_idle_timeout_fires_when_only_pings_received() {
3675        // Regression: pings and pongs are keep-alive frames, not application data,
3676        // so a peer that only emits control frames must still trip the idle timeout.
3677        // The peer keeps pinging for well past the observation window so the
3678        // pre-fix behavior (reset-on-ping) would keep the client active; under the
3679        // fix the idle timer never resets and fires after ~500ms.
3680        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3681        let port = listener.local_addr().unwrap().port();
3682
3683        let server = task::spawn(async move {
3684            let (stream, _) = listener.accept().await.unwrap();
3685            let mut ws = accept_async(stream).await.unwrap();
3686
3687            for _ in 0..60 {
3688                sleep(Duration::from_millis(100)).await;
3689
3690                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
3691                    break;
3692                }
3693            }
3694        });
3695
3696        let (handler, _rx) = channel_message_handler();
3697
3698        let config = WebSocketConfig {
3699            url: format!("ws://127.0.0.1:{port}"),
3700            headers: vec![],
3701            heartbeat: None,
3702            heartbeat_msg: None,
3703            reconnect_timeout_ms: Some(2_000),
3704            reconnect_delay_initial_ms: Some(50),
3705            reconnect_delay_max_ms: Some(100),
3706            reconnect_backoff_factor: Some(1.0),
3707            reconnect_jitter_ms: Some(0),
3708            reconnect_max_attempts: Some(1),
3709            idle_timeout_ms: Some(500),
3710            backend: TransportBackend::Tungstenite,
3711            proxy_url: None,
3712        };
3713
3714        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3715            .await
3716            .unwrap();
3717
3718        assert!(client.is_active());
3719
3720        // Observation window is shorter than the ping stream (6s). If the idle
3721        // timer mistakenly reset on every ping the client would still be active
3722        // here; under the fix it goes inactive at ~500ms.
3723        wait_until_async(
3724            || async { client.is_reconnecting() || client.is_disconnected() },
3725            Duration::from_millis(1_500),
3726        )
3727        .await;
3728
3729        assert!(
3730            !client.is_active(),
3731            "Client should not be active after idle timeout when only pings/pongs flow"
3732        );
3733
3734        client.disconnect().await;
3735        server.abort();
3736    }
3737
3738    #[rstest]
3739    #[tokio::test]
3740    async fn test_idle_timeout_fires_when_only_pongs_received() {
3741        // Regression for the heartbeat-reply path. When the client heartbeat is
3742        // enabled, the peer auto-replies with pongs for every outgoing ping. If
3743        // those pongs refreshed last_data_time the idle timer would never fire on
3744        // a zombie connection (the motivating Polymarket scenario).
3745        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3746        let port = listener.local_addr().unwrap().port();
3747
3748        let server = task::spawn(async move {
3749            let (stream, _) = listener.accept().await.unwrap();
3750            let mut ws = accept_async(stream).await.unwrap();
3751
3752            // Drain incoming frames so tungstenite's internal pong replies are
3753            // actually flushed to the client. Hold the connection open well past
3754            // the observation window.
3755            let deadline = tokio::time::Instant::now() + Duration::from_secs(6);
3756            while tokio::time::Instant::now() < deadline {
3757                if let Ok(Some(Err(_)) | None) =
3758                    tokio::time::timeout(Duration::from_millis(100), ws.next()).await
3759                {
3760                    break;
3761                }
3762            }
3763        });
3764
3765        let (handler, _rx) = channel_message_handler();
3766
3767        let config = WebSocketConfig {
3768            url: format!("ws://127.0.0.1:{port}"),
3769            headers: vec![],
3770            heartbeat: Some(1),
3771            heartbeat_msg: None,
3772            reconnect_timeout_ms: Some(2_000),
3773            reconnect_delay_initial_ms: Some(50),
3774            reconnect_delay_max_ms: Some(100),
3775            reconnect_backoff_factor: Some(1.0),
3776            reconnect_jitter_ms: Some(0),
3777            reconnect_max_attempts: Some(1),
3778            idle_timeout_ms: Some(1_500),
3779            backend: TransportBackend::Tungstenite,
3780            proxy_url: None,
3781        };
3782
3783        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3784            .await
3785            .unwrap();
3786
3787        assert!(client.is_active());
3788
3789        // Heartbeat cadence is 1s; each ping draws a pong reply. Under the fix
3790        // the idle timer ignores those pongs and fires at ~1.5s. Under the bug
3791        // every pong reset the timer and the client would stay active.
3792        wait_until_async(
3793            || async { client.is_reconnecting() || client.is_disconnected() },
3794            Duration::from_millis(2_500),
3795        )
3796        .await;
3797
3798        assert!(
3799            !client.is_active(),
3800            "Client should not be active after idle timeout when only pongs flow"
3801        );
3802
3803        client.disconnect().await;
3804        server.abort();
3805    }
3806
3807    #[rstest]
3808    #[tokio::test]
3809    async fn test_disconnect_during_backoff_exits_promptly() {
3810        // Verify that disconnect interrupts backoff sleep (Finding 1).
3811        // Server accepts then drops, no second listener -> reconnect fails -> enters backoff.
3812        // We disconnect while backing off and assert the client shuts down quickly.
3813        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3814        let port = listener.local_addr().unwrap().port();
3815
3816        let server = task::spawn(async move {
3817            // Accept first connection, close immediately
3818            if let Ok((stream, _)) = listener.accept().await {
3819                let _ = accept_async(stream).await;
3820            }
3821            // Don't accept again so reconnect fails and enters backoff
3822            sleep(Duration::from_mins(1)).await;
3823        });
3824
3825        let (handler, _rx) = channel_message_handler();
3826
3827        let config = WebSocketConfig {
3828            url: format!("ws://127.0.0.1:{port}"),
3829            headers: vec![],
3830            heartbeat: None,
3831            heartbeat_msg: None,
3832            reconnect_timeout_ms: Some(1_000),
3833            reconnect_delay_initial_ms: Some(10_000), // 10s backoff to ensure we're sleeping
3834            reconnect_delay_max_ms: Some(10_000),
3835            reconnect_backoff_factor: Some(1.0),
3836            reconnect_jitter_ms: Some(0),
3837            reconnect_max_attempts: None,
3838            idle_timeout_ms: None,
3839            backend: TransportBackend::Tungstenite,
3840            proxy_url: None,
3841        };
3842
3843        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
3844            .await
3845            .unwrap();
3846
3847        // Wait for client to enter reconnect
3848        wait_until_async(
3849            || async { client.is_reconnecting() },
3850            Duration::from_secs(3),
3851        )
3852        .await;
3853
3854        // Wait a bit more for the reconnect attempt to fail and enter backoff sleep
3855        sleep(Duration::from_millis(1_500)).await;
3856
3857        // Disconnect while backing off
3858        let start = std::time::Instant::now();
3859        client.disconnect().await;
3860        let elapsed = start.elapsed();
3861
3862        assert!(client.is_disconnected(), "Client should be disconnected");
3863        // Should exit well before the 10s backoff sleep completes
3864        assert!(
3865            elapsed < Duration::from_secs(2),
3866            "Disconnect should interrupt backoff sleep, took {elapsed:?}"
3867        );
3868
3869        server.abort();
3870    }
3871
3872    #[rstest]
3873    #[tokio::test]
3874    async fn test_rate_limit_cancelled_on_disconnect() {
3875        // Verify that a send blocked on rate limiting returns Closed when
3876        // the client disconnects (Finding 6).
3877        use std::{num::NonZeroU32, sync::Arc};
3878
3879        use crate::ratelimiter::quota::Quota;
3880
3881        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3882        let port = listener.local_addr().unwrap().port();
3883
3884        let server = task::spawn(async move {
3885            if let Ok((stream, _)) = listener.accept().await {
3886                let mut ws = accept_async(stream).await.unwrap();
3887                // Keep alive and echo
3888                while let Some(Ok(msg)) = ws.next().await {
3889                    if ws.send(msg).await.is_err() {
3890                        break;
3891                    }
3892                }
3893            }
3894        });
3895
3896        let (handler, _rx) = channel_message_handler();
3897
3898        let config = WebSocketConfig {
3899            url: format!("ws://127.0.0.1:{port}"),
3900            headers: vec![],
3901            heartbeat: None,
3902            heartbeat_msg: None,
3903            reconnect_timeout_ms: Some(5_000),
3904            reconnect_delay_initial_ms: Some(100),
3905            reconnect_delay_max_ms: Some(500),
3906            reconnect_backoff_factor: Some(1.5),
3907            reconnect_jitter_ms: Some(0),
3908            reconnect_max_attempts: None,
3909            idle_timeout_ms: None,
3910            backend: TransportBackend::Tungstenite,
3911            proxy_url: None,
3912        };
3913
3914        // Very restrictive: 1 req per 60 seconds
3915        let quota = Quota::with_period(Duration::from_mins(1))
3916            .unwrap()
3917            .allow_burst(NonZeroU32::new(1).unwrap());
3918
3919        let client = Arc::new(
3920            WebSocketClient::connect(
3921                config,
3922                Some(handler),
3923                None,
3924                None,
3925                vec![("rate_key".to_string(), quota)],
3926                None,
3927            )
3928            .await
3929            .unwrap(),
3930        );
3931
3932        let test_key: [Ustr; 1] = [Ustr::from("rate_key")];
3933
3934        // Exhaust the burst quota
3935        client
3936            .send_text("exhaust".to_string(), Some(test_key.as_slice()))
3937            .await
3938            .unwrap();
3939
3940        // Spawn a send that will block on rate limiter
3941        let client_clone = client.clone();
3942        let send_handle = task::spawn(async move {
3943            client_clone
3944                .send_text("blocked".to_string(), Some(&[Ustr::from("rate_key")]))
3945                .await
3946        });
3947
3948        // Let the send block on rate limit
3949        sleep(Duration::from_millis(200)).await;
3950
3951        // Disconnect while send is blocked
3952        let start = std::time::Instant::now();
3953        client.disconnect().await;
3954        let elapsed_disconnect = start.elapsed();
3955
3956        // The blocked send should return Closed
3957        let result = tokio::time::timeout(Duration::from_secs(2), send_handle)
3958            .await
3959            .expect("Send task should complete quickly")
3960            .expect("Send task should not panic");
3961
3962        assert!(
3963            matches!(result, Err(crate::error::SendError::Closed)),
3964            "Blocked send should return Closed, was: {result:?}"
3965        );
3966
3967        // Disconnect should be fast, not waiting for the 60s rate limit
3968        assert!(
3969            elapsed_disconnect < Duration::from_secs(3),
3970            "Disconnect should not wait for rate limiter, took {elapsed_disconnect:?}"
3971        );
3972
3973        server.abort();
3974    }
3975
3976    #[rstest]
3977    #[tokio::test]
3978    async fn test_stream_mode_transitions_to_closed_on_dead_write_task() {
3979        // Verify that stream mode transitions to CLOSED (not RECONNECT) when
3980        // the write task dies (Finding 4). We force write failure by sending
3981        // after the server closes the connection.
3982        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3983        let port = listener.local_addr().unwrap().port();
3984
3985        let server = task::spawn(async move {
3986            if let Ok((stream, _)) = listener.accept().await
3987                && let Ok(ws) = accept_async(stream).await
3988            {
3989                // Close immediately to cause write errors
3990                drop(ws);
3991            }
3992        });
3993
3994        let config = WebSocketConfig {
3995            url: format!("ws://127.0.0.1:{port}"),
3996            headers: vec![],
3997            heartbeat: None,
3998            heartbeat_msg: None,
3999            reconnect_timeout_ms: Some(1_000),
4000            reconnect_delay_initial_ms: Some(50),
4001            reconnect_delay_max_ms: Some(100),
4002            reconnect_backoff_factor: Some(1.0),
4003            reconnect_jitter_ms: Some(0),
4004            reconnect_max_attempts: None,
4005            idle_timeout_ms: None,
4006            backend: TransportBackend::Tungstenite,
4007            proxy_url: None,
4008        };
4009
4010        let (_reader, client) = WebSocketClient::connect_stream(config, vec![], None, None)
4011            .await
4012            .unwrap();
4013
4014        assert!(client.is_active(), "Client should start active");
4015
4016        // Wait for server to close, then send to trigger write task failure
4017        sleep(Duration::from_millis(100)).await;
4018
4019        // Keep sending until the write task detects the broken connection
4020        for _ in 0..20 {
4021            let _ = client.send_text("ping".to_string(), None).await;
4022            sleep(Duration::from_millis(50)).await;
4023
4024            if !client.is_active() {
4025                break;
4026            }
4027        }
4028
4029        // Wait for controller to process the state change
4030        wait_until_async(|| async { !client.is_active() }, Duration::from_secs(5)).await;
4031
4032        // Stream mode should go to CLOSED, not RECONNECT
4033        assert!(
4034            client.is_closed() || client.is_disconnected(),
4035            "Stream mode should transition to CLOSED, not RECONNECT. \
4036             is_reconnecting={}, is_closed={}, is_disconnected={}",
4037            client.is_reconnecting(),
4038            client.is_closed(),
4039            client.is_disconnected(),
4040        );
4041        assert!(
4042            !client.is_reconnecting(),
4043            "Stream mode should never attempt reconnection"
4044        );
4045
4046        server.abort();
4047    }
4048
4049    #[derive(Default)]
4050    struct BlockingFailState {
4051        send_entered: AtomicBool,
4052        fail: AtomicBool,
4053        waker: std::sync::Mutex<Option<std::task::Waker>>,
4054    }
4055
4056    impl BlockingFailState {
4057        fn trigger_failure(&self) {
4058            self.fail.store(true, Ordering::SeqCst);
4059
4060            if let Some(waker) = self.waker.lock().unwrap().take() {
4061                waker.wake();
4062            }
4063        }
4064    }
4065
4066    /// Transport whose sends block until [`BlockingFailState::trigger_failure`],
4067    /// then fail. Used to interleave a disconnect with an in-flight send.
4068    struct BlockingFailTransport {
4069        state: Arc<BlockingFailState>,
4070    }
4071
4072    impl futures_util::Stream for BlockingFailTransport {
4073        type Item = Result<Message, TransportError>;
4074
4075        fn poll_next(
4076            self: std::pin::Pin<&mut Self>,
4077            _cx: &mut std::task::Context<'_>,
4078        ) -> std::task::Poll<Option<Self::Item>> {
4079            std::task::Poll::Pending
4080        }
4081    }
4082
4083    impl futures_util::Sink<Message> for BlockingFailTransport {
4084        type Error = TransportError;
4085
4086        fn poll_ready(
4087            self: std::pin::Pin<&mut Self>,
4088            _cx: &mut std::task::Context<'_>,
4089        ) -> std::task::Poll<Result<(), Self::Error>> {
4090            std::task::Poll::Ready(Ok(()))
4091        }
4092
4093        fn start_send(self: std::pin::Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
4094            Ok(())
4095        }
4096
4097        fn poll_flush(
4098            self: std::pin::Pin<&mut Self>,
4099            cx: &mut std::task::Context<'_>,
4100        ) -> std::task::Poll<Result<(), Self::Error>> {
4101            // Store the waker before checking the flag so trigger_failure
4102            // cannot slip between the check and the registration
4103            *self.state.waker.lock().unwrap() = Some(cx.waker().clone());
4104            self.state.send_entered.store(true, Ordering::SeqCst);
4105
4106            if self.state.fail.load(Ordering::SeqCst) {
4107                std::task::Poll::Ready(Err(TransportError::ConnectionReset))
4108            } else {
4109                std::task::Poll::Pending
4110            }
4111        }
4112
4113        fn poll_close(
4114            self: std::pin::Pin<&mut Self>,
4115            _cx: &mut std::task::Context<'_>,
4116        ) -> std::task::Poll<Result<(), Self::Error>> {
4117            std::task::Poll::Ready(Ok(()))
4118        }
4119    }
4120
4121    #[rstest]
4122    #[tokio::test]
4123    async fn test_new_with_writer_rejects_zero_heartbeat() {
4124        // Stream mode shares connect_url's validation: a zero heartbeat would
4125        // spawn a busy-loop ping flood
4126        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
4127            state: Arc::new(BlockingFailState::default()),
4128        });
4129        let (writer, _reader) = transport.split();
4130
4131        let config = WebSocketConfig {
4132            url: "ws://127.0.0.1:1".to_string(),
4133            headers: vec![],
4134            heartbeat: Some(0),
4135            heartbeat_msg: None,
4136            reconnect_timeout_ms: None,
4137            reconnect_delay_initial_ms: None,
4138            reconnect_delay_max_ms: None,
4139            reconnect_backoff_factor: None,
4140            reconnect_jitter_ms: None,
4141            reconnect_max_attempts: None,
4142            idle_timeout_ms: None,
4143            backend: TransportBackend::Tungstenite,
4144            proxy_url: None,
4145        };
4146
4147        let err = WebSocketClientInner::new_with_writer(config, writer)
4148            .await
4149            .expect_err("zero heartbeat should be rejected in stream mode");
4150        assert!(
4151            err.to_string()
4152                .contains("Heartbeat interval cannot be zero"),
4153            "error should mention zero heartbeat, was: {err}"
4154        );
4155    }
4156
4157    #[rstest]
4158    #[tokio::test]
4159    async fn test_connect_times_out_on_silent_server() {
4160        // A server that accepts TCP but never completes the WebSocket upgrade
4161        // must not hang connect() indefinitely
4162        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4163        let port = listener.local_addr().unwrap().port();
4164
4165        let server = task::spawn(async move {
4166            // Accept and hold the socket open without responding
4167            if let Ok((_stream, _)) = listener.accept().await {
4168                sleep(Duration::from_secs(30)).await;
4169            }
4170        });
4171
4172        let (handler, _rx) = channel_message_handler();
4173
4174        let config = WebSocketConfig {
4175            url: format!("ws://127.0.0.1:{port}"),
4176            headers: vec![],
4177            heartbeat: None,
4178            heartbeat_msg: None,
4179            reconnect_timeout_ms: Some(500),
4180            reconnect_delay_initial_ms: Some(50),
4181            reconnect_delay_max_ms: Some(100),
4182            reconnect_backoff_factor: Some(1.0),
4183            reconnect_jitter_ms: Some(0),
4184            reconnect_max_attempts: None,
4185            idle_timeout_ms: None,
4186            backend: TransportBackend::Tungstenite,
4187            proxy_url: None,
4188        };
4189
4190        let result = tokio::time::timeout(
4191            Duration::from_secs(5),
4192            WebSocketClient::connect(config, Some(handler), None, None, vec![], None),
4193        )
4194        .await
4195        .expect("connect should not hang on a silent server");
4196
4197        assert!(result.is_err(), "connect should fail with a timeout error");
4198        let err_msg = result.unwrap_err().to_string();
4199        assert!(
4200            err_msg.contains("timed out"),
4201            "error should mention the timeout, was: {err_msg}"
4202        );
4203
4204        server.abort();
4205    }
4206
4207    #[rstest]
4208    #[tokio::test]
4209    async fn test_reconnect_succeeds_with_timeout_shorter_than_swap_ceremony() {
4210        // Regression: the reconnect timeout used to cover the writer swap and
4211        // graceful-shutdown delays (~200ms minimum), so a short timeout caused
4212        // every otherwise-successful reconnect to be discarded mid-swap and the
4213        // already-swapped writer to be orphaned. The timeout now bounds only
4214        // connection establishment.
4215        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4216        let port = listener.local_addr().unwrap().port();
4217
4218        let server = task::spawn(async move {
4219            // First connection: accept and immediately drop
4220            if let Ok((stream, _)) = listener.accept().await
4221                && let Ok(ws) = accept_async(stream).await
4222            {
4223                drop(ws);
4224            }
4225
4226            // Second connection: announce and keep alive
4227            if let Ok((stream, _)) = listener.accept().await
4228                && let Ok(mut ws) = accept_async(stream).await
4229            {
4230                let _ = ws
4231                    .send(WsMessage::Text("reconnected-msg".to_string().into()))
4232                    .await;
4233                sleep(Duration::from_secs(5)).await;
4234            }
4235        });
4236
4237        let (handler, mut rx) = channel_message_handler();
4238
4239        let config = WebSocketConfig {
4240            url: format!("ws://127.0.0.1:{port}"),
4241            headers: vec![],
4242            heartbeat: None,
4243            heartbeat_msg: None,
4244            reconnect_timeout_ms: Some(150), // Shorter than the ~200ms swap ceremony
4245            reconnect_delay_initial_ms: Some(25),
4246            reconnect_delay_max_ms: Some(50),
4247            reconnect_backoff_factor: Some(1.0),
4248            reconnect_jitter_ms: Some(0),
4249            reconnect_max_attempts: None,
4250            idle_timeout_ms: None,
4251            backend: TransportBackend::Tungstenite,
4252            proxy_url: None,
4253        };
4254
4255        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4256            .await
4257            .unwrap();
4258
4259        let received = tokio::time::timeout(Duration::from_secs(5), async {
4260            loop {
4261                if let Ok(WsMessage::Text(text)) = rx.try_recv()
4262                    && text.as_str() == "reconnected-msg"
4263                {
4264                    return true;
4265                }
4266                tokio::time::sleep(Duration::from_millis(10)).await;
4267            }
4268        })
4269        .await;
4270
4271        assert!(
4272            received.is_ok(),
4273            "Reconnect should complete despite a timeout shorter than the swap ceremony"
4274        );
4275
4276        client.disconnect().await;
4277        server.abort();
4278    }
4279
4280    #[rstest]
4281    #[tokio::test]
4282    async fn test_idle_timeout_fires_under_ping_flood() {
4283        // Regression: the idle check used to run only when nothing arrived for a
4284        // full check interval (10ms), so pings flooding faster than that starved
4285        // it and a ping-only zombie connection never tripped the timeout
4286        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4287        let port = listener.local_addr().unwrap().port();
4288
4289        let server = task::spawn(async move {
4290            let (stream, _) = listener.accept().await.unwrap();
4291            let mut ws = accept_async(stream).await.unwrap();
4292
4293            for _ in 0..600 {
4294                sleep(Duration::from_millis(5)).await;
4295
4296                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
4297                    break;
4298                }
4299            }
4300        });
4301
4302        let (handler, _rx) = channel_message_handler();
4303
4304        let config = WebSocketConfig {
4305            url: format!("ws://127.0.0.1:{port}"),
4306            headers: vec![],
4307            heartbeat: None,
4308            heartbeat_msg: None,
4309            reconnect_timeout_ms: Some(2_000),
4310            reconnect_delay_initial_ms: Some(50),
4311            reconnect_delay_max_ms: Some(100),
4312            reconnect_backoff_factor: Some(1.0),
4313            reconnect_jitter_ms: Some(0),
4314            reconnect_max_attempts: Some(1),
4315            idle_timeout_ms: Some(500),
4316            backend: TransportBackend::Tungstenite,
4317            proxy_url: None,
4318        };
4319
4320        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4321            .await
4322            .unwrap();
4323
4324        assert!(client.is_active());
4325
4326        wait_until_async(
4327            || async { client.is_reconnecting() || client.is_disconnected() },
4328            Duration::from_millis(1_500),
4329        )
4330        .await;
4331
4332        assert!(
4333            !client.is_active(),
4334            "Client should not be active after idle timeout under a ping flood"
4335        );
4336
4337        client.disconnect().await;
4338        server.abort();
4339    }
4340
4341    #[rstest]
4342    #[tokio::test]
4343    async fn test_send_failure_does_not_overwrite_disconnect() {
4344        let state = Arc::new(BlockingFailState::default());
4345        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
4346            state: Arc::clone(&state),
4347        });
4348        let (writer, _reader) = transport.split();
4349
4350        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
4351        let state_notify = Arc::new(tokio::sync::Notify::new());
4352        let auth_tracker = Arc::new(OnceLock::new());
4353        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
4354
4355        let (writer_tx, writer_rx) = tokio::sync::mpsc::unbounded_channel();
4356        let write_task = WebSocketClientInner::spawn_write_task(
4357            Arc::clone(&connection_state),
4358            Arc::clone(&state_notify),
4359            writer,
4360            writer_rx,
4361            Arc::clone(&auth_tracker),
4362            Arc::clone(&reconnect_buffer_waits_for_auth),
4363        );
4364
4365        writer_tx
4366            .send(WriterCommand::Send(Message::text("doomed")))
4367            .unwrap();
4368
4369        // Wait until the writer task is blocked inside the transport send
4370        wait_until_async(
4371            || {
4372                let state = Arc::clone(&state);
4373                async move { state.send_entered.load(Ordering::SeqCst) }
4374            },
4375            Duration::from_secs(2),
4376        )
4377        .await;
4378
4379        // Disconnect lands while the send is in flight, then the send fails;
4380        // the writer error path must not overwrite DISCONNECT with RECONNECT
4381        connection_state.store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
4382        state.trigger_failure();
4383
4384        tokio::time::timeout(Duration::from_secs(2), write_task)
4385            .await
4386            .expect("write task should exit after disconnect")
4387            .unwrap();
4388
4389        assert_eq!(
4390            ConnectionMode::from_atomic(&connection_state),
4391            ConnectionMode::Disconnect,
4392            "Send failure must not resurrect a disconnecting client into RECONNECT"
4393        );
4394    }
4395
4396    #[tokio::test]
4397    async fn test_write_task_waits_for_auth_before_replaying_buffer() {
4398        use nautilus_common::testing::wait_until_async;
4399
4400        let server = RecordingServer::setup().await;
4401        let url = format!("ws://127.0.0.1:{}", server.port);
4402        let (writer, _reader) = WebSocketClientInner::connect_with_server(
4403            &url,
4404            vec![],
4405            TransportBackend::Tungstenite,
4406            None,
4407        )
4408        .await
4409        .unwrap();
4410
4411        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
4412        let state_notify = Arc::new(tokio::sync::Notify::new());
4413        let auth_tracker = Arc::new(OnceLock::new());
4414        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
4415        let tracker = AuthTracker::new();
4416        auth_tracker.set(tracker.clone()).unwrap();
4417
4418        let (writer_tx, writer_rx) = tokio::sync::mpsc::unbounded_channel();
4419        let write_task = WebSocketClientInner::spawn_write_task(
4420            Arc::clone(&connection_state),
4421            Arc::clone(&state_notify),
4422            writer,
4423            writer_rx,
4424            Arc::clone(&auth_tracker),
4425            Arc::clone(&reconnect_buffer_waits_for_auth),
4426        );
4427
4428        writer_tx
4429            .send(WriterCommand::Send(Message::Text("stale".into())))
4430            .unwrap();
4431
4432        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
4433            &url,
4434            vec![],
4435            TransportBackend::Tungstenite,
4436            None,
4437        )
4438        .await
4439        .unwrap();
4440        let (tx, rx) = tokio::sync::oneshot::channel();
4441        writer_tx
4442            .send(WriterCommand::Update(new_writer, tx))
4443            .unwrap();
4444        assert!(rx.await.unwrap());
4445
4446        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
4447
4448        tokio::time::sleep(Duration::from_millis(300)).await;
4449        assert!(
4450            server.messages().await.is_empty(),
4451            "buffered messages should wait for re-authentication"
4452        );
4453
4454        tracker.succeed();
4455
4456        wait_until_async(
4457            || {
4458                let messages = Arc::clone(&server.messages);
4459                async move { !messages.lock().await.is_empty() }
4460            },
4461            Duration::from_secs(3),
4462        )
4463        .await;
4464
4465        assert_eq!(server.messages().await, vec!["stale".to_string()]);
4466
4467        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
4468        state_notify.notify_waiters();
4469        drop(writer_tx);
4470        write_task.abort();
4471    }
4472
4473    #[tokio::test]
4474    async fn test_write_task_discards_buffer_after_auth_failure() {
4475        let server = RecordingServer::setup().await;
4476        let url = format!("ws://127.0.0.1:{}", server.port);
4477        let (writer, _reader) = WebSocketClientInner::connect_with_server(
4478            &url,
4479            vec![],
4480            TransportBackend::Tungstenite,
4481            None,
4482        )
4483        .await
4484        .unwrap();
4485
4486        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
4487        let state_notify = Arc::new(tokio::sync::Notify::new());
4488        let auth_tracker = Arc::new(OnceLock::new());
4489        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
4490        let tracker = AuthTracker::new();
4491        auth_tracker.set(tracker.clone()).unwrap();
4492
4493        let (writer_tx, writer_rx) = tokio::sync::mpsc::unbounded_channel();
4494        let write_task = WebSocketClientInner::spawn_write_task(
4495            Arc::clone(&connection_state),
4496            Arc::clone(&state_notify),
4497            writer,
4498            writer_rx,
4499            Arc::clone(&auth_tracker),
4500            Arc::clone(&reconnect_buffer_waits_for_auth),
4501        );
4502
4503        writer_tx
4504            .send(WriterCommand::Send(Message::Text("stale".into())))
4505            .unwrap();
4506
4507        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
4508            &url,
4509            vec![],
4510            TransportBackend::Tungstenite,
4511            None,
4512        )
4513        .await
4514        .unwrap();
4515        let (tx, rx) = tokio::sync::oneshot::channel();
4516        writer_tx
4517            .send(WriterCommand::Update(new_writer, tx))
4518            .unwrap();
4519        assert!(rx.await.unwrap());
4520
4521        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
4522        tracker.fail("rejected");
4523        tokio::time::sleep(Duration::from_millis(300)).await;
4524        assert!(
4525            server.messages().await.is_empty(),
4526            "buffered messages should be discarded after authentication failure"
4527        );
4528
4529        let _auth_receiver = tracker.begin();
4530        tracker.succeed();
4531        tokio::time::sleep(Duration::from_millis(300)).await;
4532        assert!(
4533            server.messages().await.is_empty(),
4534            "discarded buffered messages should not replay on a later auth success"
4535        );
4536
4537        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
4538        state_notify.notify_waiters();
4539        drop(writer_tx);
4540        write_task.abort();
4541    }
4542
4543    #[rstest]
4544    #[tokio::test]
4545    async fn test_zero_idle_timeout_rejected() {
4546        let (handler, _rx) = channel_message_handler();
4547
4548        let config = WebSocketConfig {
4549            url: "ws://127.0.0.1:9999".to_string(),
4550            headers: vec![],
4551            heartbeat: None,
4552            heartbeat_msg: None,
4553            reconnect_timeout_ms: None,
4554            reconnect_delay_initial_ms: None,
4555            reconnect_delay_max_ms: None,
4556            reconnect_backoff_factor: None,
4557            reconnect_jitter_ms: None,
4558            reconnect_max_attempts: None,
4559            idle_timeout_ms: Some(0),
4560            backend: TransportBackend::Tungstenite,
4561            proxy_url: None,
4562        };
4563
4564        let result =
4565            WebSocketClient::connect(config, Some(handler), None, None, vec![], None).await;
4566
4567        assert!(result.is_err(), "Zero idle timeout should be rejected");
4568        let err_msg = result.unwrap_err().to_string();
4569        assert!(
4570            err_msg.contains("Idle timeout cannot be zero"),
4571            "Error should mention zero idle timeout, was: {err_msg}"
4572        );
4573    }
4574
4575    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4576    #[rstest]
4577    #[tokio::test]
4578    async fn test_sockudo_backend_rejects_reserved_headers_before_connect() {
4579        let (handler, _rx) = channel_message_handler();
4580
4581        let config = WebSocketConfig {
4582            url: "ws://127.0.0.1:1".to_string(),
4583            headers: vec![("Host".to_string(), "example.com".to_string())],
4584            heartbeat: None,
4585            heartbeat_msg: None,
4586            reconnect_timeout_ms: None,
4587            reconnect_delay_initial_ms: None,
4588            reconnect_delay_max_ms: None,
4589            reconnect_backoff_factor: None,
4590            reconnect_jitter_ms: None,
4591            reconnect_max_attempts: None,
4592            idle_timeout_ms: None,
4593            backend: TransportBackend::Sockudo,
4594            proxy_url: None,
4595        };
4596
4597        let err = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4598            .await
4599            .expect_err("reserved header should fail before TCP connect");
4600
4601        assert!(
4602            err.to_string()
4603                .contains("reserved upgrade header not allowed in extra_headers"),
4604            "expected reserved-header failure, was: {err}"
4605        );
4606    }
4607
4608    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4609    #[rstest]
4610    #[tokio::test]
4611    async fn test_sockudo_backend_replays_leftover_without_custom_headers() {
4612        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4613        let port = listener.local_addr().unwrap().port();
4614
4615        let server = task::spawn(async move {
4616            if let Ok((mut stream, _)) = listener.accept().await {
4617                let request = read_http_request(&mut stream).await;
4618                let request = String::from_utf8(request).unwrap();
4619                let sec_websocket_key = extract_header(&request, "Sec-WebSocket-Key").unwrap();
4620                let accept = sockudo_handshake::generate_accept_key(sec_websocket_key);
4621                let mut response = format!(
4622                    concat!(
4623                        "HTTP/1.1 101 Switching Protocols\r\n",
4624                        "Upgrade: websocket\r\n",
4625                        "Connection: Upgrade\r\n",
4626                        "Sec-WebSocket-Accept: {}\r\n",
4627                        "\r\n",
4628                    ),
4629                    accept
4630                )
4631                .into_bytes();
4632                response.extend_from_slice(b"\x81\x05hello");
4633                stream.write_all(&response).await.unwrap();
4634            }
4635        });
4636
4637        let (handler, mut rx) = channel_message_handler();
4638
4639        let config = WebSocketConfig {
4640            url: format!("ws://127.0.0.1:{port}/ws"),
4641            headers: vec![],
4642            heartbeat: None,
4643            heartbeat_msg: None,
4644            reconnect_timeout_ms: Some(2_000),
4645            reconnect_delay_initial_ms: Some(50),
4646            reconnect_delay_max_ms: Some(100),
4647            reconnect_backoff_factor: Some(1.0),
4648            reconnect_jitter_ms: Some(0),
4649            reconnect_max_attempts: None,
4650            idle_timeout_ms: None,
4651            backend: TransportBackend::Sockudo,
4652            proxy_url: None,
4653        };
4654
4655        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4656            .await
4657            .expect("sockudo connect without custom headers");
4658
4659        let received = tokio::time::timeout(Duration::from_secs(3), async {
4660            loop {
4661                if let Ok(msg) = rx.try_recv() {
4662                    return msg;
4663                }
4664                tokio::time::sleep(Duration::from_millis(10)).await;
4665            }
4666        })
4667        .await
4668        .expect("did not receive leftover frame before timeout");
4669
4670        match received {
4671            WsMessage::Text(t) => assert_eq!(t.as_str(), "hello"),
4672            other => panic!("expected text, was {other:?}"),
4673        }
4674
4675        client.disconnect().await;
4676        tokio::time::timeout(Duration::from_secs(3), server)
4677            .await
4678            .expect("server did not close before timeout")
4679            .unwrap();
4680    }
4681
4682    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4683    #[rstest]
4684    #[tokio::test]
4685    async fn test_sockudo_backend_sends_custom_headers() {
4686        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4687        let port = listener.local_addr().unwrap().port();
4688
4689        let server = task::spawn(async move {
4690            if let Ok((stream, _)) = listener.accept().await {
4691                let callback = HeaderAssertCallback {
4692                    key: "X-Test".to_string(),
4693                    value: HeaderValue::from_static("value"),
4694                };
4695
4696                if let Ok(mut ws) = accept_hdr_async(stream, callback).await {
4697                    while let Some(Ok(msg)) = ws.next().await {
4698                        if msg.is_text() || msg.is_binary() {
4699                            if ws.send(msg).await.is_err() {
4700                                break;
4701                            }
4702
4703                            continue;
4704                        }
4705
4706                        if msg.is_close() {
4707                            let _ = ws.close(None).await;
4708                            break;
4709                        }
4710                    }
4711                }
4712            }
4713        });
4714
4715        let (handler, mut rx) = channel_message_handler();
4716
4717        let config = WebSocketConfig {
4718            url: format!("ws://127.0.0.1:{port}"),
4719            headers: vec![("X-Test".to_string(), "value".to_string())],
4720            heartbeat: None,
4721            heartbeat_msg: None,
4722            reconnect_timeout_ms: Some(2_000),
4723            reconnect_delay_initial_ms: Some(50),
4724            reconnect_delay_max_ms: Some(100),
4725            reconnect_backoff_factor: Some(1.0),
4726            reconnect_jitter_ms: Some(0),
4727            reconnect_max_attempts: None,
4728            idle_timeout_ms: None,
4729            backend: TransportBackend::Sockudo,
4730            proxy_url: None,
4731        };
4732
4733        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4734            .await
4735            .expect("sockudo connect with custom headers");
4736
4737        client.send_text("ping".to_string(), None).await.unwrap();
4738
4739        let received = tokio::time::timeout(Duration::from_secs(3), async {
4740            loop {
4741                if let Ok(msg) = rx.try_recv() {
4742                    return msg;
4743                }
4744                tokio::time::sleep(Duration::from_millis(10)).await;
4745            }
4746        })
4747        .await
4748        .expect("did not receive echo before timeout");
4749
4750        match received {
4751            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
4752            other => panic!("expected text, was {other:?}"),
4753        }
4754
4755        client.disconnect().await;
4756        tokio::time::timeout(Duration::from_secs(3), server)
4757            .await
4758            .expect("server did not close before timeout")
4759            .unwrap();
4760    }
4761
4762    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4763    #[rstest]
4764    #[tokio::test]
4765    async fn test_sockudo_backend_round_trip_text() {
4766        // tokio-tungstenite test peer paired with a sockudo client.
4767        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4768        let port = listener.local_addr().unwrap().port();
4769
4770        let server = task::spawn(async move {
4771            if let Ok((stream, _)) = listener.accept().await
4772                && let Ok(mut ws) = accept_async(stream).await
4773            {
4774                while let Some(Ok(msg)) = ws.next().await {
4775                    match msg {
4776                        WsMessage::Text(_) | WsMessage::Binary(_) => {
4777                            if ws.send(msg).await.is_err() {
4778                                break;
4779                            }
4780                        }
4781                        WsMessage::Close(_) => {
4782                            let _ = ws.close(None).await;
4783                            break;
4784                        }
4785                        _ => {}
4786                    }
4787                }
4788            }
4789        });
4790
4791        let (handler, mut rx) = channel_message_handler();
4792        let config = WebSocketConfig {
4793            url: format!("ws://127.0.0.1:{port}"),
4794            headers: vec![],
4795            heartbeat: None,
4796            heartbeat_msg: None,
4797            reconnect_timeout_ms: Some(2_000),
4798            reconnect_delay_initial_ms: Some(50),
4799            reconnect_delay_max_ms: Some(100),
4800            reconnect_backoff_factor: Some(1.0),
4801            reconnect_jitter_ms: Some(0),
4802            reconnect_max_attempts: None,
4803            idle_timeout_ms: None,
4804            backend: TransportBackend::Sockudo,
4805            proxy_url: None,
4806        };
4807
4808        let client = WebSocketClient::connect(config, Some(handler), None, None, vec![], None)
4809            .await
4810            .expect("sockudo connect");
4811
4812        client.send_text("ping".to_string(), None).await.unwrap();
4813
4814        let received = tokio::time::timeout(Duration::from_secs(3), async {
4815            loop {
4816                if let Ok(msg) = rx.try_recv() {
4817                    return msg;
4818                }
4819                tokio::time::sleep(Duration::from_millis(10)).await;
4820            }
4821        })
4822        .await
4823        .expect("did not receive echo before timeout");
4824
4825        match received {
4826            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
4827            other => panic!("expected text, was {other:?}"),
4828        }
4829
4830        client.disconnect().await;
4831        server.abort();
4832    }
4833
4834    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4835    #[rstest]
4836    #[case::ws_default_port("ws://example.com/ws", "example.com", "example.com", 80, "/ws", false)]
4837    #[case::wss_default_port(
4838        "wss://example.com/ws",
4839        "example.com",
4840        "example.com",
4841        443,
4842        "/ws",
4843        true
4844    )]
4845    // url::Url normalises explicit default ports (`:80` for ws, `:443` for wss)
4846    // away, so `parsed.port()` reports `None` here and Host stays unqualified.
4847    #[case::ws_explicit_default(
4848        "ws://example.com:80/ws",
4849        "example.com",
4850        "example.com",
4851        80,
4852        "/ws",
4853        false
4854    )]
4855    #[case::ws_non_default(
4856        "ws://example.com:8443/feed",
4857        "example.com",
4858        "example.com:8443",
4859        8443,
4860        "/feed",
4861        false
4862    )]
4863    #[case::wss_non_default(
4864        "wss://example.com:9443/feed",
4865        "example.com",
4866        "example.com:9443",
4867        9443,
4868        "/feed",
4869        true
4870    )]
4871    #[case::root_path(
4872        "ws://example.com:9000/",
4873        "example.com",
4874        "example.com:9000",
4875        9000,
4876        "/",
4877        false
4878    )]
4879    #[case::query_string(
4880        "ws://example.com/feed?token=abc&channel=trades",
4881        "example.com",
4882        "example.com",
4883        80,
4884        "/feed?token=abc&channel=trades",
4885        false
4886    )]
4887    // IPv6: bare host strips brackets for DNS/TCP/SNI; Host header keeps them.
4888    #[case::ipv6_default("ws://[::1]/feed", "::1", "[::1]", 80, "/feed", false)]
4889    #[case::ipv6_explicit_port("ws://[::1]:9000/feed", "::1", "[::1]:9000", 9000, "/feed", false)]
4890    #[case::ipv6_wss(
4891        "wss://[2001:db8::1]:8443/",
4892        "2001:db8::1",
4893        "[2001:db8::1]:8443",
4894        8443,
4895        "/",
4896        true
4897    )]
4898    fn sockudo_target_parses_url(
4899        #[case] url: &str,
4900        #[case] host: &str,
4901        #[case] host_header: &str,
4902        #[case] port: u16,
4903        #[case] path: &str,
4904        #[case] is_tls: bool,
4905    ) {
4906        let target = super::SockudoTarget::parse(url).expect("parse should succeed");
4907        assert_eq!(target.host, host);
4908        assert_eq!(target.host_header, host_header);
4909        assert_eq!(target.port, port);
4910        assert_eq!(target.path, path);
4911        assert_eq!(target.is_tls, is_tls);
4912    }
4913
4914    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4915    #[rstest]
4916    fn sockudo_target_rejects_unsupported_scheme() {
4917        let err = super::SockudoTarget::parse("http://example.com/feed").expect_err("not a ws URL");
4918        let msg = err.to_string();
4919        assert!(
4920            msg.contains("expected ws:// or wss://"),
4921            "unexpected error: {msg}"
4922        );
4923    }
4924
4925    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
4926    #[rstest]
4927    fn sockudo_target_rejects_malformed_url() {
4928        let err = super::SockudoTarget::parse("not a url").expect_err("malformed URL");
4929        assert!(
4930            matches!(err, super::TransportError::InvalidUrl(_)),
4931            "expected InvalidUrl, was: {err:?}"
4932        );
4933    }
4934}
4935
4936#[cfg(test)]
4937mod property_tests {
4938    use std::{
4939        collections::{HashSet, VecDeque},
4940        sync::{Arc, OnceLock, atomic::AtomicBool},
4941    };
4942
4943    use proptest::prelude::*;
4944    use rstest::rstest;
4945
4946    use super::{super::auth::AuthResultReceiver, *};
4947
4948    const AUTH_FAILED: &str = "model auth failed";
4949
4950    #[derive(Debug, Clone)]
4951    enum ReconnectBufferTraceOp {
4952        BeginAuth,
4953        AuthSucceeds,
4954        AuthFails,
4955        AuthInvalidates,
4956        ReconnectStarts,
4957        ReconnectCompletes,
4958        BufferedMessage(u8),
4959    }
4960
4961    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
4962    enum ModelConnectionMode {
4963        Active,
4964        Reconnect,
4965    }
4966
4967    #[derive(Debug, Clone, Copy)]
4968    enum ExpectedReconnectBufferAction {
4969        Drain,
4970        Wait,
4971        Discard,
4972    }
4973
4974    #[derive(Debug)]
4975    struct ReconnectBufferModel {
4976        mode: ModelConnectionMode,
4977        auth_state: AuthState,
4978        buffer: VecDeque<String>,
4979        released: Vec<String>,
4980        discarded: Vec<String>,
4981        live_sent: Vec<String>,
4982        handler_controls: Vec<&'static str>,
4983        next_message_index: usize,
4984    }
4985
4986    impl ReconnectBufferModel {
4987        fn new() -> Self {
4988            Self {
4989                mode: ModelConnectionMode::Active,
4990                auth_state: AuthState::Unauthenticated,
4991                buffer: VecDeque::new(),
4992                released: Vec::new(),
4993                discarded: Vec::new(),
4994                live_sent: Vec::new(),
4995                handler_controls: Vec::new(),
4996                next_message_index: 0,
4997            }
4998        }
4999
5000        fn next_payload(&mut self, raw: u8) -> String {
5001            let payload = format!("message-{}-{raw}", self.next_message_index);
5002            self.next_message_index += 1;
5003            payload
5004        }
5005
5006        fn expected_action(&self, waits_for_auth: bool) -> ExpectedReconnectBufferAction {
5007            if !waits_for_auth {
5008                return ExpectedReconnectBufferAction::Drain;
5009            }
5010
5011            match self.auth_state {
5012                AuthState::Authenticated => ExpectedReconnectBufferAction::Drain,
5013                AuthState::Failed => ExpectedReconnectBufferAction::Discard,
5014                AuthState::Unauthenticated => ExpectedReconnectBufferAction::Wait,
5015            }
5016        }
5017    }
5018
5019    fn reconnect_buffer_trace_op_strategy() -> impl Strategy<Value = ReconnectBufferTraceOp> {
5020        prop_oneof![
5021            Just(ReconnectBufferTraceOp::BeginAuth),
5022            Just(ReconnectBufferTraceOp::AuthSucceeds),
5023            Just(ReconnectBufferTraceOp::AuthFails),
5024            Just(ReconnectBufferTraceOp::AuthInvalidates),
5025            Just(ReconnectBufferTraceOp::ReconnectStarts),
5026            Just(ReconnectBufferTraceOp::ReconnectCompletes),
5027            any::<u8>().prop_map(ReconnectBufferTraceOp::BufferedMessage),
5028        ]
5029    }
5030
5031    fn reconnect_buffer_actions_match(
5032        actual: ReconnectBufferAction,
5033        expected: ExpectedReconnectBufferAction,
5034    ) -> bool {
5035        matches!(
5036            (actual, expected),
5037            (
5038                ReconnectBufferAction::Drain,
5039                ExpectedReconnectBufferAction::Drain
5040            ) | (
5041                ReconnectBufferAction::Wait,
5042                ExpectedReconnectBufferAction::Wait
5043            ) | (
5044                ReconnectBufferAction::Discard,
5045                ExpectedReconnectBufferAction::Discard
5046            )
5047        )
5048    }
5049
5050    fn apply_ready_reconnect_buffer_action(
5051        model: &mut ReconnectBufferModel,
5052        reconnect_buffer_waits_for_auth: &AtomicBool,
5053        auth_tracker: &Arc<OnceLock<AuthTracker>>,
5054        waits_for_auth: bool,
5055        step: usize,
5056        op: &ReconnectBufferTraceOp,
5057    ) -> Result<(), TestCaseError> {
5058        if model.mode != ModelConnectionMode::Active || model.buffer.is_empty() {
5059            return Ok(());
5060        }
5061
5062        let expected = model.expected_action(waits_for_auth);
5063        let actual = WebSocketClientInner::can_drain_reconnect_buffer(
5064            reconnect_buffer_waits_for_auth,
5065            auth_tracker,
5066        );
5067
5068        prop_assert!(
5069            reconnect_buffer_actions_match(actual, expected),
5070            "reconnect buffer action mismatch at step {}, op {:?}, waits_for_auth={}, auth_state={:?}",
5071            step,
5072            op,
5073            waits_for_auth,
5074            model.auth_state
5075        );
5076
5077        match expected {
5078            ExpectedReconnectBufferAction::Drain => {
5079                model.released.extend(model.buffer.drain(..));
5080            }
5081            ExpectedReconnectBufferAction::Wait => {}
5082            ExpectedReconnectBufferAction::Discard => {
5083                model.discarded.extend(model.buffer.drain(..));
5084            }
5085        }
5086
5087        Ok(())
5088    }
5089
5090    fn assert_reconnected_control_stays_separate(
5091        model: &ReconnectBufferModel,
5092        step: usize,
5093    ) -> Result<(), TestCaseError> {
5094        prop_assert!(
5095            model
5096                .handler_controls
5097                .iter()
5098                .all(|message| *message == RECONNECTED),
5099            "handler control stream contained a non-RECONNECTED message at step {}",
5100            step
5101        );
5102        prop_assert!(
5103            !model.buffer.iter().any(|message| message == RECONNECTED),
5104            "RECONNECTED control message entered reconnect buffer at step {}",
5105            step
5106        );
5107        prop_assert!(
5108            !model.released.iter().any(|message| message == RECONNECTED),
5109            "RECONNECTED control message entered replayed messages at step {}",
5110            step
5111        );
5112        prop_assert!(
5113            !model.discarded.iter().any(|message| message == RECONNECTED),
5114            "RECONNECTED control message entered discarded messages at step {}",
5115            step
5116        );
5117        prop_assert!(
5118            !model.live_sent.iter().any(|message| message == RECONNECTED),
5119            "RECONNECTED control message entered application sends at step {}",
5120            step
5121        );
5122
5123        Ok(())
5124    }
5125
5126    fn assert_messages_accounted_once(
5127        model: &ReconnectBufferModel,
5128        step: usize,
5129    ) -> Result<(), TestCaseError> {
5130        let mut seen = HashSet::new();
5131
5132        for message in model
5133            .released
5134            .iter()
5135            .chain(model.discarded.iter())
5136            .chain(model.buffer.iter())
5137            .chain(model.live_sent.iter())
5138        {
5139            prop_assert!(
5140                seen.insert(message.as_str()),
5141                "message {} appeared more than once at step {}",
5142                message,
5143                step
5144            );
5145        }
5146
5147        Ok(())
5148    }
5149
5150    fn apply_reconnect_buffer_trace_op(
5151        model: &mut ReconnectBufferModel,
5152        tracker: &AuthTracker,
5153        auth_receivers: &mut Vec<AuthResultReceiver>,
5154        op: &ReconnectBufferTraceOp,
5155    ) -> Result<(), TestCaseError> {
5156        match op {
5157            ReconnectBufferTraceOp::BeginAuth => {
5158                auth_receivers.push(tracker.begin());
5159                model.auth_state = AuthState::Unauthenticated;
5160            }
5161            ReconnectBufferTraceOp::AuthSucceeds => {
5162                tracker.succeed();
5163                model.auth_state = AuthState::Authenticated;
5164            }
5165            ReconnectBufferTraceOp::AuthFails => {
5166                tracker.fail(AUTH_FAILED);
5167                model.auth_state = AuthState::Failed;
5168            }
5169            ReconnectBufferTraceOp::AuthInvalidates => {
5170                tracker.invalidate();
5171                model.auth_state = AuthState::Unauthenticated;
5172            }
5173            ReconnectBufferTraceOp::ReconnectStarts => {
5174                tracker.invalidate();
5175                model.auth_state = AuthState::Unauthenticated;
5176                model.mode = ModelConnectionMode::Reconnect;
5177            }
5178            ReconnectBufferTraceOp::ReconnectCompletes => {
5179                model.mode = ModelConnectionMode::Active;
5180                model.handler_controls.push(RECONNECTED);
5181            }
5182            ReconnectBufferTraceOp::BufferedMessage(raw) => {
5183                let payload = model.next_payload(*raw);
5184                prop_assert_ne!(payload.as_str(), RECONNECTED);
5185
5186                if model.mode == ModelConnectionMode::Reconnect {
5187                    model.buffer.push_back(payload);
5188                } else {
5189                    model.live_sent.push(payload);
5190                }
5191            }
5192        }
5193
5194        Ok(())
5195    }
5196
5197    proptest! {
5198        #![proptest_config(ProptestConfig::with_cases(256))]
5199
5200        /// Property: reconnect-buffer traces match the auth-gated release and
5201        /// discard model, and `RECONNECTED` remains a separate control signal.
5202        #[rstest]
5203        fn test_reconnect_buffer_trace_matches_auth_gate_model(
5204            waits_for_auth in any::<bool>(),
5205            ops in proptest::collection::vec(reconnect_buffer_trace_op_strategy(), 1..100)
5206        ) {
5207            let auth_tracker = Arc::new(OnceLock::new());
5208            let reconnect_buffer_waits_for_auth = AtomicBool::new(waits_for_auth);
5209            let tracker = AuthTracker::new();
5210            auth_tracker.set(tracker.clone()).unwrap();
5211            let mut auth_receivers = Vec::new();
5212            let mut model = ReconnectBufferModel::new();
5213
5214            for (step, op) in ops.iter().enumerate() {
5215                apply_reconnect_buffer_trace_op(
5216                    &mut model,
5217                    &tracker,
5218                    &mut auth_receivers,
5219                    op,
5220                )?;
5221
5222                prop_assert_eq!(
5223                    tracker.auth_state(),
5224                    model.auth_state,
5225                    "auth state mismatch at step {}, op {:?}",
5226                    step,
5227                    op
5228                );
5229
5230                apply_ready_reconnect_buffer_action(
5231                    &mut model,
5232                    &reconnect_buffer_waits_for_auth,
5233                    &auth_tracker,
5234                    waits_for_auth,
5235                    step,
5236                    op,
5237                )?;
5238                assert_reconnected_control_stays_separate(&model, step)?;
5239                prop_assert_eq!(
5240                    model.handler_controls.len(),
5241                    ops[..=step]
5242                        .iter()
5243                        .filter(|op| matches!(op, ReconnectBufferTraceOp::ReconnectCompletes))
5244                        .count(),
5245                    "handler control count mismatch at step {}",
5246                    step
5247                );
5248                assert_messages_accounted_once(&model, step)?;
5249            }
5250        }
5251
5252        /// Property: successful re-authentication releases buffered messages
5253        /// exactly once when replay is configured to wait for auth.
5254        #[rstest]
5255        fn test_reconnect_buffer_releases_after_auth_success_once(
5256            payloads in proptest::collection::vec(any::<u8>(), 1..32),
5257            extra_success_ticks in 0usize..16
5258        ) {
5259            let auth_tracker = Arc::new(OnceLock::new());
5260            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
5261            let tracker = AuthTracker::new();
5262            auth_tracker.set(tracker.clone()).unwrap();
5263            let mut auth_receivers = Vec::new();
5264            let mut model = ReconnectBufferModel::new();
5265
5266            apply_reconnect_buffer_trace_op(
5267                &mut model,
5268                &tracker,
5269                &mut auth_receivers,
5270                &ReconnectBufferTraceOp::ReconnectStarts,
5271            )?;
5272            apply_reconnect_buffer_trace_op(
5273                &mut model,
5274                &tracker,
5275                &mut auth_receivers,
5276                &ReconnectBufferTraceOp::BeginAuth,
5277            )?;
5278
5279            for payload in payloads {
5280                apply_reconnect_buffer_trace_op(
5281                    &mut model,
5282                    &tracker,
5283                    &mut auth_receivers,
5284                    &ReconnectBufferTraceOp::BufferedMessage(payload),
5285                )?;
5286            }
5287
5288            let buffered_len = model.buffer.len();
5289            apply_reconnect_buffer_trace_op(
5290                &mut model,
5291                &tracker,
5292                &mut auth_receivers,
5293                &ReconnectBufferTraceOp::ReconnectCompletes,
5294            )?;
5295            apply_ready_reconnect_buffer_action(
5296                &mut model,
5297                &reconnect_buffer_waits_for_auth,
5298                &auth_tracker,
5299                true,
5300                0,
5301                &ReconnectBufferTraceOp::ReconnectCompletes,
5302            )?;
5303
5304            prop_assert_eq!(model.released.len(), 0);
5305            prop_assert_eq!(model.buffer.len(), buffered_len);
5306
5307            apply_reconnect_buffer_trace_op(
5308                &mut model,
5309                &tracker,
5310                &mut auth_receivers,
5311                &ReconnectBufferTraceOp::AuthSucceeds,
5312            )?;
5313            apply_ready_reconnect_buffer_action(
5314                &mut model,
5315                &reconnect_buffer_waits_for_auth,
5316                &auth_tracker,
5317                true,
5318                1,
5319                &ReconnectBufferTraceOp::AuthSucceeds,
5320            )?;
5321
5322            prop_assert_eq!(model.released.len(), buffered_len);
5323            prop_assert!(model.buffer.is_empty());
5324            assert_messages_accounted_once(&model, 1)?;
5325
5326            for tick in 0..extra_success_ticks {
5327                apply_reconnect_buffer_trace_op(
5328                    &mut model,
5329                    &tracker,
5330                    &mut auth_receivers,
5331                    &ReconnectBufferTraceOp::AuthSucceeds,
5332                )?;
5333                apply_ready_reconnect_buffer_action(
5334                    &mut model,
5335                    &reconnect_buffer_waits_for_auth,
5336                    &auth_tracker,
5337                    true,
5338                    tick + 2,
5339                    &ReconnectBufferTraceOp::AuthSucceeds,
5340                )?;
5341                prop_assert_eq!(
5342                    model.released.len(),
5343                    buffered_len,
5344                    "buffered messages replayed more than once at tick {}",
5345                    tick
5346                );
5347            }
5348        }
5349
5350        /// Property: auth failure discards messages buffered before or after
5351        /// that failure, and later auth success does not replay discarded data.
5352        #[rstest]
5353        fn test_reconnect_buffer_discards_after_auth_failure(
5354            before_failure_payloads in proptest::collection::vec(any::<u8>(), 0..16),
5355            after_failure_payloads in proptest::collection::vec(any::<u8>(), 1..16),
5356            later_success_ticks in 0usize..16
5357        ) {
5358            let auth_tracker = Arc::new(OnceLock::new());
5359            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
5360            let tracker = AuthTracker::new();
5361            auth_tracker.set(tracker.clone()).unwrap();
5362            let mut auth_receivers = Vec::new();
5363            let mut model = ReconnectBufferModel::new();
5364
5365            apply_reconnect_buffer_trace_op(
5366                &mut model,
5367                &tracker,
5368                &mut auth_receivers,
5369                &ReconnectBufferTraceOp::ReconnectStarts,
5370            )?;
5371            apply_reconnect_buffer_trace_op(
5372                &mut model,
5373                &tracker,
5374                &mut auth_receivers,
5375                &ReconnectBufferTraceOp::BeginAuth,
5376            )?;
5377
5378            for payload in before_failure_payloads {
5379                apply_reconnect_buffer_trace_op(
5380                    &mut model,
5381                    &tracker,
5382                    &mut auth_receivers,
5383                    &ReconnectBufferTraceOp::BufferedMessage(payload),
5384                )?;
5385            }
5386
5387            apply_reconnect_buffer_trace_op(
5388                &mut model,
5389                &tracker,
5390                &mut auth_receivers,
5391                &ReconnectBufferTraceOp::AuthFails,
5392            )?;
5393
5394            for payload in after_failure_payloads {
5395                apply_reconnect_buffer_trace_op(
5396                    &mut model,
5397                    &tracker,
5398                    &mut auth_receivers,
5399                    &ReconnectBufferTraceOp::BufferedMessage(payload),
5400                )?;
5401            }
5402
5403            let buffered_len = model.buffer.len();
5404            apply_reconnect_buffer_trace_op(
5405                &mut model,
5406                &tracker,
5407                &mut auth_receivers,
5408                &ReconnectBufferTraceOp::ReconnectCompletes,
5409            )?;
5410            apply_ready_reconnect_buffer_action(
5411                &mut model,
5412                &reconnect_buffer_waits_for_auth,
5413                &auth_tracker,
5414                true,
5415                0,
5416                &ReconnectBufferTraceOp::ReconnectCompletes,
5417            )?;
5418
5419            prop_assert_eq!(model.discarded.len(), buffered_len);
5420            prop_assert!(model.released.is_empty());
5421            prop_assert!(model.buffer.is_empty());
5422            assert_messages_accounted_once(&model, 0)?;
5423
5424            for tick in 0..later_success_ticks {
5425                apply_reconnect_buffer_trace_op(
5426                    &mut model,
5427                    &tracker,
5428                    &mut auth_receivers,
5429                    &ReconnectBufferTraceOp::BeginAuth,
5430                )?;
5431                apply_reconnect_buffer_trace_op(
5432                    &mut model,
5433                    &tracker,
5434                    &mut auth_receivers,
5435                    &ReconnectBufferTraceOp::AuthSucceeds,
5436                )?;
5437                apply_ready_reconnect_buffer_action(
5438                    &mut model,
5439                    &reconnect_buffer_waits_for_auth,
5440                    &auth_tracker,
5441                    true,
5442                    tick + 1,
5443                    &ReconnectBufferTraceOp::AuthSucceeds,
5444                )?;
5445                prop_assert!(
5446                    model.released.is_empty(),
5447                    "discarded messages replayed after later auth success at tick {}",
5448                    tick
5449                );
5450            }
5451        }
5452    }
5453}
5454
5455#[cfg(test)]
5456#[cfg(feature = "turmoil")]
5457mod turmoil_tests {
5458    use std::{sync::Arc, time::Duration};
5459
5460    use futures_util::{SinkExt, StreamExt};
5461    use nautilus_common::testing::wait_until_async;
5462    use rstest::rstest;
5463    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
5464    use turmoil::{Builder, net};
5465
5466    use super::*;
5467    use crate::websocket::types::channel_message_handler;
5468
5469    const AUTH_BUFFER_WAIT_SEED: u64 = 0xA17B_0001;
5470    const AUTH_BUFFER_DISCARD_SEED: u64 = 0xA17B_0002;
5471
5472    fn seeded_turmoil_builder(seed: u64) -> Builder {
5473        let mut builder = Builder::new();
5474        builder.rng_seed(seed);
5475        builder
5476    }
5477
5478    #[rstest]
5479    fn test_turmoil_reconnect_buffer_waits_for_auth() {
5480        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_WAIT_SEED).build();
5481        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
5482        let server_messages = Arc::clone(&messages);
5483
5484        sim.host("server", move || {
5485            let messages = Arc::clone(&server_messages);
5486            auth_buffer_server(messages)
5487        });
5488
5489        sim.client("client", async move {
5490            let tracker = AuthTracker::new();
5491            let (handler, _rx) = channel_message_handler();
5492            let client = WebSocketClient::connect(
5493                turmoil_websocket_config(),
5494                Some(handler),
5495                None,
5496                None,
5497                vec![],
5498                None,
5499            )
5500            .await
5501            .expect("Should connect");
5502
5503            client.set_auth_tracker(tracker.clone(), true);
5504            assert!(client.is_active(), "Client should start active");
5505
5506            wait_until_async(
5507                || async { client.is_reconnecting() },
5508                Duration::from_secs(3),
5509            )
5510            .await;
5511
5512            client
5513                .writer_tx
5514                .send(WriterCommand::Send(Message::Text("stale".into())))
5515                .unwrap();
5516
5517            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
5518
5519            let _auth_receiver = tracker.begin();
5520
5521            tokio::time::sleep(Duration::from_millis(300)).await;
5522            assert!(
5523                messages.lock().await.is_empty(),
5524                "buffered messages should wait for auth after reconnect"
5525            );
5526
5527            tracker.succeed();
5528
5529            wait_until_async(
5530                || {
5531                    let messages = Arc::clone(&messages);
5532                    async move { messages.lock().await.as_slice() == ["stale"] }
5533                },
5534                Duration::from_secs(3),
5535            )
5536            .await;
5537
5538            assert_eq!(messages.lock().await.as_slice(), ["stale"]);
5539
5540            client.disconnect().await;
5541            assert!(client.is_disconnected());
5542
5543            Ok(())
5544        });
5545
5546        sim.run().unwrap();
5547    }
5548
5549    #[rstest]
5550    fn test_turmoil_reconnect_buffer_discards_after_auth_failure() {
5551        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_DISCARD_SEED).build();
5552        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
5553        let server_messages = Arc::clone(&messages);
5554
5555        sim.host("server", move || {
5556            let messages = Arc::clone(&server_messages);
5557            auth_buffer_server(messages)
5558        });
5559
5560        sim.client("client", async move {
5561            let tracker = AuthTracker::new();
5562            let (handler, _rx) = channel_message_handler();
5563            let client = WebSocketClient::connect(
5564                turmoil_websocket_config(),
5565                Some(handler),
5566                None,
5567                None,
5568                vec![],
5569                None,
5570            )
5571            .await
5572            .expect("Should connect");
5573
5574            client.set_auth_tracker(tracker.clone(), true);
5575            assert!(client.is_active(), "Client should start active");
5576
5577            wait_until_async(
5578                || async { client.is_reconnecting() },
5579                Duration::from_secs(3),
5580            )
5581            .await;
5582
5583            client
5584                .writer_tx
5585                .send(WriterCommand::Send(Message::Text("stale".into())))
5586                .unwrap();
5587
5588            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
5589
5590            let _auth_receiver = tracker.begin();
5591            tracker.fail("rejected");
5592
5593            tokio::time::sleep(Duration::from_millis(300)).await;
5594            assert!(
5595                messages.lock().await.is_empty(),
5596                "buffered messages should be discarded after auth failure"
5597            );
5598
5599            let _retry_auth_receiver = tracker.begin();
5600            tracker.succeed();
5601
5602            tokio::time::sleep(Duration::from_millis(300)).await;
5603            assert!(
5604                messages.lock().await.is_empty(),
5605                "discarded messages should not replay on a later auth success"
5606            );
5607
5608            client.disconnect().await;
5609            assert!(client.is_disconnected());
5610
5611            Ok(())
5612        });
5613
5614        sim.run().unwrap();
5615    }
5616
5617    fn turmoil_websocket_config() -> WebSocketConfig {
5618        WebSocketConfig {
5619            url: "ws://server:8080".to_string(),
5620            headers: vec![],
5621            heartbeat: None,
5622            heartbeat_msg: None,
5623            reconnect_timeout_ms: Some(5_000),
5624            reconnect_delay_initial_ms: Some(50),
5625            reconnect_delay_max_ms: Some(200),
5626            reconnect_backoff_factor: Some(1.0),
5627            reconnect_jitter_ms: Some(0),
5628            reconnect_max_attempts: None,
5629            idle_timeout_ms: None,
5630            backend: TransportBackend::Tungstenite,
5631            proxy_url: None,
5632        }
5633    }
5634
5635    async fn auth_buffer_server(
5636        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
5637    ) -> Result<(), Box<dyn std::error::Error>> {
5638        let listener = net::TcpListener::bind("0.0.0.0:8080").await?;
5639
5640        let (stream, _) = listener.accept().await?;
5641        let mut websocket = accept_async(stream).await?;
5642        let _ = websocket.send(WsMessage::Text("first".into())).await;
5643        drop(websocket);
5644
5645        tokio::time::sleep(Duration::from_millis(200)).await;
5646
5647        let (stream, _) = listener.accept().await?;
5648        let mut websocket = accept_async(stream).await?;
5649
5650        while let Some(msg) = websocket.next().await {
5651            match msg {
5652                Ok(WsMessage::Text(text)) => {
5653                    messages.lock().await.push(text.to_string());
5654                }
5655                Ok(WsMessage::Close(_)) => {
5656                    let _ = websocket.close(None).await;
5657                    break;
5658                }
5659                Ok(_) => {}
5660                Err(_) => break,
5661            }
5662        }
5663
5664        Ok(())
5665    }
5666}