nautilus_derive/common/

retry.rs

// -------------------------------------------------------------------------------------------------
//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
//  https://nautechsystems.io
//
//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
//  You may not use this file except in compliance with the License.
//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// -------------------------------------------------------------------------------------------------

//! Retry classification for the Derive adapter.
//!
//! Splits [`DeriveHttpError`] and [`DeriveWsError`] into retryable, terminal,
//! and fatal categories. The HTTP client routes errors through these classifiers
//! when driving [`nautilus_network::retry::RetryManager`]; the adapter-level
//! [`crate::common::error::DeriveError`] reuses them for `is_retryable` /
//! `is_fatal`.

use nautilus_network::retry::{RetryConfig, RetryError};

use crate::{http::DeriveHttpError, websocket::DeriveWsError};

/// Builds a [`RetryConfig`] for Derive HTTP calls from the adapter's config
/// fields.
///
/// `max_retries` is the budget; `initial_delay_ms` and `max_delay_ms` bound
/// the exponential backoff. Other fields use values tuned for Derive: a
/// 60-second per-attempt timeout (REST endpoints can return slow during venue
/// load), a 3-minute overall budget, and 1s of jitter to avoid synchronizing
/// retry storms across processes.
#[must_use]
pub fn http_retry_config(
    max_retries: u32,
    initial_delay_ms: u64,
    max_delay_ms: u64,
) -> RetryConfig {
    RetryConfig {
        max_retries,
        initial_delay_ms,
        max_delay_ms,
        backoff_factor: 2.0,
        jitter_ms: 1_000,
        operation_timeout_ms: Some(60_000),
        immediate_first: false,
        max_elapsed_ms: Some(180_000),
    }
}

/// Returns `true` for transient HTTP errors eligible for read retries with backoff.
///
/// This classifier does not establish the outcome of a state-changing request.
///
/// Retryable categories:
///
/// - Transport failures (connection reset, timeout, DNS).
/// - HTTP 5xx and 408 / 429.
/// - JSON-RPC `Server error` codes in the `-32099..=-32000` range.
/// - JSON-RPC internal error (`-32603`).
/// - Read-backend unavailability (`9002`).
///
/// Everything else (validation, signed-fee-too-low, insufficient-margin,
/// auth failure) is non-retryable under this policy.
#[must_use]
pub fn should_retry_http_error(error: &DeriveHttpError) -> bool {
    match error {
        DeriveHttpError::Transport(_) => true,
        DeriveHttpError::Retry(RetryError::OperationTimeout { .. }) => true,
        DeriveHttpError::Http { status, .. } => is_retryable_status(*status),
        DeriveHttpError::JsonRpc { code, .. } => is_retryable_jsonrpc_code(*code),
        DeriveHttpError::MissingResult { .. }
        | DeriveHttpError::Decode(_)
        | DeriveHttpError::Serde(_)
        | DeriveHttpError::Auth(_)
        | DeriveHttpError::MissingCredentials { .. }
        | DeriveHttpError::Retry(_) => false,
    }
}

/// Returns `true` for HTTP errors that signal a fatal session state requiring
/// operator intervention (auth header rejection, session key deregistered,
/// subaccount withdrawn).
///
/// Fatal errors are a subset of non-retryable: they should also short-circuit
/// any caller-level retry budgets.
#[must_use]
pub fn is_fatal_http_error(error: &DeriveHttpError) -> bool {
    match error {
        DeriveHttpError::Auth(_) | DeriveHttpError::MissingCredentials { .. } => true,
        DeriveHttpError::Http { status, .. } => matches!(*status, 401 | 403),
        DeriveHttpError::JsonRpc { code, .. } => is_fatal_jsonrpc_code(*code),
        _ => false,
    }
}

/// Returns `true` for transient WebSocket errors eligible for recovery retries.
///
/// A timeout or transport error does not authorize replaying an order write.
#[must_use]
pub fn should_retry_ws_error(error: &DeriveWsError) -> bool {
    match error {
        DeriveWsError::Transport(_)
        | DeriveWsError::RequestCancelled { .. }
        | DeriveWsError::Timeout { .. }
        | DeriveWsError::RateLimited { .. } => true,
        DeriveWsError::JsonRpc { code, .. } => is_retryable_jsonrpc_code(*code),
        DeriveWsError::NotConnected
        | DeriveWsError::Serde(_)
        | DeriveWsError::Auth(_)
        | DeriveWsError::Authentication { .. }
        | DeriveWsError::Subscription { .. }
        | DeriveWsError::MissingCredentials { .. } => false,
    }
}

/// Returns `true` for WebSocket errors that indicate a fatal session state.
#[must_use]
pub fn is_fatal_ws_error(error: &DeriveWsError) -> bool {
    match error {
        DeriveWsError::Auth(_)
        | DeriveWsError::Authentication { .. }
        | DeriveWsError::MissingCredentials { .. } => true,
        DeriveWsError::JsonRpc { code, .. } => is_fatal_jsonrpc_code(*code),
        _ => false,
    }
}

/// Classifies an HTTP status code.
#[must_use]
fn is_retryable_status(status: u16) -> bool {
    matches!(status, 408 | 429) || (500..600).contains(&status)
}

/// Classifies a JSON-RPC error code.
///
/// Read retries cover backend unavailability and generic transient failures.
/// Confirmation timeouts do not authorize replaying a state-changing command.
#[must_use]
pub(crate) fn is_retryable_jsonrpc_code(code: i64) -> bool {
    matches!(code, -32603 | 9002) || (-32099..=-32000).contains(&code)
}

/// Returns `true` only for JSON-RPC codes where the *outcome of a state-changing
/// write* is ambiguous: the venue may have processed the request and merely
/// failed to respond. Retry eligibility does not establish a write's outcome.
///
/// The retry classifier covers transient transport-style failures, including
/// venue-defined codes like `-32000 Rate limit exceeded`. Rate-limit (and most
/// other Derive server errors) is a **definitive** rejection: the gateway threw
/// the request out before the matching engine saw it. Treating those as
/// ambiguous leaves the order hanging in `Submitted` forever because no WS
/// frame will come for an order that was never placed.
///
/// Internal error `-32603` and confirmation timeouts `9000` / `9001` leave
/// acceptance unresolved. The venue requires querying those orders before
/// resubmission.
#[must_use]
pub(crate) fn is_write_outcome_ambiguous_jsonrpc(code: i64) -> bool {
    matches!(code, -32603 | 9000 | 9001)
}

/// Returns `true` when a WebSocket write's outcome is unknown (sent, but no
/// clear venue verdict), so the caller emits no terminal event and lets
/// reconciliation settle the order. `JsonRpc` defers to the shared code policy
/// in [`is_write_outcome_ambiguous_jsonrpc`].
///
/// Two non-obvious calls: `Serde` is ambiguous because it is a failure to decode
/// the *response* (the request cannot fail to serialize), so the action may have
/// been processed; `NotConnected` is definitive because it is returned before
/// the frame is sent, so the order was never placed.
#[must_use]
pub(crate) fn is_write_outcome_ambiguous_ws(error: &DeriveWsError) -> bool {
    match error {
        DeriveWsError::Transport(_)
        | DeriveWsError::RequestCancelled { .. }
        | DeriveWsError::Timeout { .. }
        | DeriveWsError::Serde(_) => true,
        DeriveWsError::JsonRpc { code, .. } => is_write_outcome_ambiguous_jsonrpc(*code),
        DeriveWsError::NotConnected
        | DeriveWsError::Auth(_)
        | DeriveWsError::Authentication { .. }
        | DeriveWsError::Subscription { .. }
        | DeriveWsError::RateLimited { .. }
        | DeriveWsError::MissingCredentials { .. } => false,
    }
}

/// Classifies unrecoverable framing and account, signer, or session failures.
#[must_use]
fn is_fatal_jsonrpc_code(code: i64) -> bool {
    matches!(
        code,
        -32600
            | -32700
            | 14000
            | 14001
            | 14013
            | 14014
            | 14020
            | 14021
            | 14023
            | 14026
            | 14030
            | 14031
    )
}

#[cfg(test)]
mod tests {
    use rstest::rstest;
    use serde_json::Value;

    use super::*;

    #[rstest]
    #[case(14000, true, false)]
    #[case(14001, true, false)]
    #[case(14013, true, false)]
    #[case(14014, true, false)]
    #[case(14020, true, false)]
    #[case(14021, true, false)]
    #[case(14023, true, false)]
    #[case(14026, true, false)]
    #[case(14030, true, false)]
    #[case(14031, true, false)]
    #[case(9002, false, true)]
    #[case(9000, false, false)]
    #[case(12001, false, false)]
    fn test_venue_session_error_classification(
        #[case] code: i64,
        #[case] fatal: bool,
        #[case] retry: bool,
    ) {
        let http = DeriveHttpError::JsonRpc {
            code,
            message: "opaque".to_string(),
            data: None,
            retry_after: None,
        };

        let ws = DeriveWsError::JsonRpc {
            code,
            message: "opaque".to_string(),
            data: None,
        };

        assert_eq!(is_fatal_http_error(&http), fatal);
        assert_eq!(is_fatal_ws_error(&ws), fatal);
        assert_eq!(should_retry_http_error(&http), retry);
        assert_eq!(should_retry_ws_error(&ws), retry);
    }

    #[rstest]
    fn test_transport_error_retryable() {
        let err = DeriveHttpError::transport("conn reset");
        assert!(should_retry_http_error(&err));
        assert!(!is_fatal_http_error(&err));
    }

    #[rstest]
    #[case(500, true)]
    #[case(502, true)]
    #[case(503, true)]
    #[case(504, true)]
    #[case(429, true)]
    #[case(408, true)]
    #[case(400, false)]
    #[case(404, false)]
    #[case(409, false)]
    #[case(422, false)]
    fn test_http_status_retry_classification(#[case] status: u16, #[case] retryable: bool) {
        let err = DeriveHttpError::http(status, "body");
        assert_eq!(should_retry_http_error(&err), retryable);
    }

    #[rstest]
    #[case(401)]
    #[case(403)]
    fn test_http_auth_status_is_fatal(#[case] status: u16) {
        let err = DeriveHttpError::http(status, "Unauthorized");
        assert!(is_fatal_http_error(&err));
        assert!(!should_retry_http_error(&err));
    }

    #[rstest]
    fn test_jsonrpc_invalid_params_not_retryable() {
        // Venue surfaces `signed_max_fee_too_low`, `insufficient_margin`, etc.
        // as standard JSON-RPC -32602 invalid-params payloads. These reflect
        // caller-side state and must never be retried.
        let err = DeriveHttpError::JsonRpc {
            code: -32602,
            message: "signed_max_fee_too_low".into(),
            data: None,
            retry_after: None,
        };

        assert!(!should_retry_http_error(&err));
        assert!(!is_fatal_http_error(&err));
    }

    #[rstest]
    fn test_jsonrpc_server_error_range_retryable() {
        let err = DeriveHttpError::JsonRpc {
            code: -32050,
            message: "Server busy".into(),
            data: None,
            retry_after: None,
        };

        assert!(should_retry_http_error(&err));
    }

    #[rstest]
    fn test_jsonrpc_internal_error_retryable() {
        let err = DeriveHttpError::JsonRpc {
            code: -32603,
            message: "Internal error".into(),
            data: None,
            retry_after: None,
        };

        assert!(should_retry_http_error(&err));
    }

    #[rstest]
    fn test_jsonrpc_invalid_request_is_fatal() {
        let err = DeriveHttpError::JsonRpc {
            code: -32600,
            message: "Invalid request".into(),
            data: Some(Value::Null),
            retry_after: None,
        };

        assert!(is_fatal_http_error(&err));
        assert!(!should_retry_http_error(&err));
    }

    #[rstest]
    fn test_missing_credentials_terminal() {
        let err = DeriveHttpError::MissingCredentials {
            method: "private/order".into(),
        };

        assert!(!should_retry_http_error(&err));
        assert!(is_fatal_http_error(&err));
    }

    #[rstest]
    fn test_ws_transport_retryable() {
        let err = DeriveWsError::transport("send failed");
        assert!(should_retry_ws_error(&err));
    }

    #[rstest]
    fn test_ws_not_connected_terminal() {
        let err = DeriveWsError::NotConnected;
        assert!(!should_retry_ws_error(&err));
        assert!(!is_fatal_ws_error(&err));
    }

    #[rstest]
    fn test_ws_request_cancelled_retryable() {
        // The handler drops the oneshot on reconnect; the caller can re-issue
        // after the new session is up.
        let err = DeriveWsError::RequestCancelled {
            method: "subscribe".into(),
        };

        assert!(should_retry_ws_error(&err));
    }

    #[rstest]
    fn test_ws_timeout_retryable_not_fatal() {
        let err = DeriveWsError::Timeout {
            method: "private/order".into(),
        };

        assert!(should_retry_ws_error(&err));
        assert!(!is_fatal_ws_error(&err));
    }

    #[rstest]
    fn test_ws_authentication_and_subscription_failures_are_terminal() {
        let authentication = DeriveWsError::Authentication {
            operation: "private/order".into(),
            reason: "session recovery failed".into(),
        };

        let subscription = DeriveWsError::Subscription {
            details: "30769.trades: unauthorized".into(),
        };

        assert!(!should_retry_ws_error(&authentication));
        assert!(is_fatal_ws_error(&authentication));
        assert!(!should_retry_ws_error(&subscription));
        assert!(!is_fatal_ws_error(&subscription));
    }

    #[rstest]
    fn test_ws_write_outcome_ambiguous_classification() {
        // Sent-but-unconfirmed outcomes are ambiguous; everything else is a
        // definitive rejection the caller can surface as a terminal event.
        let ambiguous = [
            DeriveWsError::transport("send failed"),
            DeriveWsError::RequestCancelled {
                method: "private/order".into(),
            },
            DeriveWsError::Timeout {
                method: "private/order".into(),
            },
            // A response the client cannot decode: the action may have been
            // processed, so await reconciliation rather than reject.
            DeriveWsError::Serde(serde_json::from_str::<Value>("{").unwrap_err()),
            DeriveWsError::JsonRpc {
                code: -32603,
                message: "Internal error".into(),
                data: None,
            },
        ];

        let definitive = [
            DeriveWsError::NotConnected,
            DeriveWsError::JsonRpc {
                code: -32602,
                message: "signed_max_fee_too_low".into(),
                data: None,
            },
            DeriveWsError::MissingCredentials {
                operation: "private/order".into(),
            },
            DeriveWsError::Authentication {
                operation: "private/order".into(),
                reason: "session recovery failed".into(),
            },
            DeriveWsError::Subscription {
                details: "30769.trades: unauthorized".into(),
            },
        ];

        for err in &ambiguous {
            assert!(
                is_write_outcome_ambiguous_ws(err),
                "expected ambiguous: {err}"
            );
        }

        for err in &definitive {
            assert!(
                !is_write_outcome_ambiguous_ws(err),
                "expected definitive: {err}",
            );
        }
    }
}