nautilus_serialization/
lib.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Data serialization and format conversion for [NautilusTrader](https://nautilustrader.io).
17//!
18//! The `nautilus-serialization` crate provides data serialization capabilities for converting
19//! trading data between different formats including Apache Arrow and Cap'n Proto.
20//! This enables efficient data storage, retrieval, and interoperability across different systems:
21//!
22//! - **Apache Arrow integration**: Schema definitions and encoding/decoding for market data types.
23//! - **Parquet file operations**: High-performance columnar storage for historical data analysis.
24//! - **Record batch processing**: Efficient batch operations for time-series data.
25//! - **Schema management**: Type-safe schema definitions with metadata preservation.
26//! - **Cross-format conversion**: Data interchange between Arrow, Cap'n Proto, and native types.
27//! - **Cap'n Proto serialization**: Zero-copy, schema-based serialization for efficient data interchange (requires `capnp` feature).
28//! - **SBE decode utilities**: Zero-copy cursor and shared decode errors for SBE parsers (requires `sbe` feature).
29//!
30//! # NautilusTrader
31//!
32//! [NautilusTrader](https://nautilustrader.io) is an open-source, production-grade, Rust-native
33//! engine for multi-asset, multi-venue trading systems.
34//!
35//! The system spans research, deterministic simulation, and live execution within a single
36//! event-driven architecture, providing research-to-live semantic parity.
37//!
38//! # Feature Flags
39//!
40//! This crate provides feature flags to control source code inclusion during compilation,
41//! depending on the intended use case, i.e. whether to provide Python bindings
42//! for the [nautilus_trader](https://pypi.org/project/nautilus_trader) Python package,
43//! or as part of a Rust only build.
44//!
45//! - `arrow`: Enables Apache Arrow schema definitions and RecordBatch encoding/decoding.
46//! - `arrow-display`: Enables display-friendly Arrow encoders for market data and requires `arrow`.
47//! - `capnp`: Enables [Cap'n Proto](https://capnproto.org) serialization support.
48//! - `extension-module`: Builds as a Python extension module.
49//! - `high-precision`: Enables
50//!   [high-precision mode](https://nautilustrader.io/docs/nightly/getting_started/installation/#precision-mode)
51//!   to use 128-bit value types.
52//! - `python`: Enables Python bindings from [PyO3](https://pyo3.rs).
53//! - `sbe`: Enables generic SBE (Simple Binary Encoding) decode utilities.
54//!
55//! **Warning:** SBE and Cap'n Proto schemas are not yet stable and may break between releases.
56
57#![warn(rustc::all)]
58#![warn(clippy::pedantic)]
59#![warn(clippy::clone_on_ref_ptr)]
60#![deny(unsafe_code)]
61#![deny(unsafe_op_in_unsafe_fn)]
62#![deny(nonstandard_style)]
63#![deny(missing_debug_implementations)]
64#![deny(clippy::missing_errors_doc)]
65#![deny(clippy::missing_panics_doc)]
66#![deny(rustdoc::broken_intra_doc_links)]
67#![allow(
68    clippy::doc_markdown,
69    reason = "serialization docs are heavy on Arrow and domain-specific type names where blanket backticks add noise"
70)]
71#![allow(
72    clippy::too_many_lines,
73    reason = "wide encode and decode functions mirror protocol schemas and Arrow record layouts"
74)]
75#![allow(
76    clippy::implicit_hasher,
77    reason = "serialization metadata uses a standardized HashMap<String, String> shape across traits and codecs"
78)]
79#![allow(
80    clippy::similar_names,
81    reason = "domain terms such as trade/trader and side/size are intentionally similar"
82)]
83#![allow(
84    clippy::cast_possible_truncation,
85    clippy::cast_possible_wrap,
86    clippy::cast_precision_loss,
87    clippy::cast_sign_loss,
88    clippy::cast_lossless,
89    reason = "wire-format and fixed-point conversions in serialization code require explicit numeric casts"
90)]
91#![allow(
92    clippy::assert_is_empty,
93    reason = "`assert!(x.is_empty())` is clearer than comparing against an empty value"
94)]
95#![cfg_attr(
96    test,
97    allow(
98        clippy::float_cmp,
99        reason = "serialization tests assert exact float encodings and decoded values"
100    )
101)]
102
103#[cfg(feature = "arrow")]
104pub mod arrow;
105
106/// Re-export custom data registration for use by persistence and tests.
107#[cfg(feature = "arrow")]
108pub use arrow::custom::ensure_custom_data_registered;
109/// Re-exports MsgPack codecs for consumers configuring serialization through this crate.
110pub use nautilus_core::serialization::msgpack;
111
112#[cfg(feature = "capnp")]
113pub mod capnp;
114
115#[cfg(feature = "sbe")]
116pub mod sbe;
117
118#[cfg(any(feature = "capnp", feature = "sbe"))]
119mod numeric;
120
121#[cfg(feature = "capnp")]
122macro_rules! include_capnp_module {
123    ($name:ident, $path:expr) => {
124        #[cfg(all(feature = "capnp", not(docsrs)))]
125        #[allow(
126            clippy::all,
127            clippy::missing_errors_doc,
128            clippy::missing_panics_doc,
129            warnings,
130            dead_code,
131            missing_debug_implementations
132        )]
133        pub mod $name {
134            include!(concat!(env!("OUT_DIR"), $path));
135        }
136
137        #[cfg(all(feature = "capnp", docsrs))]
138        #[allow(
139            clippy::all,
140            clippy::missing_errors_doc,
141            clippy::missing_panics_doc,
142            warnings,
143            dead_code,
144            missing_debug_implementations
145        )]
146        pub mod $name {
147            include!(concat!(
148                env!("CARGO_MANIFEST_DIR"),
149                "/generated/capnp",
150                $path
151            ));
152        }
153    };
154}
155
156#[cfg(feature = "capnp")]
157include_capnp_module!(base_capnp, "/common/base_capnp.rs");
158#[cfg(feature = "capnp")]
159include_capnp_module!(identifiers_capnp, "/common/identifiers_capnp.rs");
160#[cfg(feature = "capnp")]
161include_capnp_module!(types_capnp, "/common/types_capnp.rs");
162#[cfg(feature = "capnp")]
163include_capnp_module!(enums_capnp, "/common/enums_capnp.rs");
164#[cfg(feature = "capnp")]
165include_capnp_module!(trading_capnp, "/commands/trading_capnp.rs");
166#[cfg(feature = "capnp")]
167include_capnp_module!(data_capnp, "/commands/data_capnp.rs");
168#[cfg(feature = "capnp")]
169include_capnp_module!(order_capnp, "/events/order_capnp.rs");
170#[cfg(feature = "capnp")]
171include_capnp_module!(position_capnp, "/events/position_capnp.rs");
172#[cfg(feature = "capnp")]
173include_capnp_module!(account_capnp, "/events/account_capnp.rs");
174#[cfg(feature = "capnp")]
175include_capnp_module!(market_capnp, "/data/market_capnp.rs");
176#[cfg(feature = "capnp")]
177include_capnp_module!(instruments_capnp, "/data/instruments_capnp.rs");
178
179#[cfg(feature = "python")]
180pub mod python;
181
182/// Generates typed Arrow encoding for custom data.
183#[cfg(feature = "arrow")]
184pub use nautilus_macros::arrow_custom_data;
nautilus_serialization/lib.rs

nautilus_serialization/
lib.rs
