nautilus_serialization/arrow/
snapshot.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::collections::HashMap;
17
18use arrow::{datatypes::Schema, error::ArrowError, record_batch::RecordBatch};
19use nautilus_model::events::{OrderSnapshot, PositionSnapshot};
20
21use super::{
22    ArrowSchemaProvider, DecodeTypedFromRecordBatch, EncodeToRecordBatch, EncodingError,
23    KEY_INSTRUMENT_ID,
24    json::{JsonFieldSpec, decode_batch, encode_batch, metadata_for_type, schema_for_type},
25};
26
27const ORDER_SNAPSHOT_FIELDS: &[JsonFieldSpec] = &[
28    JsonFieldSpec::utf8("trader_id", false),
29    JsonFieldSpec::utf8("strategy_id", false),
30    JsonFieldSpec::utf8("instrument_id", false),
31    JsonFieldSpec::utf8("client_order_id", false),
32    JsonFieldSpec::utf8("venue_order_id", true),
33    JsonFieldSpec::utf8("position_id", true),
34    JsonFieldSpec::utf8("account_id", true),
35    JsonFieldSpec::utf8("last_trade_id", true),
36    JsonFieldSpec::utf8("order_type", false),
37    JsonFieldSpec::utf8("order_side", false),
38    JsonFieldSpec::utf8("quantity", false),
39    JsonFieldSpec::utf8("price", true),
40    JsonFieldSpec::utf8("trigger_price", true),
41    JsonFieldSpec::utf8("trigger_type", true),
42    JsonFieldSpec::utf8("limit_offset", true),
43    JsonFieldSpec::utf8("trailing_offset", true),
44    JsonFieldSpec::utf8("trailing_offset_type", true),
45    JsonFieldSpec::utf8("time_in_force", false),
46    JsonFieldSpec::timestamp("expire_time", true),
47    JsonFieldSpec::utf8("filled_qty", false),
48    JsonFieldSpec::utf8("liquidity_side", true),
49    JsonFieldSpec::decimal_str("avg_px", true),
50    JsonFieldSpec::decimal_str("slippage", true),
51    JsonFieldSpec::utf8_json("commissions", false),
52    JsonFieldSpec::utf8("status", false),
53    JsonFieldSpec::boolean("is_post_only", false),
54    JsonFieldSpec::boolean("is_reduce_only", false),
55    JsonFieldSpec::boolean("is_quote_quantity", false),
56    JsonFieldSpec::utf8("display_qty", true),
57    JsonFieldSpec::utf8("emulation_trigger", true),
58    JsonFieldSpec::utf8("trigger_instrument_id", true),
59    JsonFieldSpec::utf8("contingency_type", true),
60    JsonFieldSpec::utf8("order_list_id", true),
61    JsonFieldSpec::utf8_json("linked_order_ids", true),
62    JsonFieldSpec::utf8("parent_order_id", true),
63    JsonFieldSpec::utf8("exec_algorithm_id", true),
64    JsonFieldSpec::utf8_json("exec_algorithm_params", true),
65    JsonFieldSpec::utf8("exec_spawn_id", true),
66    JsonFieldSpec::utf8_json("tags", true),
67    JsonFieldSpec::utf8("init_id", false),
68    JsonFieldSpec::timestamp("ts_init", false),
69    JsonFieldSpec::timestamp("ts_last", false),
70    // Appended (not inserted) so older batches without this column fail with a clean
71    // `MissingColumn` error rather than silently reading a shifted column.
72    JsonFieldSpec::utf8("activation_price", true),
73];
74
75const POSITION_SNAPSHOT_FIELDS: &[JsonFieldSpec] = &[
76    JsonFieldSpec::utf8("trader_id", false),
77    JsonFieldSpec::utf8("strategy_id", false),
78    JsonFieldSpec::utf8("instrument_id", false),
79    JsonFieldSpec::utf8("position_id", false),
80    JsonFieldSpec::utf8("account_id", false),
81    JsonFieldSpec::utf8("opening_order_id", false),
82    JsonFieldSpec::utf8("closing_order_id", true),
83    JsonFieldSpec::utf8("entry", false),
84    JsonFieldSpec::utf8("side", false),
85    JsonFieldSpec::f64("signed_qty", false),
86    JsonFieldSpec::utf8("quantity", false),
87    JsonFieldSpec::utf8("peak_qty", false),
88    JsonFieldSpec::utf8("quote_currency", false),
89    JsonFieldSpec::utf8("base_currency", true),
90    JsonFieldSpec::utf8("settlement_currency", false),
91    JsonFieldSpec::f64("avg_px_open", false),
92    JsonFieldSpec::f64("avg_px_close", true),
93    JsonFieldSpec::f64("realized_return", true),
94    JsonFieldSpec::utf8("realized_pnl", true),
95    JsonFieldSpec::utf8("unrealized_pnl", true),
96    JsonFieldSpec::utf8_json("commissions", false),
97    JsonFieldSpec::u64("duration_ns", true),
98    JsonFieldSpec::timestamp("ts_opened", false),
99    JsonFieldSpec::timestamp("ts_closed", true),
100    JsonFieldSpec::timestamp("ts_init", false),
101    JsonFieldSpec::timestamp("ts_last", false),
102    JsonFieldSpec::utf8_json("replay_state", true),
103];
104
105fn instrument_metadata(type_name: &'static str, instrument_id: &str) -> HashMap<String, String> {
106    let mut metadata = metadata_for_type(type_name);
107    metadata.insert(KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string());
108    metadata
109}
110
111macro_rules! impl_snapshot_arrow {
112    ($type:ty, $type_name:expr, $fields:expr) => {
113        impl ArrowSchemaProvider for $type {
114            fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema {
115                schema_for_type($type_name, metadata, $fields)
116            }
117        }
118
119        impl EncodeToRecordBatch for $type {
120            fn encode_batch<T>(
121                metadata: &HashMap<String, String>,
122                data: &[T],
123            ) -> Result<RecordBatch, ArrowError>
124            where
125                T: std::borrow::Borrow<Self>,
126            {
127                encode_batch(
128                    $type_name,
129                    metadata,
130                    data.iter().map(std::borrow::Borrow::borrow),
131                    $fields,
132                )
133            }
134
135            fn metadata(&self) -> HashMap<String, String> {
136                instrument_metadata($type_name, &self.instrument_id.to_string())
137            }
138        }
139
140        impl DecodeTypedFromRecordBatch for $type {
141            fn decode_typed_batch(
142                metadata: &HashMap<String, String>,
143                record_batch: RecordBatch,
144            ) -> Result<Vec<Self>, EncodingError> {
145                decode_batch(metadata, &record_batch, $fields, Some($type_name))
146            }
147        }
148    };
149}
150
151impl_snapshot_arrow!(OrderSnapshot, "OrderSnapshot", ORDER_SNAPSHOT_FIELDS);
152impl_snapshot_arrow!(
153    PositionSnapshot,
154    "PositionSnapshot",
155    POSITION_SNAPSHOT_FIELDS
156);
157
158#[cfg(test)]
159mod tests {
160    use std::str::FromStr;
161
162    use arrow::datatypes::DataType;
163    use nautilus_core::{DurationNanos, UnixNanos};
164    use nautilus_model::{
165        enums::{OrderSide, OrderType, PositionSide, TrailingOffsetType},
166        identifiers::{AccountId, ClientOrderId, InstrumentId, PositionId, StrategyId, TraderId},
167        orders::OrderTestBuilder,
168        types::{Currency, Money, Price, Quantity},
169    };
170    use rstest::rstest;
171    use rust_decimal::Decimal;
172    use rust_decimal_macros::dec;
173
174    use super::*;
175
176    #[rstest]
177    fn test_order_snapshot_round_trip_preserves_decimal_precision() {
178        let order = OrderTestBuilder::new(OrderType::TrailingStopLimit)
179            .instrument_id(InstrumentId::from("BTCUSDT.BINANCE"))
180            .side(OrderSide::Buy)
181            .price(Price::from("50000"))
182            .trigger_price(Price::from("50500"))
183            .limit_offset(Decimal::from_str("0.123456789123456789").unwrap())
184            .trailing_offset(Decimal::from_str("0.987654321987654321").unwrap())
185            .trailing_offset_type(TrailingOffsetType::Price)
186            .quantity(Quantity::from("0.5"))
187            .build();
188        let snapshot = OrderSnapshot::from(order);
189        let metadata = snapshot.metadata();
190        let batch =
191            OrderSnapshot::encode_batch(&metadata, std::slice::from_ref(&snapshot)).unwrap();
192        let decoded = OrderSnapshot::decode_typed_batch(batch.schema().metadata(), batch).unwrap();
193
194        assert_eq!(decoded, vec![snapshot]);
195    }
196
197    fn make_order_snapshot(avg_px: Option<Decimal>, slippage: Option<Decimal>) -> OrderSnapshot {
198        let order = OrderTestBuilder::new(OrderType::Limit)
199            .instrument_id(InstrumentId::from("BTCUSDT.BINANCE"))
200            .side(OrderSide::Buy)
201            .price(Price::from("50000"))
202            .quantity(Quantity::from("0.5"))
203            .build();
204        let mut snapshot = OrderSnapshot::from(order);
205        snapshot.avg_px = avg_px;
206        snapshot.slippage = slippage;
207        snapshot
208    }
209
210    // The catalog spec before `avg_px` and `slippage` became exact
211    fn legacy_float64_fields() -> Vec<JsonFieldSpec> {
212        ORDER_SNAPSHOT_FIELDS
213            .iter()
214            .map(|spec| match spec.name {
215                "avg_px" | "slippage" => JsonFieldSpec::f64(spec.name, spec.nullable),
216                _ => *spec,
217            })
218            .collect()
219    }
220
221    #[rstest]
222    fn test_order_snapshot_round_trip_preserves_exact_avg_px_and_slippage() {
223        // A quotient at full `Decimal` scale, which is the precision the `Float64` column could
224        // not hold.
225        let snapshot = make_order_snapshot(
226            Some(Decimal::from_str("1.6666666666666666666666666667").unwrap()),
227            Some(Decimal::from_str("0.0000000000000000000000000001").unwrap()),
228        );
229        let metadata = snapshot.metadata();
230        let batch =
231            OrderSnapshot::encode_batch(&metadata, std::slice::from_ref(&snapshot)).unwrap();
232
233        let avg_px_field = batch.schema().field_with_name("avg_px").unwrap().clone();
234        let slippage_field = batch.schema().field_with_name("slippage").unwrap().clone();
235        let decoded = OrderSnapshot::decode_typed_batch(batch.schema().metadata(), batch).unwrap();
236
237        assert_eq!(avg_px_field.data_type(), &DataType::Utf8);
238        assert_eq!(slippage_field.data_type(), &DataType::Utf8);
239        assert_eq!(decoded, vec![snapshot]);
240    }
241
242    #[rstest]
243    fn test_order_snapshot_round_trip_null_avg_px_and_slippage() {
244        let snapshot = make_order_snapshot(None, None);
245        let metadata = snapshot.metadata();
246        let batch =
247            OrderSnapshot::encode_batch(&metadata, std::slice::from_ref(&snapshot)).unwrap();
248        let decoded = OrderSnapshot::decode_typed_batch(batch.schema().metadata(), batch).unwrap();
249
250        assert_eq!(decoded, vec![snapshot]);
251    }
252
253    #[rstest]
254    fn test_order_snapshot_decodes_legacy_float64_columns() {
255        // A catalog file written while the fields were `f64`: the columns are `Float64`, not
256        // `Utf8`, and must still decode to the same economic state without a version marker.
257        let snapshot = make_order_snapshot(Some(dec!(1.07)), Some(dec!(0.07)));
258        let metadata = snapshot.metadata();
259        let legacy_batch = encode_batch(
260            "OrderSnapshot",
261            &metadata,
262            std::slice::from_ref(&snapshot),
263            &legacy_float64_fields(),
264        )
265        .unwrap();
266
267        assert_eq!(
268            legacy_batch
269                .schema()
270                .field_with_name("avg_px")
271                .unwrap()
272                .data_type(),
273            &DataType::Float64
274        );
275
276        let decoded =
277            OrderSnapshot::decode_typed_batch(legacy_batch.schema().metadata(), legacy_batch)
278                .unwrap();
279
280        assert_eq!(decoded, vec![snapshot]);
281    }
282
283    #[rstest]
284    fn test_order_snapshot_decodes_legacy_float64_null_columns() {
285        let snapshot = make_order_snapshot(None, None);
286        let metadata = snapshot.metadata();
287        let legacy_batch = encode_batch(
288            "OrderSnapshot",
289            &metadata,
290            std::slice::from_ref(&snapshot),
291            &legacy_float64_fields(),
292        )
293        .unwrap();
294        let decoded =
295            OrderSnapshot::decode_typed_batch(legacy_batch.schema().metadata(), legacy_batch)
296                .unwrap();
297
298        assert_eq!(decoded, vec![snapshot]);
299    }
300
301    fn make_position_snapshot() -> PositionSnapshot {
302        PositionSnapshot {
303            trader_id: TraderId::from("TRADER-001"),
304            strategy_id: StrategyId::from("EMA-CROSS"),
305            instrument_id: InstrumentId::from("EURUSD.SIM"),
306            position_id: PositionId::from("P-001"),
307            account_id: AccountId::from("SIM-001"),
308            opening_order_id: ClientOrderId::from("O-1"),
309            closing_order_id: Some(ClientOrderId::from("O-2")),
310            entry: OrderSide::Buy,
311            side: PositionSide::Long,
312            signed_qty: 100.0,
313            quantity: Quantity::from("100"),
314            peak_qty: Quantity::from("100"),
315            quote_currency: Currency::USD(),
316            base_currency: Some(Currency::EUR()),
317            settlement_currency: Currency::USD(),
318            avg_px_open: 1.0500,
319            avg_px_close: Some(1.0600),
320            realized_return: Some(0.0095),
321            realized_pnl: Some(Money::new(100.0, Currency::USD())),
322            unrealized_pnl: Some(Money::new(50.0, Currency::USD())),
323            commissions: vec![Money::new(2.0, Currency::USD())],
324            duration_ns: Some(DurationNanos::new(3_600_000_000_000)),
325            ts_opened: UnixNanos::from(1_000_000_000),
326            ts_closed: Some(UnixNanos::from(4_600_000_000)),
327            ts_init: UnixNanos::from(2_000_000_000),
328            ts_last: UnixNanos::from(4_600_000_000),
329            replay_state: None,
330        }
331    }
332
333    #[rstest]
334    fn test_position_snapshot_round_trip() {
335        let mut snapshot = make_position_snapshot();
336        snapshot.replay_state = Some(serde_json::json!({"fill_voids": []}));
337        let metadata = snapshot.metadata();
338        let batch =
339            PositionSnapshot::encode_batch(&metadata, std::slice::from_ref(&snapshot)).unwrap();
340        let decoded =
341            PositionSnapshot::decode_typed_batch(batch.schema().metadata(), batch).unwrap();
342
343        assert_eq!(decoded, vec![snapshot]);
344    }
345
346    #[rstest]
347    fn test_position_snapshot_round_trip_null_optionals() {
348        let mut snapshot = make_position_snapshot();
349        snapshot.closing_order_id = None;
350        snapshot.base_currency = None;
351        snapshot.avg_px_close = None;
352        snapshot.realized_return = None;
353        snapshot.realized_pnl = None;
354        snapshot.unrealized_pnl = None;
355        snapshot.duration_ns = None;
356        snapshot.ts_closed = None;
357
358        let metadata = snapshot.metadata();
359        let batch =
360            PositionSnapshot::encode_batch(&metadata, std::slice::from_ref(&snapshot)).unwrap();
361        let decoded =
362            PositionSnapshot::decode_typed_batch(batch.schema().metadata(), batch).unwrap();
363
364        assert_eq!(decoded, vec![snapshot]);
365    }
366}
nautilus_serialization/arrow/snapshot.rs

nautilus_serialization/arrow/
snapshot.rs
