1use std::{ffi::c_char, num::NonZeroUsize};
17
18use databento::dbn;
19use nautilus_core::{UnixNanos, datetime::NANOSECONDS_IN_SECOND};
20use nautilus_model::{
21 data::{
22 Bar, BarSpecification, BarType, BookOrder, DEPTH10_LEN, Data, InstrumentStatus,
23 OrderBookDelta, OrderBookDepth10, QuoteTick, TradeTick,
24 },
25 enums::{AggregationSource, BarAggregation, FromU16, MarketStatusAction, OrderSide, PriceType},
26 identifiers::{InstrumentId, TradeId},
27};
28
29use super::primitives::{
30 decode_price_or_undef, decode_quantity, parse_aggressor_side, parse_book_action,
31 parse_optional_bool, parse_order_side, parse_status_reason, parse_status_trading_event,
32};
33
34const STEP_ONE: NonZeroUsize = NonZeroUsize::new(1).unwrap();
35
36const BAR_SPEC_1S: BarSpecification = BarSpecification {
37 step: STEP_ONE,
38 aggregation: BarAggregation::Second,
39 price_type: PriceType::Last,
40};
41const BAR_SPEC_1M: BarSpecification = BarSpecification {
42 step: STEP_ONE,
43 aggregation: BarAggregation::Minute,
44 price_type: PriceType::Last,
45};
46const BAR_SPEC_1H: BarSpecification = BarSpecification {
47 step: STEP_ONE,
48 aggregation: BarAggregation::Hour,
49 price_type: PriceType::Last,
50};
51const BAR_SPEC_1D: BarSpecification = BarSpecification {
52 step: STEP_ONE,
53 aggregation: BarAggregation::Day,
54 price_type: PriceType::Last,
55};
56
57pub(super) const BAR_CLOSE_ADJUSTMENT_1S: u64 = NANOSECONDS_IN_SECOND;
58pub(super) const BAR_CLOSE_ADJUSTMENT_1M: u64 = NANOSECONDS_IN_SECOND * 60;
59pub(super) const BAR_CLOSE_ADJUSTMENT_1H: u64 = NANOSECONDS_IN_SECOND * 60 * 60;
60pub(super) const BAR_CLOSE_ADJUSTMENT_1D: u64 = NANOSECONDS_IN_SECOND * 60 * 60 * 24;
61
62const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
64const FNV_PRIME: u64 = 0x0100_0000_01b3;
65
66fn fnv1a_mix_bytes(hash: &mut u64, bytes: &[u8]) {
67 for &byte in bytes {
68 *hash ^= u64::from(byte);
69 *hash = hash.wrapping_mul(FNV_PRIME);
70 }
71}
72
73fn fnv1a_finish_field(hash: &mut u64) {
74 *hash ^= 0xff;
75 *hash = hash.wrapping_mul(FNV_PRIME);
76}
77
78fn fnv1a_mix(hash: &mut u64, bytes: &[u8]) {
79 fnv1a_mix_bytes(hash, bytes);
80 fnv1a_finish_field(hash);
81}
82
83pub(super) fn derive_cmbp_trade_id(
89 instrument_id: InstrumentId,
90 ts_event: u64,
91 ts_recv: u64,
92 price: i64,
93 size: u32,
94 side: c_char,
95) -> TradeId {
96 let mut hash: u64 = FNV_OFFSET_BASIS;
97 fnv1a_mix_bytes(&mut hash, instrument_id.symbol.as_str().as_bytes());
98 fnv1a_mix_bytes(&mut hash, b".");
99 fnv1a_mix_bytes(&mut hash, instrument_id.venue.as_str().as_bytes());
100 fnv1a_finish_field(&mut hash);
101 fnv1a_mix(&mut hash, &ts_event.to_le_bytes());
102 fnv1a_mix(&mut hash, &ts_recv.to_le_bytes());
103 fnv1a_mix(&mut hash, &price.to_le_bytes());
104 fnv1a_mix(&mut hash, &size.to_le_bytes());
105 fnv1a_mix(&mut hash, &[side as u8]);
106 trade_id_from_hash(hash)
107}
108
109fn trade_id_from_hash(hash: u64) -> TradeId {
110 const HEX: &[u8; 16] = b"0123456789abcdef";
111
112 let mut bytes = [0u8; 16];
113 let mut value = hash;
114 for byte in bytes.iter_mut().rev() {
115 *byte = HEX[(value & 0x0f) as usize];
116 value >>= 4;
117 }
118
119 TradeId::from_bytes(&bytes).expect("16 lowercase hex bytes are valid TradeId")
120}
121
122#[inline(always)]
123#[must_use]
124pub(super) fn is_trade_msg(action: c_char) -> bool {
125 action as u8 as char == 'T'
126}
127
128#[inline(always)]
133#[must_use]
134fn has_valid_bid_ask(bid_px: i64, ask_px: i64) -> bool {
135 bid_px != i64::MAX && ask_px != i64::MAX
136}
137
138pub fn decode_mbo_msg(
147 msg: &dbn::MboMsg,
148 instrument_id: InstrumentId,
149 price_precision: u8,
150 ts_init: Option<UnixNanos>,
151 include_trades: bool,
152) -> anyhow::Result<(Option<OrderBookDelta>, Option<TradeTick>)> {
153 let side = parse_order_side(msg.side);
154 if is_trade_msg(msg.action) {
155 if include_trades && msg.size > 0 {
156 let price = decode_price_or_undef(msg.price, price_precision);
157 let size = decode_quantity(msg.size as u64);
158 let aggressor_side = parse_aggressor_side(msg.side);
159 let trade_id = TradeId::new(itoa::Buffer::new().format(msg.sequence));
160 let ts_event = msg.ts_recv.into();
161 let ts_init = ts_init.unwrap_or(ts_event);
162
163 let trade = TradeTick::new(
164 instrument_id,
165 price,
166 size,
167 aggressor_side,
168 trade_id,
169 ts_event,
170 ts_init,
171 );
172 return Ok((None, Some(trade)));
173 }
174
175 return Ok((None, None));
176 }
177
178 let action = parse_book_action(msg.action)?;
179 let price = decode_price_or_undef(msg.price, price_precision);
180 let size = decode_quantity(msg.size as u64);
181 let order = BookOrder::new(side, price, size, msg.order_id);
182
183 let ts_event = msg.ts_recv.into();
184 let ts_init = ts_init.unwrap_or(ts_event);
185
186 let delta = OrderBookDelta::new(
187 instrument_id,
188 action,
189 order,
190 msg.flags.raw(),
191 msg.sequence.into(),
192 ts_event,
193 ts_init,
194 );
195
196 Ok((Some(delta), None))
197}
198
199pub fn decode_trade_msg(
205 msg: &dbn::TradeMsg,
206 instrument_id: InstrumentId,
207 price_precision: u8,
208 ts_init: Option<UnixNanos>,
209) -> anyhow::Result<TradeTick> {
210 let ts_event = msg.ts_recv.into();
211 let ts_init = ts_init.unwrap_or(ts_event);
212
213 let trade = TradeTick::new(
214 instrument_id,
215 decode_price_or_undef(msg.price, price_precision),
216 decode_quantity(msg.size as u64),
217 parse_aggressor_side(msg.side),
218 TradeId::new(itoa::Buffer::new().format(msg.sequence)),
219 ts_event,
220 ts_init,
221 );
222
223 Ok(trade)
224}
225
226pub fn decode_tbbo_msg(
235 msg: &dbn::TbboMsg,
236 instrument_id: InstrumentId,
237 price_precision: u8,
238 ts_init: Option<UnixNanos>,
239) -> anyhow::Result<(Option<QuoteTick>, TradeTick)> {
240 let top_level = &msg.levels[0];
241 let ts_event = msg.ts_recv.into();
242 let ts_init = ts_init.unwrap_or(ts_event);
243
244 let maybe_quote = if has_valid_bid_ask(top_level.bid_px, top_level.ask_px) {
245 Some(QuoteTick::new(
246 instrument_id,
247 decode_price_or_undef(top_level.bid_px, price_precision),
248 decode_price_or_undef(top_level.ask_px, price_precision),
249 decode_quantity(top_level.bid_sz as u64),
250 decode_quantity(top_level.ask_sz as u64),
251 ts_event,
252 ts_init,
253 ))
254 } else {
255 None
256 };
257
258 let trade = TradeTick::new(
259 instrument_id,
260 decode_price_or_undef(msg.price, price_precision),
261 decode_quantity(msg.size as u64),
262 parse_aggressor_side(msg.side),
263 TradeId::new(itoa::Buffer::new().format(msg.sequence)),
264 ts_event,
265 ts_init,
266 );
267
268 Ok((maybe_quote, trade))
269}
270
271pub fn decode_mbp1_msg(
279 msg: &dbn::Mbp1Msg,
280 instrument_id: InstrumentId,
281 price_precision: u8,
282 ts_init: Option<UnixNanos>,
283 include_trades: bool,
284) -> anyhow::Result<(Option<QuoteTick>, Option<TradeTick>)> {
285 let top_level = &msg.levels[0];
286 let ts_event = msg.ts_recv.into();
287 let ts_init = ts_init.unwrap_or(ts_event);
288
289 let maybe_quote = if has_valid_bid_ask(top_level.bid_px, top_level.ask_px) {
290 Some(QuoteTick::new(
291 instrument_id,
292 decode_price_or_undef(top_level.bid_px, price_precision),
293 decode_price_or_undef(top_level.ask_px, price_precision),
294 decode_quantity(top_level.bid_sz as u64),
295 decode_quantity(top_level.ask_sz as u64),
296 ts_event,
297 ts_init,
298 ))
299 } else {
300 None
301 };
302
303 let maybe_trade = if include_trades && is_trade_msg(msg.action) {
304 Some(TradeTick::new(
305 instrument_id,
306 decode_price_or_undef(msg.price, price_precision),
307 decode_quantity(msg.size as u64),
308 parse_aggressor_side(msg.side),
309 TradeId::new(itoa::Buffer::new().format(msg.sequence)),
310 ts_event,
311 ts_init,
312 ))
313 } else {
314 None
315 };
316
317 Ok((maybe_quote, maybe_trade))
318}
319
320pub fn decode_bbo_msg(
328 msg: &dbn::BboMsg,
329 instrument_id: InstrumentId,
330 price_precision: u8,
331 ts_init: Option<UnixNanos>,
332) -> anyhow::Result<Option<QuoteTick>> {
333 let top_level = &msg.levels[0];
334 if !has_valid_bid_ask(top_level.bid_px, top_level.ask_px) {
335 return Ok(None);
336 }
337
338 let ts_event = msg.ts_recv.into();
339 let ts_init = ts_init.unwrap_or(ts_event);
340
341 let quote = QuoteTick::new(
342 instrument_id,
343 decode_price_or_undef(top_level.bid_px, price_precision),
344 decode_price_or_undef(top_level.ask_px, price_precision),
345 decode_quantity(top_level.bid_sz as u64),
346 decode_quantity(top_level.ask_sz as u64),
347 ts_event,
348 ts_init,
349 );
350
351 Ok(Some(quote))
352}
353
354pub fn decode_mbp10_msg(
360 msg: &dbn::Mbp10Msg,
361 instrument_id: InstrumentId,
362 price_precision: u8,
363 ts_init: Option<UnixNanos>,
364) -> anyhow::Result<OrderBookDepth10> {
365 let mut bids = [BookOrder::default(); DEPTH10_LEN];
366 let mut asks = [BookOrder::default(); DEPTH10_LEN];
367 let mut bid_counts = [0u32; DEPTH10_LEN];
368 let mut ask_counts = [0u32; DEPTH10_LEN];
369
370 for (index, level) in msg.levels.iter().enumerate() {
371 if level.bid_px != i64::MAX {
374 bids[index] = BookOrder::new(
375 OrderSide::Buy,
376 decode_price_or_undef(level.bid_px, price_precision),
377 decode_quantity(level.bid_sz as u64),
378 0,
379 );
380 }
381
382 if level.ask_px != i64::MAX {
383 asks[index] = BookOrder::new(
384 OrderSide::Sell,
385 decode_price_or_undef(level.ask_px, price_precision),
386 decode_quantity(level.ask_sz as u64),
387 0,
388 );
389 }
390
391 bid_counts[index] = level.bid_ct;
392 ask_counts[index] = level.ask_ct;
393 }
394
395 let ts_event = msg.ts_recv.into();
396 let ts_init = ts_init.unwrap_or(ts_event);
397
398 let depth = OrderBookDepth10::new(
399 instrument_id,
400 bids,
401 asks,
402 bid_counts,
403 ask_counts,
404 msg.flags.raw(),
405 msg.sequence.into(),
406 ts_event,
407 ts_init,
408 );
409
410 Ok(depth)
411}
412
413pub fn decode_cmbp1_msg(
422 msg: &dbn::Cmbp1Msg,
423 instrument_id: InstrumentId,
424 price_precision: u8,
425 ts_init: Option<UnixNanos>,
426 include_trades: bool,
427) -> anyhow::Result<(Option<QuoteTick>, Option<TradeTick>)> {
428 let top_level = &msg.levels[0];
429 let ts_event = msg.ts_recv.into();
430 let ts_init = ts_init.unwrap_or(ts_event);
431
432 let maybe_quote = if has_valid_bid_ask(top_level.bid_px, top_level.ask_px) {
433 Some(QuoteTick::new(
434 instrument_id,
435 decode_price_or_undef(top_level.bid_px, price_precision),
436 decode_price_or_undef(top_level.ask_px, price_precision),
437 decode_quantity(top_level.bid_sz as u64),
438 decode_quantity(top_level.ask_sz as u64),
439 ts_event,
440 ts_init,
441 ))
442 } else {
443 None
444 };
445
446 let maybe_trade = if include_trades && is_trade_msg(msg.action) {
447 let trade_id = derive_cmbp_trade_id(
449 instrument_id,
450 msg.hd.ts_event,
451 msg.ts_recv,
452 msg.price,
453 msg.size,
454 msg.side,
455 );
456 Some(TradeTick::new(
457 instrument_id,
458 decode_price_or_undef(msg.price, price_precision),
459 decode_quantity(msg.size as u64),
460 parse_aggressor_side(msg.side),
461 trade_id,
462 ts_event,
463 ts_init,
464 ))
465 } else {
466 None
467 };
468
469 Ok((maybe_quote, maybe_trade))
470}
471
472pub fn decode_cbbo_msg(
480 msg: &dbn::CbboMsg,
481 instrument_id: InstrumentId,
482 price_precision: u8,
483 ts_init: Option<UnixNanos>,
484) -> anyhow::Result<Option<QuoteTick>> {
485 let top_level = &msg.levels[0];
486 if !has_valid_bid_ask(top_level.bid_px, top_level.ask_px) {
487 return Ok(None);
488 }
489
490 let ts_event = msg.ts_recv.into();
491 let ts_init = ts_init.unwrap_or(ts_event);
492
493 let quote = QuoteTick::new(
494 instrument_id,
495 decode_price_or_undef(top_level.bid_px, price_precision),
496 decode_price_or_undef(top_level.ask_px, price_precision),
497 decode_quantity(top_level.bid_sz as u64),
498 decode_quantity(top_level.ask_sz as u64),
499 ts_event,
500 ts_init,
501 );
502
503 Ok(Some(quote))
504}
505
506pub fn decode_tcbbo_msg(
515 msg: &dbn::TcbboMsg,
516 instrument_id: InstrumentId,
517 price_precision: u8,
518 ts_init: Option<UnixNanos>,
519) -> anyhow::Result<(Option<QuoteTick>, TradeTick)> {
520 let (maybe_quote, maybe_trade) =
521 decode_cmbp1_msg(msg, instrument_id, price_precision, ts_init, true)?;
522 let trade = maybe_trade.ok_or_else(|| {
523 anyhow::anyhow!(
524 "Invalid `TcbboMsg`: expected trade action, was {}",
525 msg.action as u8 as char
526 )
527 })?;
528
529 Ok((maybe_quote, trade))
530}
531
532pub fn decode_bar_type(
536 msg: &dbn::OhlcvMsg,
537 instrument_id: InstrumentId,
538) -> anyhow::Result<BarType> {
539 let bar_type = match msg.hd.rtype {
540 32 => {
541 BarType::new(instrument_id, BAR_SPEC_1S, AggregationSource::External)
543 }
544 33 => {
545 BarType::new(instrument_id, BAR_SPEC_1M, AggregationSource::External)
547 }
548 34 => {
549 BarType::new(instrument_id, BAR_SPEC_1H, AggregationSource::External)
551 }
552 35 => {
553 BarType::new(instrument_id, BAR_SPEC_1D, AggregationSource::External)
555 }
556 36 => {
557 BarType::new(instrument_id, BAR_SPEC_1D, AggregationSource::External)
559 }
560 _ => anyhow::bail!(
561 "`rtype` is not a supported bar aggregation, was {}",
562 msg.hd.rtype
563 ),
564 };
565
566 Ok(bar_type)
567}
568
569pub fn decode_ts_event_adjustment(msg: &dbn::OhlcvMsg) -> anyhow::Result<UnixNanos> {
573 let adjustment = match msg.hd.rtype {
574 32 => {
575 BAR_CLOSE_ADJUSTMENT_1S
577 }
578 33 => {
579 BAR_CLOSE_ADJUSTMENT_1M
581 }
582 34 => {
583 BAR_CLOSE_ADJUSTMENT_1H
585 }
586 35 | 36 => {
587 BAR_CLOSE_ADJUSTMENT_1D
589 }
590 _ => anyhow::bail!(
591 "`rtype` is not a supported bar aggregation, was {}",
592 msg.hd.rtype
593 ),
594 };
595
596 Ok(adjustment.into())
597}
598
599pub fn decode_ohlcv_msg(
603 msg: &dbn::OhlcvMsg,
604 instrument_id: InstrumentId,
605 price_precision: u8,
606 ts_init: Option<UnixNanos>,
607 timestamp_on_close: bool,
608) -> anyhow::Result<Bar> {
609 let bar_type = decode_bar_type(msg, instrument_id)?;
610 let ts_event_adjustment = decode_ts_event_adjustment(msg)?;
611
612 let ts_event_raw = msg.hd.ts_event.into();
613 let ts_close = ts_event_raw + ts_event_adjustment;
614 let ts_init = ts_init.unwrap_or(ts_close); let ts_event = if timestamp_on_close {
617 ts_close
618 } else {
619 ts_event_raw
620 };
621
622 let bar = Bar::new(
623 bar_type,
624 decode_price_or_undef(msg.open, price_precision),
625 decode_price_or_undef(msg.high, price_precision),
626 decode_price_or_undef(msg.low, price_precision),
627 decode_price_or_undef(msg.close, price_precision),
628 decode_quantity(msg.volume),
629 ts_event,
630 ts_init,
631 );
632
633 Ok(bar)
634}
635
636pub fn decode_status_msg(
642 msg: &dbn::StatusMsg,
643 instrument_id: InstrumentId,
644 ts_init: Option<UnixNanos>,
645) -> anyhow::Result<InstrumentStatus> {
646 let ts_event = msg.hd.ts_event.into();
647 let ts_init = ts_init.unwrap_or(ts_event);
648
649 let action = MarketStatusAction::from_u16(msg.action)
650 .ok_or_else(|| anyhow::anyhow!("Invalid `MarketStatusAction` value: {}", msg.action))?;
651
652 let status = InstrumentStatus::new(
653 instrument_id,
654 action,
655 ts_event,
656 ts_init,
657 parse_status_reason(msg.reason)?,
658 parse_status_trading_event(msg.trading_event)?,
659 parse_optional_bool(msg.is_trading),
660 parse_optional_bool(msg.is_quoting),
661 parse_optional_bool(msg.is_short_sell_restricted),
662 );
663
664 Ok(status)
665}
666
667pub fn decode_record(
671 record: &dbn::RecordRef,
672 instrument_id: InstrumentId,
673 price_precision: u8,
674 ts_init: Option<UnixNanos>,
675 include_trades: bool,
676 bars_timestamp_on_close: bool,
677) -> anyhow::Result<(Option<Data>, Option<Data>)> {
678 let result = if let Some(msg) = record.get::<dbn::MboMsg>() {
679 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
680 let result = decode_mbo_msg(
681 msg,
682 instrument_id,
683 price_precision,
684 Some(ts_init),
685 include_trades,
686 )?;
687
688 match result {
689 (Some(delta), None) => (Some(Data::Delta(delta)), None),
690 (None, Some(trade)) => (Some(Data::Trade(trade)), None),
691 (None, None) => (None, None),
692 _ => anyhow::bail!("Invalid `MboMsg` parsing combination"),
693 }
694 } else if let Some(msg) = record.get::<dbn::TradeMsg>() {
695 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
696 let trade = decode_trade_msg(msg, instrument_id, price_precision, Some(ts_init))?;
697 (Some(Data::Trade(trade)), None)
698 } else if let Some(msg) = record.get::<dbn::Mbp1Msg>() {
699 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
700 let (maybe_quote, maybe_trade) = decode_mbp1_msg(
701 msg,
702 instrument_id,
703 price_precision,
704 Some(ts_init),
705 include_trades,
706 )?;
707 (maybe_quote.map(Data::Quote), maybe_trade.map(Data::Trade))
708 } else if let Some(msg) = record.get::<dbn::Bbo1SMsg>() {
709 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
710 let maybe_quote = decode_bbo_msg(msg, instrument_id, price_precision, Some(ts_init))?;
711 (maybe_quote.map(Data::Quote), None)
712 } else if let Some(msg) = record.get::<dbn::Bbo1MMsg>() {
713 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
714 let maybe_quote = decode_bbo_msg(msg, instrument_id, price_precision, Some(ts_init))?;
715 (maybe_quote.map(Data::Quote), None)
716 } else if let Some(msg) = record.get::<dbn::Mbp10Msg>() {
717 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
718 let depth = decode_mbp10_msg(msg, instrument_id, price_precision, Some(ts_init))?;
719 (Some(Data::from(depth)), None)
720 } else if let Some(msg) = record.get::<dbn::OhlcvMsg>() {
721 let bar = decode_ohlcv_msg(
724 msg,
725 instrument_id,
726 price_precision,
727 ts_init,
728 bars_timestamp_on_close,
729 )?;
730 (Some(Data::Bar(bar)), None)
731 } else if let Some(msg) = record.get::<dbn::Cmbp1Msg>() {
732 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
733 if msg.hd.rtype == dbn::enums::rtype::TCBBO {
734 let (maybe_quote, trade) =
735 decode_tcbbo_msg(msg, instrument_id, price_precision, Some(ts_init))?;
736 (maybe_quote.map(Data::Quote), Some(Data::Trade(trade)))
737 } else {
738 let (maybe_quote, maybe_trade) = decode_cmbp1_msg(
739 msg,
740 instrument_id,
741 price_precision,
742 Some(ts_init),
743 include_trades,
744 )?;
745 (maybe_quote.map(Data::Quote), maybe_trade.map(Data::Trade))
746 }
747 } else if let Some(msg) = record.get::<dbn::TbboMsg>() {
748 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
750 let (maybe_quote, trade) =
751 decode_tbbo_msg(msg, instrument_id, price_precision, Some(ts_init))?;
752 (maybe_quote.map(Data::Quote), Some(Data::Trade(trade)))
753 } else if let Some(msg) = record.get::<dbn::CbboMsg>() {
754 let ts_init = determine_timestamp(ts_init, msg.ts_recv.into());
755 let maybe_quote = decode_cbbo_msg(msg, instrument_id, price_precision, Some(ts_init))?;
756 (maybe_quote.map(Data::Quote), None)
757 } else {
758 anyhow::bail!("DBN message type is not currently supported")
759 };
760
761 Ok(result)
762}
763
764const fn determine_timestamp(ts_init: Option<UnixNanos>, msg_timestamp: UnixNanos) -> UnixNanos {
765 match ts_init {
766 Some(ts_init) => ts_init,
767 None => msg_timestamp,
768 }
769}