pub struct PyParquetDataCatalog { /* private fields */ }Expand description
A catalog for writing data to Parquet files.
Implementations§
Source§impl PyParquetDataCatalog
impl PyParquetDataCatalog
Sourcepub fn new(
base_path: &str,
storage_options: Option<HashMap<String, String>>,
batch_size: Option<usize>,
compression: Option<u8>,
max_row_group_size: Option<usize>,
) -> Self
pub fn new( base_path: &str, storage_options: Option<HashMap<String, String>>, batch_size: Option<usize>, compression: Option<u8>, max_row_group_size: Option<usize>, ) -> Self
Create a new ParquetCatalog with the given base path and optional parameters.
§Parameters
base_path: The base path for the catalogstorage_options: Optional storage configuration for cloud backendsbatch_size: Optional batch size for processing (default: 5000)compression: Optional compression type (0=UNCOMPRESSED, 1=SNAPPY, 2=GZIP, 3=LZO, 4=BROTLI, 5=LZ4, 6=ZSTD)max_row_group_size: Optional maximum row group size (default: 5000)
Sourcepub fn write_quote_ticks(
&self,
data: Vec<QuoteTick>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_quote_ticks( &self, data: Vec<QuoteTick>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Sourcepub fn write_trade_ticks(
&self,
data: Vec<TradeTick>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_trade_ticks( &self, data: Vec<TradeTick>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Sourcepub fn write_order_book_deltas(
&self,
data: Vec<OrderBookDelta>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_order_book_deltas( &self, data: Vec<OrderBookDelta>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Sourcepub fn write_bars(
&self,
data: Vec<Bar>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_bars( &self, data: Vec<Bar>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Sourcepub fn write_order_book_depths(
&self,
data: Vec<OrderBookDepth10>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_order_book_depths( &self, data: Vec<OrderBookDepth10>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Sourcepub fn write_mark_price_updates(
&self,
data: Vec<MarkPriceUpdate>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_mark_price_updates( &self, data: Vec<MarkPriceUpdate>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Write mark price update data to Parquet files.
§Parameters
data: Vector of mark price updates to writestart: Optional start timestamp override (nanoseconds since Unix epoch)end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns
Returns the path of the created file as a string.
Sourcepub fn write_index_price_updates(
&self,
data: Vec<IndexPriceUpdate>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_index_price_updates( &self, data: Vec<IndexPriceUpdate>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Write index price update data to Parquet files.
§Parameters
data: Vector of index price updates to writestart: Optional start timestamp override (nanoseconds since Unix epoch)end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns
Returns the path of the created file as a string.
Sourcepub fn write_instruments(
&self,
data: &Bound<'_, PyAny>,
) -> PyResult<Vec<String>>
pub fn write_instruments( &self, data: &Bound<'_, PyAny>, ) -> PyResult<Vec<String>>
Write instruments to Parquet files in the catalog.
Instruments are stored under data/instruments/{instrument_id}/ using timestamp-ranged
parquet file names, allowing multiple historical versions of the same instrument to be
written across separate calls.
§Parameters
data: A Python list of instrument objects (e.g. CurrencyPair, Equity).
§Returns
Returns a list of written file paths.
Sourcepub fn instruments(
&self,
instrument_ids: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
) -> PyResult<Vec<Py<PyAny>>>
pub fn instruments( &self, instrument_ids: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, ) -> PyResult<Vec<Py<PyAny>>>
Query instruments from the catalog.
§Parameters
instrument_ids: Optional list of instrument IDs to filter by. IfNone, returns all instruments.start: Optional inclusive lower bound forts_initfiltering.end: Optional inclusive upper bound forts_initfiltering.
§Returns
Returns a list of instrument objects (e.g. CurrencyPair, Equity).
Sourcepub fn extend_file_name(
&self,
data_cls: &str,
instrument_id: Option<String>,
start: u64,
end: u64,
) -> PyResult<()>
pub fn extend_file_name( &self, data_cls: &str, instrument_id: Option<String>, start: u64, end: u64, ) -> PyResult<()>
Extend file names in the catalog with additional timestamp information.
§Parameters
data_cls: The data class nameinstrument_id: Optional instrument ID filterstart: Start timestamp (nanoseconds since Unix epoch)end: End timestamp (nanoseconds since Unix epoch)
Sourcepub fn consolidate_catalog(
&self,
start: Option<u64>,
end: Option<u64>,
ensure_contiguous_files: Option<bool>,
deduplicate: Option<bool>,
) -> PyResult<()>
pub fn consolidate_catalog( &self, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, deduplicate: Option<bool>, ) -> PyResult<()>
Consolidate all data files in the catalog within the specified time range.
§Parameters
start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)ensure_contiguous_files: Optional flag to ensure files are contiguousdeduplicate: Optional flag to deduplicate rows when combining files
Sourcepub fn consolidate_data(
&self,
type_name: &str,
instrument_id: Option<String>,
start: Option<u64>,
end: Option<u64>,
ensure_contiguous_files: Option<bool>,
deduplicate: Option<bool>,
) -> PyResult<()>
pub fn consolidate_data( &self, type_name: &str, instrument_id: Option<String>, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, deduplicate: Option<bool>, ) -> PyResult<()>
Consolidate data files for a specific data type within the specified time range.
§Parameters
type_name: The data type name to consolidateinstrument_id: Optional instrument ID filterstart: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)ensure_contiguous_files: Optional flag to ensure files are contiguousdeduplicate: Optional flag to deduplicate rows when combining files
Sourcepub fn consolidate_catalog_by_period(
&mut self,
period_nanos: Option<u64>,
start: Option<u64>,
end: Option<u64>,
ensure_contiguous_files: Option<bool>,
) -> PyResult<()>
pub fn consolidate_catalog_by_period( &mut self, period_nanos: Option<u64>, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, ) -> PyResult<()>
Consolidate all data files in the catalog by splitting them into fixed time periods.
This method identifies all leaf directories in the catalog that contain parquet files
and consolidates them by period. A leaf directory is one that contains files but no subdirectories.
This is a convenience method that effectively calls consolidate_data_by_period for all data types
and instrument IDs in the catalog.
§Parameters
period_nanos: Optional period duration for consolidation in nanoseconds. Default is 1 day (86400000000000). Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)start: Optional start timestamp for the consolidation range (nanoseconds since Unix epoch)end: Optional end timestamp for the consolidation range (nanoseconds since Unix epoch)ensure_contiguous_files: Optional flag to control file naming strategy
Sourcepub fn consolidate_data_by_period(
&mut self,
type_name: &str,
identifier: Option<String>,
period_nanos: Option<u64>,
start: Option<u64>,
end: Option<u64>,
ensure_contiguous_files: Option<bool>,
) -> PyResult<()>
pub fn consolidate_data_by_period( &mut self, type_name: &str, identifier: Option<String>, period_nanos: Option<u64>, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, ) -> PyResult<()>
Consolidate data files by splitting them into fixed time periods.
This method queries data by period and writes consolidated files immediately, using efficient period-based consolidation logic. When start/end boundaries intersect existing files, the function automatically splits those files to preserve all data.
§Parameters
type_name: The data type directory name (e.g., “quotes”, “trades”, “bars”)identifier: Optional instrument ID to consolidate. If None, consolidates all instrumentsperiod_nanos: Optional period duration for consolidation in nanoseconds. Default is 1 day (86400000000000). Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)start: Optional start timestamp for consolidation range (nanoseconds since Unix epoch)end: Optional end timestamp for consolidation range (nanoseconds since Unix epoch)ensure_contiguous_files: Optional flag to control file naming strategy
Sourcepub fn reset_all_file_names(&self) -> PyResult<()>
pub fn reset_all_file_names(&self) -> PyResult<()>
Reset all catalog file names to their canonical form.
Sourcepub fn reset_data_file_names(
&self,
data_cls: &str,
instrument_id: Option<String>,
) -> PyResult<()>
pub fn reset_data_file_names( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<()>
Reset data file names for a specific data class to their canonical form.
§Parameters
data_cls: The data class nameinstrument_id: Optional instrument ID filter
Sourcepub fn delete_catalog_range(
&mut self,
start: Option<u64>,
end: Option<u64>,
) -> PyResult<()>
pub fn delete_catalog_range( &mut self, start: Option<u64>, end: Option<u64>, ) -> PyResult<()>
Delete data within a specified time range across the entire catalog.
This method identifies all leaf directories in the catalog that contain parquet files
and deletes data within the specified time range from each directory. A leaf directory
is one that contains files but no subdirectories. This is a convenience method that
effectively calls delete_data_range for all data types and instrument IDs in the catalog.
§Parameters
start: Optional start timestamp for the deletion range (nanoseconds since Unix epoch)end: Optional end timestamp for the deletion range (nanoseconds since Unix epoch)
§Notes
- This operation permanently removes data and cannot be undone
- The deletion process handles file intersections intelligently by splitting files when they partially overlap with the deletion range
- Files completely within the deletion range are removed entirely
- Files partially overlapping the deletion range are split to preserve data outside the range
- This method is useful for bulk data cleanup operations across the entire catalog
- Empty directories are not automatically removed after deletion
Sourcepub fn delete_data_range(
&mut self,
type_name: &str,
instrument_id: Option<String>,
start: Option<u64>,
end: Option<u64>,
) -> PyResult<()>
pub fn delete_data_range( &mut self, type_name: &str, instrument_id: Option<String>, start: Option<u64>, end: Option<u64>, ) -> PyResult<()>
Delete data within a specified time range for a specific data type and instrument.
This method identifies all parquet files that intersect with the specified time range and handles them appropriately:
- Files completely within the range are deleted
- Files partially overlapping the range are split to preserve data outside the range
- The original intersecting files are removed after processing
§Parameters
type_name: The data type directory name (e.g., “quotes”, “trades”, “bars”)instrument_id: Optional instrument ID to delete data for. If None, deletes data across all instrumentsstart: Optional start timestamp for the deletion range (nanoseconds since Unix epoch)end: Optional end timestamp for the deletion range (nanoseconds since Unix epoch)
§Notes
- This operation permanently removes data and cannot be undone
- Files that partially overlap the deletion range are split to preserve data outside the range
- The method ensures data integrity by using atomic operations where possible
- Empty directories are not automatically removed after deletion
Sourcepub fn write_custom_data(
&self,
_py: Python<'_>,
data: Vec<Bound<'_, PyAny>>,
start: Option<u64>,
end: Option<u64>,
skip_disjoint_check: bool,
) -> PyResult<String>
pub fn write_custom_data( &self, _py: Python<'_>, data: Vec<Bound<'_, PyAny>>, start: Option<u64>, end: Option<u64>, skip_disjoint_check: bool, ) -> PyResult<String>
Write custom data to Parquet files.
Requires CustomData wrappers. Callers must wrap raw custom objects in
CustomData(data_type=DataType(cls, metadata=...), data=...) before writing.
Sourcepub fn list_instruments(&self, data_type: &str) -> PyResult<Vec<String>>
pub fn list_instruments(&self, data_type: &str) -> PyResult<Vec<String>>
List all instrument IDs available in the catalog for a given data type.
Sourcepub fn list_parquet_files(
&self,
data_type: &str,
instrument_id: &str,
) -> PyResult<Vec<String>>
pub fn list_parquet_files( &self, data_type: &str, instrument_id: &str, ) -> PyResult<Vec<String>>
List all Parquet files in the catalog for a given data type and instrument.
Sourcepub fn query_files(
&self,
data_cls: &str,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
) -> PyResult<Vec<String>>
pub fn query_files( &self, data_cls: &str, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, ) -> PyResult<Vec<String>>
Query files in the catalog matching the specified criteria.
§Parameters
data_cls: The data class name to queryidentifiers: Optional list of identifiers to filter by. Can be instrument_id strings (e.g., “EUR/USD.SIM”) or bar_type strings (e.g., “EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL”). For bars, partial matching is supported.start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)
§Returns
Returns a list of file paths matching the criteria.
Sourcepub fn get_missing_intervals_for_request(
&self,
start: u64,
end: u64,
data_cls: &str,
instrument_id: Option<String>,
) -> PyResult<Vec<(u64, u64)>>
pub fn get_missing_intervals_for_request( &self, start: u64, end: u64, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Vec<(u64, u64)>>
Get missing time intervals for a data request.
§Parameters
start: Start timestamp (nanoseconds since Unix epoch)end: End timestamp (nanoseconds since Unix epoch)data_cls: The data class nameinstrument_id: Optional instrument ID filter
§Returns
Returns a list of (start, end) timestamp tuples representing missing intervals.
Sourcepub fn query_first_timestamp(
&self,
data_cls: &str,
instrument_id: Option<String>,
) -> PyResult<Option<u64>>
pub fn query_first_timestamp( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Option<u64>>
Sourcepub fn query_last_timestamp(
&self,
data_cls: &str,
instrument_id: Option<String>,
) -> PyResult<Option<u64>>
pub fn query_last_timestamp( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Option<u64>>
Sourcepub fn get_intervals(
&self,
data_cls: &str,
instrument_id: Option<String>,
) -> PyResult<Vec<(u64, u64)>>
pub fn get_intervals( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Vec<(u64, u64)>>
Sourcepub fn query(
&mut self,
py: Python<'_>,
data_type: &str,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
files: Option<Vec<String>>,
optimize_file_loading: bool,
) -> PyResult<Vec<Py<PyAny>>>
pub fn query( &mut self, py: Python<'_>, data_type: &str, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, files: Option<Vec<String>>, optimize_file_loading: bool, ) -> PyResult<Vec<Py<PyAny>>>
Query Parquet files for data matching the given criteria.
Sourcepub fn query_quote_ticks(
&mut self,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<QuoteTick>>
pub fn query_quote_ticks( &mut self, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<QuoteTick>>
Query quote tick data from Parquet files.
§Parameters
identifiers: Optional list of identifiers to filter by. Can be instrument_id strings (e.g., “EUR/USD.SIM”) or bar_type strings (e.g., “EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL”). For bars, partial matching is supported.start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of QuoteTick objects matching the query criteria.
Sourcepub fn query_trade_ticks(
&mut self,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<TradeTick>>
pub fn query_trade_ticks( &mut self, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<TradeTick>>
Query trade tick data from Parquet files.
§Parameters
identifiers: Optional list of identifiers to filter by. Can be instrument_id strings (e.g., “EUR/USD.SIM”) or bar_type strings (e.g., “EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL”). For bars, partial matching is supported.start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of TradeTick objects matching the query criteria.
Sourcepub fn query_order_book_deltas(
&mut self,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<OrderBookDelta>>
pub fn query_order_book_deltas( &mut self, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<OrderBookDelta>>
Query order book delta data from Parquet files.
§Parameters
identifiers: Optional list of identifiers to filter by. Can be instrument_id strings (e.g., “EUR/USD.SIM”) or bar_type strings (e.g., “EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL”). For bars, partial matching is supported.start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of OrderBookDelta objects matching the query criteria.
Sourcepub fn query_bars(
&mut self,
identifiers: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<Bar>>
pub fn query_bars( &mut self, identifiers: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<Bar>>
Query bar data from Parquet files.
§Parameters
identifiers: Optional list of identifiers to filter by. Can be instrument_id strings (e.g., “EUR/USD.SIM”) or bar_type strings (e.g., “EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL”). For bars, partial matching is supported (e.g., “EUR/USD.SIM” will match all bar types for that instrument).start: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of Bar objects matching the query criteria.
Sourcepub fn query_order_book_depths(
&mut self,
instrument_ids: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<OrderBookDepth10>>
pub fn query_order_book_depths( &mut self, instrument_ids: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<OrderBookDepth10>>
Query order book depth data from Parquet files.
§Parameters
instrument_ids: Optional list of instrument IDs to filter bystart: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of OrderBookDepth10 objects matching the query criteria.
Sourcepub fn query_mark_price_updates(
&mut self,
instrument_ids: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<MarkPriceUpdate>>
pub fn query_mark_price_updates( &mut self, instrument_ids: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<MarkPriceUpdate>>
Query mark price update data from Parquet files.
§Parameters
instrument_ids: Optional list of instrument IDs to filter bystart: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of MarkPriceUpdate objects matching the query criteria.
Sourcepub fn query_index_price_updates(
&mut self,
instrument_ids: Option<Vec<String>>,
start: Option<u64>,
end: Option<u64>,
where_clause: Option<&str>,
) -> PyResult<Vec<IndexPriceUpdate>>
pub fn query_index_price_updates( &mut self, instrument_ids: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, where_clause: Option<&str>, ) -> PyResult<Vec<IndexPriceUpdate>>
Query index price update data from Parquet files.
§Parameters
instrument_ids: Optional list of instrument IDs to filter bystart: Optional start timestamp (nanoseconds since Unix epoch)end: Optional end timestamp (nanoseconds since Unix epoch)where_clause: Optional SQL WHERE clause for additional filtering
§Returns
Returns a vector of IndexPriceUpdate objects matching the query criteria.
Sourcepub fn list_data_types(&self) -> PyResult<Vec<String>>
pub fn list_data_types(&self) -> PyResult<Vec<String>>
List all data types available in the catalog.
§Returns
Returns a list of data type names (as directory stems) in the catalog.
Sourcepub fn list_live_runs(&self) -> PyResult<Vec<String>>
pub fn list_live_runs(&self) -> PyResult<Vec<String>>
List all live run IDs available in the catalog.
§Returns
Returns a list of live run IDs (as directory stems) in the catalog.
Sourcepub fn list_backtest_runs(&self) -> PyResult<Vec<String>>
pub fn list_backtest_runs(&self) -> PyResult<Vec<String>>
List all backtest run IDs available in the catalog.
§Returns
Returns a list of backtest run IDs (as directory stems) in the catalog.
Sourcepub fn list_backtests(&self) -> PyResult<Vec<String>>
pub fn list_backtests(&self) -> PyResult<Vec<String>>
List all backtest run instances available in the catalog.
Sourcepub fn read_live_run(
&self,
py: Python<'_>,
instance_id: &str,
) -> PyResult<Vec<Py<PyAny>>>
pub fn read_live_run( &self, py: Python<'_>, instance_id: &str, ) -> PyResult<Vec<Py<PyAny>>>
Sourcepub fn read_backtest(
&self,
py: Python<'_>,
instance_id: &str,
) -> PyResult<Vec<Py<PyAny>>>
pub fn read_backtest( &self, py: Python<'_>, instance_id: &str, ) -> PyResult<Vec<Py<PyAny>>>
Sourcepub fn convert_stream_to_data(
&mut self,
instance_id: &str,
data_cls: &str,
subdirectory: Option<&str>,
identifiers: Option<Vec<String>>,
use_ts_event_for_ts_init: bool,
) -> PyResult<()>
pub fn convert_stream_to_data( &mut self, instance_id: &str, data_cls: &str, subdirectory: Option<&str>, identifiers: Option<Vec<String>>, use_ts_event_for_ts_init: bool, ) -> PyResult<()>
Convert stream data from feather files to parquet files.
This method reads data from feather files generated during a backtest or live run and writes it to the catalog in parquet format. It’s useful for converting temporary stream data into a more permanent and queryable format.
§Parameters
instance_id: The ID of the backtest or live run instancedata_cls: The data class name (e.g., “quotes”, “trades”, “bars”)subdirectory: Optional subdirectory containing the feather files. Either “backtest” or “live” (default: “backtest”)identifiers: Optional list of identifiers to filter by (instrument IDs or bar types)use_ts_event_for_ts_init: If true, replaces thets_initcolumn withts_eventcolumn values before deserializing
§Returns
Returns nothing on success.
§Examples
# Convert backtest stream data to parquet
catalog.convert_stream_to_data(
"instance-123",
"quotes",
subdirectory="backtest"
)
# Convert live run data with identifier filtering
catalog.convert_stream_to_data(
"instance-456",
"trades",
subdirectory="live",
identifiers=["EUR/USD.SIM"]
)Trait Implementations§
Source§impl<'py> IntoPyObject<'py> for PyParquetDataCatalog
impl<'py> IntoPyObject<'py> for PyParquetDataCatalog
Source§type Target = PyParquetDataCatalog
type Target = PyParquetDataCatalog
Source§type Output = Bound<'py, <PyParquetDataCatalog as IntoPyObject<'py>>::Target>
type Output = Bound<'py, <PyParquetDataCatalog as IntoPyObject<'py>>::Target>
Source§fn into_pyobject(
self,
py: Python<'py>,
) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>
fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>
Source§impl PyClass for PyParquetDataCatalog
impl PyClass for PyParquetDataCatalog
Source§impl PyClassImpl for PyParquetDataCatalog
impl PyClassImpl for PyParquetDataCatalog
Source§const IS_BASETYPE: bool = false
const IS_BASETYPE: bool = false
Source§const IS_SUBCLASS: bool = false
const IS_SUBCLASS: bool = false
Source§const IS_MAPPING: bool = false
const IS_MAPPING: bool = false
Source§const IS_SEQUENCE: bool = false
const IS_SEQUENCE: bool = false
Source§const IS_IMMUTABLE_TYPE: bool = false
const IS_IMMUTABLE_TYPE: bool = false
Source§const RAW_DOC: &'static CStr = /// A catalog for writing data to Parquet files.
const RAW_DOC: &'static CStr = /// A catalog for writing data to Parquet files.
Source§const DOC: &'static CStr
const DOC: &'static CStr
text_signature if a constructor is defined. Read moreSource§type Layout = <<PyParquetDataCatalog as PyClassImpl>::BaseNativeType as PyClassBaseType>::Layout<PyParquetDataCatalog>
type Layout = <<PyParquetDataCatalog as PyClassImpl>::BaseNativeType as PyClassBaseType>::Layout<PyParquetDataCatalog>
Source§type ThreadChecker = NoopThreadChecker
type ThreadChecker = NoopThreadChecker
type Inventory = Pyo3MethodsInventoryForPyParquetDataCatalog
Source§type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild
type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild
Source§type BaseNativeType = PyAny
type BaseNativeType = PyAny
PyAny by default, and when you declare
#[pyclass(extends=PyDict)], it’s PyDict.fn items_iter() -> PyClassItemsIter
fn lazy_type_object() -> &'static LazyTypeObject<Self>
§fn dict_offset() -> Option<PyObjectOffset>
fn dict_offset() -> Option<PyObjectOffset>
§fn weaklist_offset() -> Option<PyObjectOffset>
fn weaklist_offset() -> Option<PyObjectOffset>
Source§impl PyClassNewTextSignature for PyParquetDataCatalog
impl PyClassNewTextSignature for PyParquetDataCatalog
const TEXT_SIGNATURE: &'static str = "(base_path, storage_options=None, batch_size=None, compression=None, max_row_group_size=None)"
Source§impl PyStubType for PyParquetDataCatalog
impl PyStubType for PyParquetDataCatalog
Source§fn type_output() -> TypeInfo
fn type_output() -> TypeInfo
§fn type_input() -> TypeInfo
fn type_input() -> TypeInfo
Source§impl PyTypeInfo for PyParquetDataCatalog
impl PyTypeInfo for PyParquetDataCatalog
Source§const NAME: &str = <Self as ::pyo3::PyClass>::NAME
const NAME: &str = <Self as ::pyo3::PyClass>::NAME
prefer using ::type_object(py).name() to get the correct runtime value
Source§const MODULE: Option<&str> = <Self as ::pyo3::impl_::pyclass::PyClassImpl>::MODULE
const MODULE: Option<&str> = <Self as ::pyo3::impl_::pyclass::PyClassImpl>::MODULE
prefer using ::type_object(py).module() to get the correct runtime value
Source§fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject
fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject
§fn type_object(py: Python<'_>) -> Bound<'_, PyType>
fn type_object(py: Python<'_>) -> Bound<'_, PyType>
§fn is_type_of(object: &Bound<'_, PyAny>) -> bool
fn is_type_of(object: &Bound<'_, PyAny>) -> bool
object is an instance of this type or a subclass of this type.§fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool
fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool
object is an instance of this type.impl DerefToPyAny for PyParquetDataCatalog
impl ExtractPyClassWithClone for PyParquetDataCatalog
Auto Trait Implementations§
impl Freeze for PyParquetDataCatalog
impl !RefUnwindSafe for PyParquetDataCatalog
impl Send for PyParquetDataCatalog
impl Sync for PyParquetDataCatalog
impl Unpin for PyParquetDataCatalog
impl UnsafeUnpin for PyParquetDataCatalog
impl !UnwindSafe for PyParquetDataCatalog
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more§impl<'py, T> IntoPyObjectExt<'py> for Twhere
T: IntoPyObject<'py>,
impl<'py, T> IntoPyObjectExt<'py> for Twhere
T: IntoPyObject<'py>,
§fn into_bound_py_any(self, py: Python<'py>) -> Result<Bound<'py, PyAny>, PyErr>
fn into_bound_py_any(self, py: Python<'py>) -> Result<Bound<'py, PyAny>, PyErr>
self into an owned Python object, dropping type information.§fn into_py_any(self, py: Python<'py>) -> Result<Py<PyAny>, PyErr>
fn into_py_any(self, py: Python<'py>) -> Result<Py<PyAny>, PyErr>
self into an owned Python object, dropping type information and unbinding it
from the 'py lifetime.§fn into_pyobject_or_pyerr(self, py: Python<'py>) -> Result<Self::Output, PyErr>
fn into_pyobject_or_pyerr(self, py: Python<'py>) -> Result<Self::Output, PyErr>
self into a Python object. Read more§impl<'py, T> IntoPyObjectNautilusExt<'py> for Twhere
T: IntoPyObjectExt<'py>,
impl<'py, T> IntoPyObjectNautilusExt<'py> for Twhere
T: IntoPyObjectExt<'py>,
§fn into_py_any_unwrap(self, py: Python<'py>) -> Py<PyAny>
fn into_py_any_unwrap(self, py: Python<'py>) -> Py<PyAny>
§impl<T> PolicyExt for Twhere
T: ?Sized,
impl<T> PolicyExt for Twhere
T: ?Sized,
§impl<T> PyErrArguments for T
impl<T> PyErrArguments for T
§impl<T> PyTypeCheck for Twhere
T: PyTypeInfo,
impl<T> PyTypeCheck for Twhere
T: PyTypeInfo,
§const NAME: &'static str = T::NAME
const NAME: &'static str = T::NAME
Use ::classinfo_object() instead and format the type name at runtime. Note that using built-in cast features is often better than manual PyTypeCheck usage.
§fn type_check(object: &Bound<'_, PyAny>) -> bool
fn type_check(object: &Bound<'_, PyAny>) -> bool
§fn classinfo_object(py: Python<'_>) -> Bound<'_, PyAny>
fn classinfo_object(py: Python<'_>) -> Bound<'_, PyAny>
isinstance and issubclass function. Read more