Struct DataBackendSession 

pub struct DataBackendSession {
    pub chunk_size: usize,
    pub runtime: Arc<Runtime>,
    /* private fields */
}

Provides a DataFusion session and registers DataFusion queries.

The session is used to register data sources and make queries on them. A query returns a Chunk of Arrow records. It is decoded and converted into a Vec of data by types that implement [DecodeDataFromRecordBatch].

Fields

chunk_size: usize

runtime: Arc<Runtime>

Implementations

impl DataBackendSession

pub fn new(chunk_size: usize) -> Self

Creates a new DataBackendSession instance.

pub fn register_object_store( &mut self, url: &Url, object_store: Arc<dyn ObjectStore>, )

Register an object store with the session context

pub fn register_object_store_from_uri( &mut self, uri: &str, storage_options: Option<AHashMap<String, String>>, ) -> Result<()>

Register an object store with the session context from a URI with optional storage options

pub fn write_data<T: EncodeToRecordBatch>( data: &[T], metadata: &AHashMap<String, String>, stream: &mut dyn WriteStream, ) -> Result<(), DataStreamingError>

pub fn add_file<T>( &mut self, table_name: &str, file_path: &str, sql_query: Option<&str>, custom_type_name: Option<&str>, ) -> Result<()>

where T: DecodeDataFromRecordBatch,

Registers a Parquet file and adds a batch stream for decoding.

The caller must specify T to indicate the kind of data expected. table_name is the logical name for queries; file_path is the Parquet path; sql_query defaults to SELECT * FROM {table_name} ORDER BY ts_init if None.

When custom_type_name is Some, it is merged into each batch’s schema metadata before decoding (as type_name). Use this for custom data when Parquet/DataFusion does not preserve schema metadata so the decoder can look up the type in the registry.

The file data must be ordered by the ts_init in ascending order for this to work correctly.

pub fn collect_query_batches( &mut self, table_name: &str, file_path: &str, sql_query: Option<&str>, ) -> Result<Vec<RecordBatch>>

Registers a Parquet file and executes a query, returning the raw record batches.

pub fn get_query_result(&mut self) -> QueryResult

pub fn clear_registered_tables(&mut self)

Clears all registered tables and batch streams.

This is useful when the underlying files have changed and we need to re-register tables with updated data.

Trait Implementations

impl<'py> IntoPyObject<'py> for DataBackendSession

type Target = DataBackendSession

The Python output type

type Output = Bound<'py, <DataBackendSession as IntoPyObject<'py>>::Target>

The smart pointer type to use.

type Error = PyErr

The type returned in the event of a conversion error.

fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>

Performs the conversion.

impl PyClass for DataBackendSession

const NAME: &str = "DataBackendSession"

Name of the class.

type Frozen = False

Whether the pyclass is frozen.

impl PyClassImpl for DataBackendSession

const MODULE: Option<&str>

Module which the class will be associated with.

const IS_BASETYPE: bool = false

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

const IS_IMMUTABLE_TYPE: bool = false

#[pyclass(immutable_type)]

const RAW_DOC: &'static CStr = /// Provides a DataFusion session and registers DataFusion queries. /// /// The session is used to register data sources and make queries on them. A /// query returns a Chunk of Arrow records. It is decoded and converted into /// a Vec of data by types that implement [`DecodeDataFromRecordBatch`].

Docstring for the class provided on the struct or enum.

const DOC: &'static CStr

Fully rendered class doc, including the text_signature if a constructor is defined.

type Layout = <<DataBackendSession as PyClassImpl>::BaseNativeType as PyClassBaseType>::Layout<DataBackendSession>

Description of how this class is laid out in memory

type BaseType = PyAny

Base class

type ThreadChecker = ThreadCheckerImpl

This handles following two situations:

type Inventory = Pyo3MethodsInventoryForDataBackendSession

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<PyObjectOffset>

Used to provide the dictoffset slot (equivalent to tp_dictoffset)

fn weaklist_offset() -> Option<PyObjectOffset>

Used to provide the weaklistoffset slot (equivalent to tp_weaklistoffset

impl PyClassNewTextSignature for DataBackendSession

const TEXT_SIGNATURE: &'static str = "(chunk_size=10000)"

impl PyStubType for DataBackendSession

fn type_output() -> TypeInfo

The type to be used in the output signature, i.e. return type of the Python function or methods.

fn type_input() -> TypeInfo

The type to be used in the input signature, i.e. the arguments of the Python function or methods.

impl PyTypeInfo for DataBackendSession

const NAME: &str = <Self as ::pyo3::PyClass>::NAME

👎Deprecated since 0.28.0:

prefer using ::type_object(py).name() to get the correct runtime value

Class name.

const MODULE: Option<&str> = <Self as ::pyo3::impl_::pyclass::PyClassImpl>::MODULE

👎Deprecated since 0.28.0:

prefer using ::type_object(py).module() to get the correct runtime value

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.

impl DerefToPyAny for DataBackendSession

impl ExtractPyClassWithClone for DataBackendSession