nautilus_core/python/
serialization.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! (De)serialization utilities bridging Rust ↔ Python types.
17
18use std::{
19    collections::HashMap,
20    hash::{BuildHasher, Hash},
21};
22
23use indexmap::IndexMap;
24use pyo3::{
25    conversion::{FromPyObjectOwned, IntoPyObject},
26    prelude::*,
27    types::{PyAny, PyDict},
28};
29use serde::{Serialize, de::DeserializeOwned};
30
31use crate::python::to_pyvalue_err;
32
33/// Convert a Python dictionary to a Rust type that implements `DeserializeOwned`.
34///
35/// # Errors
36///
37/// Returns an error if:
38/// - The Python dictionary cannot be serialized to JSON.
39/// - The JSON string cannot be deserialized to type `T`.
40/// - The Python `json` module fails to import or execute.
41pub fn from_dict_pyo3<T>(py: Python<'_>, values: Py<PyDict>) -> Result<T, PyErr>
42where
43    T: DeserializeOwned,
44{
45    let values = values.into_any();
46    from_pyobject_pyo3(py, values.bind(py))
47}
48
49/// Convert a Python object to a Rust type that implements `DeserializeOwned`.
50///
51/// # Errors
52///
53/// Returns an error if:
54/// - The Python object cannot be serialized to JSON.
55/// - The JSON string cannot be deserialized to type `T`.
56/// - The Python `json` module fails to import or execute.
57pub fn from_pyobject_pyo3<T>(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result<T, PyErr>
58where
59    T: DeserializeOwned,
60{
61    let json_str = serialize_pyobject_pyo3(py, value)?;
62    serde_json::from_str(&json_str).map_err(to_pyvalue_err)
63}
64
65pub(super) fn serialize_pyobject_pyo3(
66    py: Python<'_>,
67    value: &Bound<'_, PyAny>,
68) -> PyResult<String> {
69    // `ensure_ascii=False` keeps non-ASCII characters as raw UTF-8 in the JSON output.
70    // Without this, `\uXXXX` escapes force `serde_json` onto the owned-string path,
71    // which `visit_str`-only visitors like `ustr::Ustr` reject as "expected a borrowed string".
72    let kwargs = PyDict::new(py);
73    kwargs.set_item("ensure_ascii", false)?;
74    PyModule::import(py, "json")?
75        .call_method("dumps", (value,), Some(&kwargs))?
76        .extract()
77}
78
79/// Convert a Rust type that implements `Serialize` to a Python dictionary.
80///
81/// # Errors
82///
83/// Returns an error if:
84/// - The Rust value cannot be serialized to JSON.
85/// - The JSON string cannot be parsed into a Python dictionary.
86/// - The Python `json` module fails to import or execute.
87pub fn to_dict_pyo3<T>(py: Python<'_>, value: &T) -> PyResult<Py<PyDict>>
88where
89    T: Serialize,
90{
91    let py_object = to_pyobject_pyo3(py, value)?;
92    let py_dict = py_object
93        .bind(py)
94        .cast::<PyDict>()
95        .map_err(Into::<PyErr>::into)?
96        .clone()
97        .unbind();
98    Ok(py_dict)
99}
100
101/// Convert a Rust type that implements `Serialize` to a Python object.
102///
103/// # Errors
104///
105/// Returns an error if:
106/// - The Rust value cannot be serialized to JSON.
107/// - The JSON string cannot be parsed into a Python object.
108/// - The Python `json` module fails to import or execute.
109pub fn to_pyobject_pyo3<T>(py: Python<'_>, value: &T) -> PyResult<Py<PyAny>>
110where
111    T: Serialize,
112{
113    let json_str = serde_json::to_string(value).map_err(to_pyvalue_err)?;
114    let py_object = PyModule::import(py, "json")?
115        .call_method("loads", (json_str,), None)?
116        .extract()?;
117    Ok(py_object)
118}
119
120/// Convert a Python mapping to an [`IndexMap`] using typed PyO3 objects when possible.
121///
122/// Falls back to the generic JSON bridge when the input is not a Python dict. Individual dict
123/// keys and values first try typed PyO3 extraction, then JSON/Serde extraction. This accepts both
124/// `{InstrumentId(...): Price(...)}` and `{"AUD/USD.SIM": "1.23456"}` style inputs.
125///
126/// # Errors
127///
128/// Returns an error if the Python object cannot be converted to the requested map type.
129pub fn indexmap_from_pyobject_pyo3<K, V>(
130    py: Python<'_>,
131    value: &Bound<'_, PyAny>,
132) -> PyResult<IndexMap<K, V>>
133where
134    K: for<'py> FromPyObjectOwned<'py> + DeserializeOwned + Eq + Hash,
135    V: for<'py> FromPyObjectOwned<'py> + DeserializeOwned,
136    IndexMap<K, V>: DeserializeOwned,
137{
138    let Ok(dict) = value.cast::<PyDict>() else {
139        return from_pyobject_pyo3(py, value);
140    };
141
142    let mut map = IndexMap::with_capacity(dict.len());
143    for (key, value) in dict.iter() {
144        map.insert(
145            extract_typed_or_json_pyo3(py, &key)?,
146            extract_typed_or_json_pyo3(py, &value)?,
147        );
148    }
149    Ok(map)
150}
151
152/// Convert a Python mapping to a [`HashMap`] using typed PyO3 objects when possible.
153///
154/// Falls back to the generic JSON bridge when the input is not a Python dict. Individual dict
155/// keys and values first try typed PyO3 extraction, then JSON/Serde extraction.
156///
157/// # Errors
158///
159/// Returns an error if the Python object cannot be converted to the requested map type.
160pub fn hashmap_from_pyobject_pyo3<K, V>(
161    py: Python<'_>,
162    value: &Bound<'_, PyAny>,
163) -> PyResult<HashMap<K, V>>
164where
165    K: for<'py> FromPyObjectOwned<'py> + DeserializeOwned + Eq + Hash,
166    V: for<'py> FromPyObjectOwned<'py> + DeserializeOwned,
167    HashMap<K, V>: DeserializeOwned,
168{
169    let Ok(dict) = value.cast::<PyDict>() else {
170        return from_pyobject_pyo3(py, value);
171    };
172
173    let mut map = HashMap::with_capacity(dict.len());
174    for (key, value) in dict.iter() {
175        map.insert(
176            extract_typed_or_json_pyo3(py, &key)?,
177            extract_typed_or_json_pyo3(py, &value)?,
178        );
179    }
180    Ok(map)
181}
182
183/// Convert an [`IndexMap`] to a Python dict using typed PyO3 keys and values.
184///
185/// # Errors
186///
187/// Returns an error if any key or value cannot be converted to Python.
188pub fn indexmap_to_pydict_pyo3<K, V>(py: Python<'_>, value: &IndexMap<K, V>) -> PyResult<Py<PyAny>>
189where
190    K: for<'py> IntoPyObject<'py> + Clone,
191    V: for<'py> IntoPyObject<'py> + Clone,
192{
193    let dict = PyDict::new(py);
194    for (key, value) in value {
195        dict.set_item(key.clone(), value.clone())?;
196    }
197    Ok(dict.into_any().unbind())
198}
199
200/// Convert a [`HashMap`] to a Python dict using typed PyO3 keys and values.
201///
202/// # Errors
203///
204/// Returns an error if any key or value cannot be converted to Python.
205pub fn hashmap_to_pydict_pyo3<K, V, S>(
206    py: Python<'_>,
207    value: &HashMap<K, V, S>,
208) -> PyResult<Py<PyAny>>
209where
210    K: for<'py> IntoPyObject<'py> + Clone,
211    V: for<'py> IntoPyObject<'py> + Clone,
212    S: BuildHasher,
213{
214    let dict = PyDict::new(py);
215    for (key, value) in value {
216        dict.set_item(key.clone(), value.clone())?;
217    }
218    Ok(dict.into_any().unbind())
219}
220
221fn extract_typed_or_json_pyo3<T>(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<T>
222where
223    T: for<'py> FromPyObjectOwned<'py> + DeserializeOwned,
224{
225    value.extract::<T>().or_else(|typed_err| {
226        let typed_err: PyErr = typed_err.into();
227        from_pyobject_pyo3(py, value).map_err(|json_err| {
228            to_pyvalue_err(format!(
229                "typed extraction failed: {typed_err}; JSON extraction failed: {json_err}"
230            ))
231        })
232    })
233}
234
235#[cfg(test)]
236mod tests {
237    use std::collections::HashMap;
238
239    use pyo3::types::PyDict;
240    use rstest::rstest;
241    use serde::{Deserialize, Serialize};
242
243    use super::*;
244    use crate::UUID4;
245
246    #[derive(Debug, PartialEq, Serialize, Deserialize)]
247    struct Payload {
248        values: HashMap<String, String>,
249    }
250
251    #[rstest]
252    fn test_from_pyobject_pyo3_preserves_non_ascii_strings() {
253        Python::initialize();
254        Python::attach(|py| {
255            let values = PyDict::new(py);
256            values.set_item("clé", "café").unwrap();
257
258            let dict = PyDict::new(py);
259            dict.set_item("values", values).unwrap();
260
261            let payload: Payload = from_pyobject_pyo3(py, dict.as_any()).unwrap();
262            assert_eq!(payload.values.get("clé").unwrap(), "café");
263        });
264    }
265
266    #[rstest]
267    fn test_typed_map_conversions() {
268        Python::initialize();
269        Python::attach(|py| {
270            let first = UUID4::from("2d89666b-1a1e-4a75-b193-4eb3b454c757");
271            let second = UUID4::from("46922ecb-4324-4e40-a56c-841e0d774cef");
272            let dict = PyDict::new(py);
273            dict.set_item(Py::new(py, first).unwrap(), 11_u64).unwrap();
274            dict.set_item(Py::new(py, second).unwrap(), 22_u64).unwrap();
275
276            let indexmap: IndexMap<UUID4, u64> =
277                indexmap_from_pyobject_pyo3(py, dict.as_any()).unwrap();
278            let hashmap: HashMap<UUID4, u64> =
279                hashmap_from_pyobject_pyo3(py, dict.as_any()).unwrap();
280            let indexmap_dict = indexmap_to_pydict_pyo3(py, &indexmap).unwrap();
281            let hashmap_dict = hashmap_to_pydict_pyo3(py, &hashmap).unwrap();
282            let restored_indexmap: IndexMap<UUID4, u64> =
283                indexmap_from_pyobject_pyo3(py, indexmap_dict.bind(py)).unwrap();
284            let restored_hashmap: HashMap<UUID4, u64> =
285                hashmap_from_pyobject_pyo3(py, hashmap_dict.bind(py)).unwrap();
286
287            assert_eq!(
288                indexmap.into_iter().collect::<Vec<_>>(),
289                [(first, 11), (second, 22)]
290            );
291            assert_eq!(hashmap, HashMap::from([(first, 11), (second, 22)]));
292            assert_eq!(
293                restored_indexmap,
294                IndexMap::from([(first, 11), (second, 22)])
295            );
296            assert_eq!(restored_hashmap, HashMap::from([(first, 11), (second, 22)]));
297        });
298    }
299}
nautilus_core/python/serialization.rs

nautilus_core/python/
serialization.rs
