diff --git a/cpp/csp/python/CMakeLists.txt b/cpp/csp/python/CMakeLists.txt index bf7770005..a1f37a432 100644 --- a/cpp/csp/python/CMakeLists.txt +++ b/cpp/csp/python/CMakeLists.txt @@ -17,7 +17,9 @@ add_library(csptypesimpl PyCspType.cpp PyStruct.cpp PyStructToJson.cpp - PyStructToDict.cpp) + PyStructToDict.cpp + PyStructFromDict.cpp + PyStructFromJson.cpp) set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}") target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1) @@ -48,7 +50,9 @@ set(CSPIMPL_PUBLIC_HEADERS PyOutputProxy.h PyConstants.h PyStructToJson.h - PyStructToDict.h) + PyStructToDict.h + PyStructFromDict.h + PyStructFromJson.h) add_library(cspimpl SHARED cspimpl.cpp diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp index acac60c16..d94f7a009 100644 --- a/cpp/csp/python/PyStruct.cpp +++ b/cpp/csp/python/PyStruct.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1052,6 +1055,40 @@ PyObject * PyStruct_to_dict( PyStruct * self, PyObject * args, PyObject * kwargs CSP_RETURN_NULL; } +PyObject * PyStruct_from_dict( PyStructMeta * cls, PyObject * args, PyObject * kwargs) { + CSP_BEGIN_METHOD; + + PyObject* dict = NULL; + + if (!PyArg_ParseTuple(args, "O:from_dict", &dict)) { + return NULL; + } + auto& struct_meta = cls->structMeta; + return toPython(structFromDict(struct_meta, dict)); + + CSP_RETURN_NULL; +} + +PyObject * PyStruct_from_json( PyStructMeta * cls, PyObject * args, PyObject * kwargs ) +{ + CSP_BEGIN_METHOD; + + const char * json = nullptr; + Py_ssize_t len = 0; + if( !PyArg_ParseTuple( args, "s#:from_json", &json, &len ) ) + return NULL; + + //kParseNanAndInfFlag to match to_json, which writes NaN / Inf for doubles + rapidjson::Document doc; + rapidjson::ParseResult ok = doc.Parse( json, len ); + if( !ok ) + CSP_THROW( ValueError, "Failed to parse json: " << rapidjson::GetParseError_En( ok.Code() ) ); + + return toPython( structFromJson( cls -> structMeta, doc ) ); + + CSP_RETURN_NULL; +} + PyObject * PyStruct_to_json( PyStruct * self, PyObject * args, PyObject * kwargs ) { CSP_BEGIN_METHOD; @@ -1087,6 +1124,8 @@ static PyMethodDef PyStruct_methods[] = { { "all_fields_set", (PyCFunction) PyStruct_all_fields_set, METH_NOARGS, "return true if all fields on the struct are set" }, { "to_dict", (PyCFunction) PyStruct_to_dict, METH_VARARGS | METH_KEYWORDS, "return a python dict of the struct by recursively converting struct members into python dicts" }, { "to_json", (PyCFunction) PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, "return a json string of the struct by recursively converting struct members into json format" }, + { "from_dict", (PyCFunction) PyStruct_from_dict, METH_VARARGS | METH_KEYWORDS | METH_CLASS, "return a struct by recursively reading the values from a python dictionary"}, + { "from_json", (PyCFunction) PyStruct_from_json, METH_VARARGS | METH_KEYWORDS | METH_CLASS, "return a struct by recursively reading the values from a json string"}, { NULL} }; diff --git a/cpp/csp/python/PyStructFromDict.cpp b/cpp/csp/python/PyStructFromDict.cpp new file mode 100644 index 000000000..3b180c46a --- /dev/null +++ b/cpp/csp/python/PyStructFromDict.cpp @@ -0,0 +1,90 @@ +#include + +namespace csp::python{ + +StructPtr structFromDict (const StructMetaPtr& struct_meta, PyObject* dict) { + PyObject* py_key; + PyObject* py_value; + Py_ssize_t ppos = 0; + + if (!PyDict_Check(dict)) { + CSP_THROW(TypeError, "Wrong type used in `from_dict`, expected a dict."); + } + + StructPtr s = struct_meta->create(); + while( PyDict_Next( dict, &ppos, &py_key, &py_value ) ) + { + if( !PyUnicode_Check( py_key ) ) + CSP_THROW( KeyError, "Unexpected key " << PyObjectPtr::incref( py_key ) + << " for type " << struct_meta -> name() ); + + auto & field = struct_meta -> field( PyUnicode_AsUTF8( py_key ) ); + if( !field ) + CSP_THROW( KeyError, "Unexpected key " << PyObjectPtr::incref( py_key ) + << " for type " << struct_meta -> name() ); + + switchCspType(field->type(), [&] (auto tag) + { + using CType = typename decltype( tag )::type; + auto * typedField = static_cast::type *>( field.get() ); + + //optional fields accept None, same as PyStruct_setattrs + if( typedField -> isOptional() && py_value == Py_None ) + { + typedField -> setNone( s.get() ); + typedField -> clearValue( s.get() ); + return; + } + + if constexpr (std::is_same_v) + { + auto& nestedMeta = static_cast(* field->type()).meta(); + typedField -> setValue(s.get(), structFromDict(nestedMeta, py_value)); + } + else if constexpr( std::is_same_v ) + { + //enums arrive as name strings unless to_dict was called with preserve_enums + auto & enumMeta = static_cast( *field -> type() ).meta(); + if( PyUnicode_Check( py_value ) ) + typedField -> setValue( s.get(), enumMeta -> fromString( PyUnicode_AsUTF8( py_value ) ) ); + else + typedField -> setValue( s.get(), fromPython( py_value, *field -> type() ) ); + } + else if constexpr( std::is_same_v> ) + { + //convert element-wise, entries may be names or enum objects + auto & elemType = static_cast( *field -> type() ).elemType(); + auto & enumMeta = static_cast( *elemType ).meta(); + + PyObjectPtr seq = PyObjectPtr::own( PySequence_Fast( py_value, "expected a sequence of enum values" ) ); + if( !seq.ptr() ) + CSP_THROW( PythonPassthrough, "" ); + + std::vector out; + Py_ssize_t n = PySequence_Fast_GET_SIZE( seq.ptr() ); + out.reserve( n ); + for( Py_ssize_t i = 0; i < n; ++i ) + { + PyObject * elem = PySequence_Fast_GET_ITEM( seq.ptr(), i ); + out.push_back( PyUnicode_Check( elem ) ? enumMeta -> fromString( PyUnicode_AsUTF8( elem ) ) + : fromPython( elem, *elemType ) ); + } + typedField -> setValue( s.get(), out ); + } + else + { + typedField -> setValue(s.get(), fromPython(py_value, *field->type())); + } + }); + + } + + //we bypass __init__ so run the strict-struct check ourselves + if( !s -> validate() ) [[unlikely]] + CSP_THROW( ValueError, "Struct " << struct_meta -> name() << " is not valid; required fields " + << s -> formatAllUnsetStrictFields() << " were not set on init" ); + + return s; +} + +} diff --git a/cpp/csp/python/PyStructFromDict.h b/cpp/csp/python/PyStructFromDict.h new file mode 100644 index 000000000..b00c6eb4d --- /dev/null +++ b/cpp/csp/python/PyStructFromDict.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace csp::python +{ + + // Build a csp struct from a python dictionary. + StructPtr structFromDict( const StructMetaPtr& struct_meta, PyObject* dict); + +} \ No newline at end of file diff --git a/cpp/csp/python/PyStructFromJson.cpp b/cpp/csp/python/PyStructFromJson.cpp new file mode 100644 index 000000000..0d9ab459f --- /dev/null +++ b/cpp/csp/python/PyStructFromJson.cpp @@ -0,0 +1,45 @@ +#include + + +namespace csp::python { + +StructPtr structFromJson( const StructMetaPtr& struct_meta, const rapidjson::Value& jValue ) +{ + StructPtr s = struct_meta->create(); + + if( !jValue.IsObject() ) { + CSP_THROW( TypeError, "Expected a json object for type " << struct_meta->name() ); + } + + // Iterate all the fields of a valid JSON object. + for (auto jit = jValue.MemberBegin(); jit != jValue.MemberEnd(); ++jit) { + auto& field = struct_meta->field(jit->name.GetString()); + if (!field) { + CSP_THROW( KeyError, "Unexpected key " << jit -> name.GetString() << " for type " + << struct_meta -> name() ); + } + switchCspType(field->type(), [&](auto tag) { + using CType = typename decltype(tag)::type; + auto * typedField = static_cast::type *>( field.get() ); + + //to_json writes null for fields that are set to None + if( typedField -> isOptional() && jit->value.IsNull() ) + { + typedField -> setNone( s.get() ); + typedField -> clearValue( s.get() ); + return; + } + + typedField -> setValue( s.get(), fromJson( jit->value, *field->type() ) ); + } + ); + } + + if (!s->validate()) { + CSP_THROW(ValueError, "struct " << struct_meta->name() << " is not valid " << "required fields " << + s->formatAllUnsetStrictFields() << " were not set on init"); + } + + return s; +} +} diff --git a/cpp/csp/python/PyStructFromJson.h b/cpp/csp/python/PyStructFromJson.h new file mode 100644 index 000000000..effa1558b --- /dev/null +++ b/cpp/csp/python/PyStructFromJson.h @@ -0,0 +1,195 @@ +#pragma once + +#include +#include + +namespace csp::python { + + StructPtr structFromJson( const StructMetaPtr& struct_meta, const rapidjson::Value& jValue ); + + template + inline T fromJson( const rapidjson::Value & jValue ) + { + static_assert( !std::is_same::value, "no fromJson method implemented for type" ); + return T{}; + } + + template + struct FromJson + { + static T impl( const rapidjson::Value & jValue, const CspType & type ) + { + return fromJson( jValue ); + } + }; + + template + inline T fromJson( const rapidjson::Value & jValue, const CspType & type ) + { + return FromJson::impl( jValue, type ); + } + + template + struct FromJson> + { + static std::vector impl( const rapidjson::Value& jValue, const CspType& arrayType ) { + if (!jValue.IsArray()) { + CSP_THROW(TypeError, "expected json array."); + } + using ElemT = typename CspType::Type::toCArrayElemType::type; + const CspType& elemType = *static_cast( arrayType ).elemType(); + + std::vector out; + out.reserve( jValue.Size() ); + + for (auto it = jValue.Begin(); it != jValue.End(); ++it) { + out.emplace_back( fromJson(*it, elemType)); + } + + return out; + } + }; + + //nested structs need the CspType to reach their own meta + template<> + struct FromJson + { + static StructPtr impl( const rapidjson::Value & jValue, const CspType & type ) + { + return structFromJson( static_cast( type ).meta(), jValue ); + } + }; + + //to_json writes enums as their name + template<> + struct FromJson + { + static CspEnum impl( const rapidjson::Value & jValue, const CspType & type ) + { + if( !jValue.IsString() ) + CSP_THROW( TypeError, "expected an enum name string in json" ); + return static_cast( type ).meta() -> fromString( jValue.GetString() ); + } + }; + + //json carries no python objects, so generic fields take the python from_json path + template<> + struct FromJson + { + static DialectGenericType impl( const rapidjson::Value & jValue, const CspType & type ) + { + CSP_THROW( TypeError, "from_json does not support generic object fields" ); + return DialectGenericType(); + } + }; + + //the date/time formats below mimic the sprintf calls in PyStructToJson.cpp. + + //"%04u-%02u-%02u" + template<> + inline Date fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a date string in json" ); + unsigned year, month, day; + if( sscanf( jValue.GetString(), "%4u-%2u-%2u", &year, &month, &day ) != 3 ) + CSP_THROW( ValueError, "malformed date in json: " << jValue.GetString() ); + return Date( year, month, day ); + } + + //%02u:%02u:%02u.%06u + template<> + inline Time fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a time string in json" ); + unsigned hour, minute, second, micros; + if( sscanf( jValue.GetString(), "%2u:%2u:%2u.%6u", &hour, &minute, &second, µs ) != 4 ) + CSP_THROW( ValueError, "malformed time in json: " << jValue.GetString() ); + return Time( hour, minute, second, micros * NANOS_PER_MICROSECOND ); + } + + //%04u-%02u-%02uT%02u:%02u:%02u.%06u+00:00, csp=utc so not parsing offset + template<> + inline DateTime fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a datetime string in json" ); + unsigned year, month, day, hour, minute, second, micros; + if( sscanf( jValue.GetString(), "%4u-%2u-%2uT%2u:%2u:%2u.%6u", + &year, &month, &day, &hour, &minute, &second, µs ) != 7 ) + CSP_THROW( ValueError, "malformed datetime in json: " << jValue.GetString() ); + return DateTime( year, month, day, hour, minute, second, micros * NANOS_PER_MICROSECOND ); + } + + //. + template<> + inline TimeDelta fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a timedelta string in json" ); + const char * str = jValue.GetString(); + bool negative = ( *str == '-' ); + if( negative || *str == '+' ) + ++str; + + uint64_t seconds; + unsigned micros; + if( sscanf( str, "%lu.%6u", &seconds, µs ) != 2 ) + CSP_THROW( ValueError, "malformed timedelta in json: " << jValue.GetString() ); + + int64_t nanos = seconds * NANOS_PER_SECOND + micros * NANOS_PER_MICROSECOND; + return TimeDelta::fromNanoseconds( negative ? -nanos : nanos ); + } + + template<> + inline int64_t fromJson( const rapidjson::Value & jValue) { + if (!jValue.IsInt64()) { + CSP_THROW(TypeError, "Expected int64 in JSON."); + } + return jValue.GetInt64(); + } + + //narrow int widths go via int64 then a range check. + template + inline T narrowIntFromJson( const rapidjson::Value & jValue ) + { + int64_t v = fromJson( jValue ); + if( v < static_cast( std::numeric_limits::min() ) || + v > static_cast( std::numeric_limits::max() ) ) + CSP_THROW( ValueError, "json value " << v << " is out of range for the field type" ); + return static_cast( v ); + } + + template<> inline int8_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + template<> inline int16_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + template<> inline int32_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + template<> inline uint8_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + template<> inline uint16_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + template<> inline uint32_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson( jValue ); } + + template<> + inline uint64_t fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsUint64() ) CSP_THROW( TypeError, "expected uint64 in json" ); + return jValue.GetUint64(); + } + + template<> + inline bool fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsBool() ) CSP_THROW( TypeError, "expected bool in json" ); + return jValue.GetBool(); + } + + template<> + inline double fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsNumber()) CSP_THROW( TypeError, "expected double in json" ); + return jValue.GetDouble(); + } + + template<> + inline std::string fromJson( const rapidjson::Value & jValue ) + { + if( !jValue.IsString() ) CSP_THROW( TypeError, "expected string in json" ); + return jValue.GetString(); + } + +} diff --git a/csp/impl/struct.py b/csp/impl/struct.py index 50540d365..b52d23ab3 100644 --- a/csp/impl/struct.py +++ b/csp/impl/struct.py @@ -1,6 +1,8 @@ import io +import json as _json import typing from copy import deepcopy +from datetime import date, datetime, time, timedelta import ruamel.yaml from deprecated import deprecated @@ -13,6 +15,75 @@ # Avoid recreating this object every call its expensive! g_YAML = ruamel.yaml.YAML() +# field annotations the C++ from_dict converts natively +_CPP_FROMDICT_SCALARS = (int, float, bool, str, bytes, datetime, date, time, timedelta, object) + + +def _cpp_fromdict_type_ok(typ, seen): + if CspTypingUtils.is_union_type(typ): + return True + if CspTypingUtils.is_generic_container(typ): + origin = CspTypingUtils.get_origin(typ) + if origin in (csp.typing.NumpyNDArray, csp.typing.Numpy1DArray): + return True + if origin in (typing.List, typing.Set, typing.Tuple, FastList): + elem = typ.__args__[0] + return isinstance(elem, type) and not issubclass(elem, Struct) and _cpp_fromdict_type_ok(elem, seen) + return False # Dict[K,V] etc: type parameters are erased before C++ + if isinstance(typ, type): + if issubclass(typ, Struct): + return typ._cpp_from_dict_ok(seen) + return issubclass(typ, csp.Enum) or typ in _CPP_FROMDICT_SCALARS + return False # Literal, ForwardRef, etc + + +# json carries no python objects +_CPP_FROMJSON_SCALARS = (int, float, bool, str, datetime, date, time, timedelta) + + +def _cpp_fromjson_type_ok(typ, seen): + if CspTypingUtils.is_generic_container(typ): + origin = CspTypingUtils.get_origin(typ) + if origin in (typing.List, typing.Set, typing.Tuple, FastList): + elem = typ.__args__[0] + return isinstance(elem, type) and _cpp_fromjson_type_ok(elem, seen) + return False # Dict[K,V], numpy arrays + if isinstance(typ, type): + if issubclass(typ, Struct): + return typ._cpp_from_json_ok(seen) + return issubclass(typ, csp.Enum) or typ in _CPP_FROMJSON_SCALARS + return False # Literal, Union, object, ForwardRef + + +def _json_to_python(obj, typ): + """Decode the string forms to_json writes back into what _obj_from_python expects""" + if obj is None: + return None + if CspTypingUtils.is_generic_container(typ): + origin = CspTypingUtils.get_origin(typ) + if origin in (typing.List, typing.Set, typing.Tuple, FastList): + return [_json_to_python(v, typ.__args__[0]) for v in obj] + if origin is typing.Dict: + key_type, value_type = typ.__args__ + return {_json_to_python(k, key_type): _json_to_python(v, value_type) for k, v in obj.items()} + return obj + if isinstance(typ, type): + if issubclass(typ, Struct): + meta = typ.__full_metadata_typed__ + return {k: _json_to_python(v, meta.get(k, object)) for k, v in obj.items()} + if issubclass(typ, csp.Enum): + return typ[obj] if isinstance(obj, str) else obj + if isinstance(obj, str): + if typ is datetime: + return datetime.fromisoformat(obj) + if typ is date: + return date.fromisoformat(obj) + if typ is time: + return time.fromisoformat(obj) + if typ is timedelta: + return timedelta(seconds=float(obj)) + return obj + class StructMeta(_csptypesimpl.PyStructMeta): def __new__(cls, name, bases, dct, strict=False): @@ -275,12 +346,49 @@ def _obj_from_python(cls, json, obj_type): else: return obj_type(json) + @classmethod + def _cpp_from_dict_ok(cls, _seen=None): + """True if every field (recursively) is convertible by the C++ from_dict, cached per class""" + ok = cls.__dict__.get("__cpp_fromdict_ok__") + if ok is None: + _seen = _seen if _seen is not None else set() + if cls in _seen: + return True + _seen.add(cls) + cls.__cpp_fromdict_ok__ = ok = all( + _cpp_fromdict_type_ok(t, _seen) for t in cls.__full_metadata_typed__.values() + ) + return ok + @classmethod def from_dict(cls, json: dict, use_pydantic: bool = False): if use_pydantic: return cls.type_adapter().validate_python(json) + if cls._cpp_from_dict_ok(): + return super().from_dict(json) return cls._obj_from_python(json, cls) + @classmethod + def _cpp_from_json_ok(cls, _seen=None): + """True if every field (recursively) is convertible by the C++ from_json, cached per class""" + ok = cls.__dict__.get("__cpp_fromjson_ok__") + if ok is None: + _seen = _seen if _seen is not None else set() + if cls in _seen: + return True + _seen.add(cls) + cls.__cpp_fromjson_ok__ = ok = all( + _cpp_fromjson_type_ok(t, _seen) for t in cls.__full_metadata_typed__.values() + ) + return ok + + @classmethod + def from_json(cls, json_str: str): + """Create a struct from the json representation produced by to_json""" + if cls._cpp_from_json_ok(): + return super().from_json(json_str) + return cls._obj_from_python(_json_to_python(_json.loads(json_str), cls), cls) + def to_dict_depr(self): res = self._obj_to_python(self) return res diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index 8bd0dc1ec..8e9319a75 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -1868,6 +1868,72 @@ class MyStruct(csp.Struct): result_dict = {"i": 456, "l_any": l_any_result} self.assertEqual(json.loads(test_struct.to_json()), result_dict) + def test_from_json(self): + class Inner(csp.Struct): + i: int = 1 + f: float = 2.5 + + class Outer(csp.Struct): + b: bool + i: int + f: float + s: str + dt: datetime + d: date + t: time + td: timedelta + e: MyEnum + inner: Inner + floats: List[float] + inners: List[Inner] + + expected = Outer( + b=True, + i=-123456789, + f=3.14, + s="hello world", + dt=datetime(2020, 1, 2, 3, 4, 5, 123456), + d=date(2021, 6, 15), + t=time(9, 30, 0, 500000), + td=timedelta(seconds=-90, microseconds=-500000), + e=MyEnum.FOO, + inner=Inner(i=7), + floats=[1.0, 2.5, -3.75], + inners=[Inner(i=8), Inner(f=9.5)], + ) + + # every field type above is handled, so this takes the C++ path + self.assertTrue(Outer._cpp_from_json_ok()) + self.assertEqual(Outer.from_json(expected.to_json()), expected) + + # fields absent from the json stay unset + partial = Outer.from_json('{"i": 5}') + self.assertEqual(partial.i, 5) + self.assertFalse(partial.all_fields_set()) + self.assertEqual(partial.to_dict(), {"i": 5}) + + # Dict[K, V] type parameters are normalized before C++ sees them, so structs + # containing one fall back to the python impl + class WithDict(csp.Struct): + m: Dict[str, MyEnum] = {"a": MyEnum.A} + n: int = 3 + + self.assertFalse(WithDict._cpp_from_json_ok()) + with_dict = WithDict() + self.assertEqual(WithDict.from_json(with_dict.to_json()), with_dict) + + # errors + with self.assertRaises(KeyError): + Outer.from_json('{"nosuchfield": 1}') + with self.assertRaises(ValueError): + Outer.from_json("not json at all") + with self.assertRaises(TypeError): + Outer.from_json('{"i": "not an int"}') + with self.assertRaises(TypeError): + Outer.from_json("[1, 2, 3]") + with self.assertRaises(ValueError): + Outer.from_json('{"dt": "not a datetime"}') + def test_to_json_dict(self): class MyStruct(csp.Struct): i: int = 123