Skip to content

Commit d087d76

Browse files
aadya940aadya940
committed
Add fast C++ path for from_json
1 parent feb6cc6 commit d087d76

6 files changed

Lines changed: 403 additions & 2 deletions

File tree

‎cpp/csp/python/CMakeLists.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ add_library(csptypesimpl
1818
PyStruct.cpp
1919
PyStructToJson.cpp
2020
PyStructToDict.cpp
21-
PyStructFromDict.cpp)
21+
PyStructFromDict.cpp
22+
PyStructFromJson.cpp)
2223

2324
set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}")
2425
target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1)
@@ -50,7 +51,8 @@ set(CSPIMPL_PUBLIC_HEADERS
5051
PyConstants.h
5152
PyStructToJson.h
5253
PyStructToDict.h
53-
PyStructFromDict.h)
54+
PyStructFromDict.h
55+
PyStructFromJson.cpp)
5456

5557
add_library(cspimpl SHARED
5658
cspimpl.cpp

‎cpp/csp/python/PyStruct.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
#include <csp/python/PyStructToJson.h>
99
#include <csp/python/PyStructToDict.h>
1010
#include <csp/python/PyStructFromDict.h>
11+
#include <csp/python/PyStructFromJson.h>
12+
#include <rapidjson/error/en.h>
1113
#include <unordered_set>
1214
#include <type_traits>
1315

@@ -1067,6 +1069,26 @@ PyObject * PyStruct_from_dict( PyStructMeta * cls, PyObject * args, PyObject * k
10671069
CSP_RETURN_NULL;
10681070
}
10691071

1072+
PyObject * PyStruct_from_json( PyStructMeta * cls, PyObject * args, PyObject * kwargs )
1073+
{
1074+
CSP_BEGIN_METHOD;
1075+
1076+
const char * json = nullptr;
1077+
Py_ssize_t len = 0;
1078+
if( !PyArg_ParseTuple( args, "s#:from_json", &json, &len ) )
1079+
return NULL;
1080+
1081+
//kParseNanAndInfFlag to match to_json, which writes NaN / Inf for doubles
1082+
rapidjson::Document doc;
1083+
rapidjson::ParseResult ok = doc.Parse<rapidjson::kParseNanAndInfFlag>( json, len );
1084+
if( !ok )
1085+
CSP_THROW( ValueError, "Failed to parse json: " << rapidjson::GetParseError_En( ok.Code() ) );
1086+
1087+
return toPython( structFromJson( cls -> structMeta, doc ) );
1088+
1089+
CSP_RETURN_NULL;
1090+
}
1091+
10701092
PyObject * PyStruct_to_json( PyStruct * self, PyObject * args, PyObject * kwargs )
10711093
{
10721094
CSP_BEGIN_METHOD;
@@ -1103,6 +1125,7 @@ static PyMethodDef PyStruct_methods[] = {
11031125
{ "to_dict", (PyCFunction) PyStruct_to_dict, METH_VARARGS | METH_KEYWORDS, "return a python dict of the struct by recursively converting struct members into python dicts" },
11041126
{ "to_json", (PyCFunction) PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, "return a json string of the struct by recursively converting struct members into json format" },
11051127
{ "from_dict", (PyCFunction) PyStruct_from_dict, METH_VARARGS | METH_KEYWORDS | METH_CLASS, "return a struct by recursively reading the values from a python dictionary"},
1128+
{ "from_json", (PyCFunction) PyStruct_from_json, METH_VARARGS | METH_KEYWORDS | METH_CLASS, "return a struct by recursively reading the values from a json string"},
11061129
{ NULL}
11071130
};
11081131

‎cpp/csp/python/PyStructFromJson.cpp‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#include <csp/python/PyStructFromJson.h>
2+
3+
4+
namespace csp::python {
5+
6+
StructPtr structFromJson( const StructMetaPtr& struct_meta, const rapidjson::Value& jValue )
7+
{
8+
StructPtr s = struct_meta->create();
9+
10+
if( !jValue.IsObject() ) {
11+
CSP_THROW( TypeError, "Expected a json object for type " << struct_meta->name() );
12+
}
13+
14+
// Iterate all the fields of a valid JSON object.
15+
for (auto jit = jValue.MemberBegin(); jit != jValue.MemberEnd(); ++jit) {
16+
auto& field = struct_meta->field(jit->name.GetString());
17+
if (!field) {
18+
CSP_THROW( KeyError, "Unexpected key " << jit -> name.GetString() << " for type "
19+
<< struct_meta -> name() );
20+
}
21+
switchCspType(field->type(), [&](auto tag) {
22+
using CType = typename decltype(tag)::type;
23+
auto * typedField = static_cast<const typename StructField::upcast<CType>::type *>( field.get() );
24+
25+
//to_json writes null for fields that are set to None
26+
if( typedField -> isOptional() && jit->value.IsNull() )
27+
{
28+
typedField -> setNone( s.get() );
29+
typedField -> clearValue( s.get() );
30+
return;
31+
}
32+
33+
typedField -> setValue( s.get(), fromJson<CType>( jit->value, *field->type() ) );
34+
}
35+
);
36+
}
37+
38+
if (!s->validate()) {
39+
CSP_THROW(ValueError, "struct " << struct_meta->name() << " is not valid " << "required fields " <<
40+
s->formatAllUnsetStrictFields() << " were not set on init");
41+
}
42+
43+
return s;
44+
}
45+
}

‎cpp/csp/python/PyStructFromJson.h‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#pragma once
2+
3+
#include <csp/python/Conversions.h>
4+
#include <rapidjson/document.h>
5+
6+
namespace csp::python {
7+
8+
StructPtr structFromJson( const StructMetaPtr& struct_meta, const rapidjson::Value& jValue );
9+
10+
template<typename T>
11+
inline T fromJson( const rapidjson::Value & jValue )
12+
{
13+
static_assert( !std::is_same<T,T>::value, "no fromJson method implemented for type" );
14+
return T{};
15+
}
16+
17+
template<typename T>
18+
struct FromJson
19+
{
20+
static T impl( const rapidjson::Value & jValue, const CspType & type )
21+
{
22+
return fromJson<T>( jValue );
23+
}
24+
};
25+
26+
template<typename T>
27+
inline T fromJson( const rapidjson::Value & jValue, const CspType & type )
28+
{
29+
return FromJson<T>::impl( jValue, type );
30+
}
31+
32+
template<typename StorageT>
33+
struct FromJson<std::vector<StorageT>>
34+
{
35+
static std::vector<StorageT> impl( const rapidjson::Value& jValue, const CspType& arrayType ) {
36+
if (!jValue.IsArray()) {
37+
CSP_THROW(TypeError, "expected json array.");
38+
}
39+
using ElemT = typename CspType::Type::toCArrayElemType<StorageT>::type;
40+
const CspType& elemType = *static_cast<const CspArrayType &>( arrayType ).elemType();
41+
42+
std::vector<StorageT> out;
43+
out.reserve( jValue.Size() );
44+
45+
for (auto it = jValue.Begin(); it != jValue.End(); ++it) {
46+
out.emplace_back( fromJson<ElemT>(*it, elemType));
47+
}
48+
49+
return out;
50+
}
51+
};
52+
53+
//nested structs need the CspType to reach their own meta
54+
template<>
55+
struct FromJson<StructPtr>
56+
{
57+
static StructPtr impl( const rapidjson::Value & jValue, const CspType & type )
58+
{
59+
return structFromJson( static_cast<const CspStructType &>( type ).meta(), jValue );
60+
}
61+
};
62+
63+
//to_json writes enums as their name
64+
template<>
65+
struct FromJson<CspEnum>
66+
{
67+
static CspEnum impl( const rapidjson::Value & jValue, const CspType & type )
68+
{
69+
if( !jValue.IsString() )
70+
CSP_THROW( TypeError, "expected an enum name string in json" );
71+
return static_cast<const CspEnumType &>( type ).meta() -> fromString( jValue.GetString() );
72+
}
73+
};
74+
75+
//json carries no python objects, so generic fields take the python from_json path
76+
template<>
77+
struct FromJson<DialectGenericType>
78+
{
79+
static DialectGenericType impl( const rapidjson::Value & jValue, const CspType & type )
80+
{
81+
CSP_THROW( TypeError, "from_json does not support generic object fields" );
82+
return DialectGenericType();
83+
}
84+
};
85+
86+
//the date/time formats below mimic the sprintf calls in PyStructToJson.cpp.
87+
88+
//"%04u-%02u-%02u"
89+
template<>
90+
inline Date fromJson( const rapidjson::Value & jValue )
91+
{
92+
if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a date string in json" );
93+
unsigned year, month, day;
94+
if( sscanf( jValue.GetString(), "%4u-%2u-%2u", &year, &month, &day ) != 3 )
95+
CSP_THROW( ValueError, "malformed date in json: " << jValue.GetString() );
96+
return Date( year, month, day );
97+
}
98+
99+
//%02u:%02u:%02u.%06u
100+
template<>
101+
inline Time fromJson( const rapidjson::Value & jValue )
102+
{
103+
if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a time string in json" );
104+
unsigned hour, minute, second, micros;
105+
if( sscanf( jValue.GetString(), "%2u:%2u:%2u.%6u", &hour, &minute, &second, &micros ) != 4 )
106+
CSP_THROW( ValueError, "malformed time in json: " << jValue.GetString() );
107+
return Time( hour, minute, second, micros * NANOS_PER_MICROSECOND );
108+
}
109+
110+
//%04u-%02u-%02uT%02u:%02u:%02u.%06u+00:00, csp=utc so not parsing offset
111+
template<>
112+
inline DateTime fromJson( const rapidjson::Value & jValue )
113+
{
114+
if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a datetime string in json" );
115+
unsigned year, month, day, hour, minute, second, micros;
116+
if( sscanf( jValue.GetString(), "%4u-%2u-%2uT%2u:%2u:%2u.%6u",
117+
&year, &month, &day, &hour, &minute, &second, &micros ) != 7 )
118+
CSP_THROW( ValueError, "malformed datetime in json: " << jValue.GetString() );
119+
return DateTime( year, month, day, hour, minute, second, micros * NANOS_PER_MICROSECOND );
120+
}
121+
122+
//<sign><seconds>.<micros>
123+
template<>
124+
inline TimeDelta fromJson( const rapidjson::Value & jValue )
125+
{
126+
if( !jValue.IsString() ) CSP_THROW( TypeError, "expected a timedelta string in json" );
127+
const char * str = jValue.GetString();
128+
bool negative = ( *str == '-' );
129+
if( negative || *str == '+' )
130+
++str;
131+
132+
uint64_t seconds;
133+
unsigned micros;
134+
if( sscanf( str, "%lu.%6u", &seconds, &micros ) != 2 )
135+
CSP_THROW( ValueError, "malformed timedelta in json: " << jValue.GetString() );
136+
137+
int64_t nanos = seconds * NANOS_PER_SECOND + micros * NANOS_PER_MICROSECOND;
138+
return TimeDelta::fromNanoseconds( negative ? -nanos : nanos );
139+
}
140+
141+
template<>
142+
inline int64_t fromJson( const rapidjson::Value & jValue) {
143+
if (!jValue.IsInt64()) {
144+
CSP_THROW(TypeError, "Expected int64 in JSON.");
145+
}
146+
return jValue.GetInt64();
147+
}
148+
149+
//narrow int widths go via int64 then a range check.
150+
template<typename T>
151+
inline T narrowIntFromJson( const rapidjson::Value & jValue )
152+
{
153+
int64_t v = fromJson<int64_t>( jValue );
154+
if( v < static_cast<int64_t>( std::numeric_limits<T>::min() ) ||
155+
v > static_cast<int64_t>( std::numeric_limits<T>::max() ) )
156+
CSP_THROW( ValueError, "json value " << v << " is out of range for the field type" );
157+
return static_cast<T>( v );
158+
}
159+
160+
template<> inline int8_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<int8_t>( jValue ); }
161+
template<> inline int16_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<int16_t>( jValue ); }
162+
template<> inline int32_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<int32_t>( jValue ); }
163+
template<> inline uint8_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<uint8_t>( jValue ); }
164+
template<> inline uint16_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<uint16_t>( jValue ); }
165+
template<> inline uint32_t fromJson( const rapidjson::Value & jValue ) { return narrowIntFromJson<uint32_t>( jValue ); }
166+
167+
template<>
168+
inline uint64_t fromJson( const rapidjson::Value & jValue )
169+
{
170+
if( !jValue.IsUint64() ) CSP_THROW( TypeError, "expected uint64 in json" );
171+
return jValue.GetUint64();
172+
}
173+
174+
template<>
175+
inline bool fromJson( const rapidjson::Value & jValue )
176+
{
177+
if( !jValue.IsBool() ) CSP_THROW( TypeError, "expected bool in json" );
178+
return jValue.GetBool();
179+
}
180+
181+
template<>
182+
inline double fromJson( const rapidjson::Value & jValue )
183+
{
184+
if( !jValue.IsNumber()) CSP_THROW( TypeError, "expected double in json" );
185+
return jValue.GetDouble();
186+
}
187+
188+
template<>
189+
inline std::string fromJson( const rapidjson::Value & jValue )
190+
{
191+
if( !jValue.IsString() ) CSP_THROW( TypeError, "expected string in json" );
192+
return jValue.GetString();
193+
}
194+
195+
}

‎csp/impl/struct.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import io
2+
import json as _json
23
import typing
34
from copy import deepcopy
45
from datetime import date, datetime, time, timedelta
@@ -36,6 +37,54 @@ def _cpp_fromdict_type_ok(typ, seen):
3637
return False # Literal, ForwardRef, etc
3738

3839

40+
# json carries no python objects
41+
_CPP_FROMJSON_SCALARS = (int, float, bool, str, datetime, date, time, timedelta)
42+
43+
44+
def _cpp_fromjson_type_ok(typ, seen):
45+
if CspTypingUtils.is_generic_container(typ):
46+
origin = CspTypingUtils.get_origin(typ)
47+
if origin in (typing.List, typing.Set, typing.Tuple, FastList):
48+
elem = typ.__args__[0]
49+
return isinstance(elem, type) and _cpp_fromjson_type_ok(elem, seen)
50+
return False # Dict[K,V], numpy arrays
51+
if isinstance(typ, type):
52+
if issubclass(typ, Struct):
53+
return typ._cpp_from_json_ok(seen)
54+
return issubclass(typ, csp.Enum) or typ in _CPP_FROMJSON_SCALARS
55+
return False # Literal, Union, object, ForwardRef
56+
57+
58+
def _json_to_python(obj, typ):
59+
"""Decode the string forms to_json writes back into what _obj_from_python expects"""
60+
if obj is None:
61+
return None
62+
if CspTypingUtils.is_generic_container(typ):
63+
origin = CspTypingUtils.get_origin(typ)
64+
if origin in (typing.List, typing.Set, typing.Tuple, FastList):
65+
return [_json_to_python(v, typ.__args__[0]) for v in obj]
66+
if origin is typing.Dict:
67+
key_type, value_type = typ.__args__
68+
return {_json_to_python(k, key_type): _json_to_python(v, value_type) for k, v in obj.items()}
69+
return obj
70+
if isinstance(typ, type):
71+
if issubclass(typ, Struct):
72+
meta = typ.__full_metadata_typed__
73+
return {k: _json_to_python(v, meta.get(k, object)) for k, v in obj.items()}
74+
if issubclass(typ, csp.Enum):
75+
return typ[obj] if isinstance(obj, str) else obj
76+
if isinstance(obj, str):
77+
if typ is datetime:
78+
return datetime.fromisoformat(obj)
79+
if typ is date:
80+
return date.fromisoformat(obj)
81+
if typ is time:
82+
return time.fromisoformat(obj)
83+
if typ is timedelta:
84+
return timedelta(seconds=float(obj))
85+
return obj
86+
87+
3988
class StructMeta(_csptypesimpl.PyStructMeta):
4089
def __new__(cls, name, bases, dct, strict=False):
4190
full_metadata = {}
@@ -319,6 +368,27 @@ def from_dict(cls, json: dict, use_pydantic: bool = False):
319368
return super().from_dict(json)
320369
return cls._obj_from_python(json, cls)
321370

371+
@classmethod
372+
def _cpp_from_json_ok(cls, _seen=None):
373+
"""True if every field (recursively) is convertible by the C++ from_json, cached per class"""
374+
ok = cls.__dict__.get("__cpp_fromjson_ok__")
375+
if ok is None:
376+
_seen = _seen if _seen is not None else set()
377+
if cls in _seen:
378+
return True
379+
_seen.add(cls)
380+
cls.__cpp_fromjson_ok__ = ok = all(
381+
_cpp_fromjson_type_ok(t, _seen) for t in cls.__full_metadata_typed__.values()
382+
)
383+
return ok
384+
385+
@classmethod
386+
def from_json(cls, json_str: str):
387+
"""Create a struct from the json representation produced by to_json"""
388+
if cls._cpp_from_json_ok():
389+
return super().from_json(json_str)
390+
return cls._obj_from_python(_json_to_python(_json.loads(json_str), cls), cls)
391+
322392
def to_dict_depr(self):
323393
res = self._obj_to_python(self)
324394
return res

0 commit comments

Comments
 (0)