|
| 1 | +"""Contains essential functions for working with JSON, JSON-SCHEMA |
| 2 | +and JSON-LD context objects. Python implementation of |
| 3 | +https://github.com/OpenSemanticLab/mediawiki-extensions-MwJson/blob/main/modules/ext.MwJson.util/MwJson_util.js |
| 4 | +""" |
| 5 | + |
| 6 | +from copy import deepcopy |
| 7 | +from enum import StrEnum |
| 8 | +from typing import Dict, Optional, TypeVar |
| 9 | + |
| 10 | +from pydantic import BaseModel |
| 11 | +from typing_extensions import deprecated |
| 12 | + |
| 13 | +JsonType = TypeVar("JsonType", dict, list, float, int, str, None) |
| 14 | + |
| 15 | + |
| 16 | +def deep_equal(x: JsonType, y: JsonType): |
| 17 | + """Compares two objects deeply. |
| 18 | +
|
| 19 | + Parameters |
| 20 | + ---------- |
| 21 | + x |
| 22 | + a dictionary, list or scalar value |
| 23 | + y |
| 24 | + another dictionary, list or scalar value |
| 25 | +
|
| 26 | + Returns |
| 27 | + ------- |
| 28 | + True if the two objects are deeply equal, False otherwise |
| 29 | + """ |
| 30 | + |
| 31 | + if x is not None and y is not None and isinstance(x, dict) and isinstance(y, dict): |
| 32 | + return len(x.keys()) == len(y.keys()) and all( |
| 33 | + deep_equal(x[key], y.get(key, None)) for key in x |
| 34 | + ) |
| 35 | + elif ( |
| 36 | + x is not None and y is not None and isinstance(x, list) and isinstance(y, list) |
| 37 | + ): |
| 38 | + return len(x) == len(y) and all( |
| 39 | + deep_equal(x[key], y[key]) for key in range(0, len(x)) |
| 40 | + ) |
| 41 | + else: |
| 42 | + return x == y |
| 43 | + # all(deep_equal(x[key], y.get(key)) for key in x) or x == y |
| 44 | + |
| 45 | + |
| 46 | +def unique_array(array: list) -> list: |
| 47 | + """Returns a new array with only unique elements by comparing them deeply. |
| 48 | +
|
| 49 | + Parameters: |
| 50 | + array: list |
| 51 | + The array to be filtered |
| 52 | +
|
| 53 | + Returns: |
| 54 | + list |
| 55 | + A new array with only unique elements |
| 56 | + """ |
| 57 | + result = [] |
| 58 | + for item in array: |
| 59 | + add = True |
| 60 | + for added_item in result: |
| 61 | + if deep_equal(added_item, item): |
| 62 | + add = False |
| 63 | + break |
| 64 | + if add: |
| 65 | + result.append(item) |
| 66 | + return result |
| 67 | + |
| 68 | + |
| 69 | +def is_object(obj): |
| 70 | + """Tests if an object is a dictionary. |
| 71 | +
|
| 72 | + Parameters |
| 73 | + ---------- |
| 74 | + obj |
| 75 | + the object to be tested |
| 76 | +
|
| 77 | + Returns |
| 78 | + ------- |
| 79 | + True if the object is a dictionary, False otherwise |
| 80 | + """ |
| 81 | + return isinstance(obj, dict) |
| 82 | + |
| 83 | + |
| 84 | +def is_array(obj): |
| 85 | + """Tests if an object is a list. |
| 86 | +
|
| 87 | + Parameters |
| 88 | + ---------- |
| 89 | + obj |
| 90 | + the object to be tested |
| 91 | +
|
| 92 | + Returns |
| 93 | + ------- |
| 94 | + True if the object is a list, False otherwise |
| 95 | + """ |
| 96 | + return isinstance(obj, list) |
| 97 | + |
| 98 | + |
| 99 | +def is_string(obj): |
| 100 | + """Tests if an object is a string. |
| 101 | +
|
| 102 | + Parameters |
| 103 | + ---------- |
| 104 | + obj |
| 105 | + the object to be tested |
| 106 | +
|
| 107 | + Returns |
| 108 | + ------- |
| 109 | + True if the object is a string, False otherwise |
| 110 | + """ |
| 111 | + return isinstance(obj, str) |
| 112 | + |
| 113 | + |
| 114 | +def copy_deep(target: JsonType) -> JsonType: |
| 115 | + """Copies an object deeply. |
| 116 | +
|
| 117 | + Parameters |
| 118 | + ---------- |
| 119 | + target |
| 120 | + the object which values will be copied |
| 121 | +
|
| 122 | + Returns |
| 123 | + ------- |
| 124 | + the copied object |
| 125 | + """ |
| 126 | + return deepcopy(target) |
| 127 | + |
| 128 | + |
| 129 | +@deprecated("Use merge_deep instead") |
| 130 | +def merge_deep_objects(target: dict, source: dict) -> JsonType: |
| 131 | + """Merges two objects deeply, does not handle lists. |
| 132 | + If dictionaries are encountered, the values of the source object |
| 133 | + will overwrite the target object. |
| 134 | + Missing keys in the target object will be added. |
| 135 | + If an array is encountered as a subelement, the arrays are |
| 136 | + concatenated and duplicates are removed. |
| 137 | + If literals are encountered, the source value will |
| 138 | + overwrite the target value. |
| 139 | +
|
| 140 | + Parameters |
| 141 | + ---------- |
| 142 | + target |
| 143 | + the object which values will be potentially overwritten |
| 144 | + source |
| 145 | + the object which values will take precedence over the target object |
| 146 | +
|
| 147 | + Returns |
| 148 | + ------- |
| 149 | + the merged object |
| 150 | + """ |
| 151 | + if not target: |
| 152 | + return source |
| 153 | + if not source: |
| 154 | + return target |
| 155 | + output = deepcopy(target) |
| 156 | + if is_object(target) and is_object(source): |
| 157 | + for key in source: |
| 158 | + if is_array(source[key]) and is_array(target.get(key)): |
| 159 | + if key not in target: |
| 160 | + output[key] = source[key] |
| 161 | + else: |
| 162 | + output[key] = unique_array(target[key] + source[key]) |
| 163 | + elif is_object(source[key]): |
| 164 | + if key not in target: |
| 165 | + output[key] = source[key] |
| 166 | + else: |
| 167 | + output[key] = merge_deep(target[key], source[key]) |
| 168 | + else: |
| 169 | + output[key] = source[key] |
| 170 | + |
| 171 | + return output |
| 172 | + |
| 173 | + |
| 174 | +def merge_deep(target: JsonType, source: JsonType) -> JsonType: |
| 175 | + """Merges two objects deeply. |
| 176 | + If dictionaries are encountered, the values of the source object |
| 177 | + will overwrite the target object. |
| 178 | + Missing keys in the target object will be added. |
| 179 | + If an array is encountered as a subelement, the arrays are |
| 180 | + concatenated and duplicates are removed. |
| 181 | + If literals are encountered, the source value will |
| 182 | + overwrite the target value. |
| 183 | +
|
| 184 | + Parameters |
| 185 | + ---------- |
| 186 | + target |
| 187 | + the object which values will be potentially overwritten |
| 188 | + source |
| 189 | + the object which values will take precedence over the target object |
| 190 | +
|
| 191 | + Returns |
| 192 | + ------- |
| 193 | + the merged object |
| 194 | + """ |
| 195 | + if not target: |
| 196 | + return source |
| 197 | + if not source: |
| 198 | + return target |
| 199 | + output = deepcopy(target) |
| 200 | + |
| 201 | + if is_object(target) and is_object(source): |
| 202 | + for key in source: |
| 203 | + output[key] = merge_deep(output.get(key, None), source[key]) |
| 204 | + elif is_array(source) and is_array(target): |
| 205 | + output = unique_array(target + source) |
| 206 | + else: |
| 207 | + output = source |
| 208 | + return output |
| 209 | + |
| 210 | + |
| 211 | +def merge_jsonld_context_object_list(context: list) -> list: |
| 212 | + """to cleanup generated json-ld context |
| 213 | + ["/some/remove/context", {"a": "ex:a"}, {"a": "ex:a", "b": "ex:b"}] |
| 214 | + => ["/some/remove/context", {"a": "ex:a", "b": "ex:b"}] |
| 215 | +
|
| 216 | + Parameters |
| 217 | + ---------- |
| 218 | + list |
| 219 | + mixed list of strings and dictionaries |
| 220 | + """ |
| 221 | + |
| 222 | + # interate over all elements |
| 223 | + # if element is a string, add it to the result list |
| 224 | + # if element is a dictionary, merge it with the last dictionary in the |
| 225 | + # result list |
| 226 | + |
| 227 | + # if not a list, return immediately |
| 228 | + if not is_array(context): |
| 229 | + return context |
| 230 | + |
| 231 | + result = [] |
| 232 | + last = None |
| 233 | + for e in context: |
| 234 | + if is_object(e): |
| 235 | + if last is None: |
| 236 | + last = e |
| 237 | + else: |
| 238 | + last = merge_deep(last, e) |
| 239 | + else: |
| 240 | + if last is not None: |
| 241 | + result.append(last) |
| 242 | + last = None |
| 243 | + result.append(e) |
| 244 | + if last is not None: |
| 245 | + result.append(last) |
| 246 | + return result |
| 247 | + |
| 248 | + |
| 249 | +class AggregateGeneratedSchemasParamMode(StrEnum): |
| 250 | + ROOT_LEVEL = "root_level" |
| 251 | + """ The generated schema is merged at the root level """ |
| 252 | + DEFINITIONS_SECTION = "definitions_section" |
| 253 | + """ The generated schema is merged into the definitions section """ |
| 254 | + |
| 255 | + |
| 256 | +class AggregateGeneratedSchemasParam(BaseModel): |
| 257 | + target_schema: Optional[dict] = {} |
| 258 | + """ The target schema to be merged with the generated schema """ |
| 259 | + generated_schemas: Dict[str, dict] |
| 260 | + """ List of JSON schemas to be aggregated """ |
| 261 | + mode: AggregateGeneratedSchemasParamMode = ( |
| 262 | + AggregateGeneratedSchemasParamMode.ROOT_LEVEL |
| 263 | + ) |
| 264 | + """ The mode to be used for aggregation """ |
| 265 | + def_key: Optional[str] = "$defs" |
| 266 | + """ The keyword for schema definitions. $defs is recommended""" |
| 267 | + gen_def_key: Optional[str] = "generated" |
| 268 | + """ The keyword to store the generated schema. |
| 269 | + Note: Having a separate section per generated schema would lead |
| 270 | + to many partial classes in code generation """ |
| 271 | + generate_root_ref: Optional[bool] = False |
| 272 | + """ If true, generate $ref: "#/def...", else allOf: [{$ref: "#/def...""}. |
| 273 | + Root refs are not supported by json_ref_parser < 0.10 and data-model-codegen """ |
| 274 | + gen_def_pointer: Optional[str] = None |
| 275 | + """ The pointer to the generated schema. If None, it will be set to |
| 276 | + "#/" + def_key + "/" + gen_def_key """ |
| 277 | + |
| 278 | + def __init__(self, **data): |
| 279 | + super().__init__(**data) |
| 280 | + if self.gen_def_pointer is None: |
| 281 | + self.gen_def_pointer = "#/" + self.def_key + "/" + self.gen_def_key |
| 282 | + |
| 283 | + |
| 284 | +class AggregateGeneratedSchemasResult(BaseModel): |
| 285 | + aggregated_schema: dict |
| 286 | + """ The aggregated schema """ |
| 287 | + |
| 288 | + |
| 289 | +def aggregate_generated_schemas( |
| 290 | + param: AggregateGeneratedSchemasParam, |
| 291 | +) -> AggregateGeneratedSchemasResult: |
| 292 | + """Applies a merge operation on two OO-LD schemas. |
| 293 | +
|
| 294 | + Parameters |
| 295 | + ---------- |
| 296 | + param |
| 297 | + see AggregateGeneratedSchemasParam |
| 298 | +
|
| 299 | + Returns |
| 300 | + ------- |
| 301 | + see AggregateGeneratedSchemasResult |
| 302 | + """ |
| 303 | + mode = param.mode |
| 304 | + def_key = param.def_key |
| 305 | + gen_def_key = param.gen_def_key |
| 306 | + gen_def_pointer = param.gen_def_pointer |
| 307 | + generate_root_ref = param.generate_root_ref |
| 308 | + schema = param.target_schema |
| 309 | + |
| 310 | + for generated_schema_id in param.generated_schemas.keys(): |
| 311 | + generated_schema = param.generated_schemas[generated_schema_id] |
| 312 | + if mode == AggregateGeneratedSchemasParamMode.ROOT_LEVEL: |
| 313 | + schema = merge_deep(schema, generated_schema) |
| 314 | + else: |
| 315 | + # Store generated schema in #/$defs/generated (force overwrite), |
| 316 | + # add $ref: #/$defs/generated to schema |
| 317 | + # note: using $def with $ leads to recursion error in |
| 318 | + # note: requires addition schema properties are allowed on the |
| 319 | + # same level as $ref. allOf: $ref would imply a superclass |
| 320 | + if "@context" in generated_schema: |
| 321 | + generated_context = copy_deep(generated_schema["@context"]) |
| 322 | + del generated_schema["@context"] |
| 323 | + existing_context = schema.get("@context", None) |
| 324 | + if existing_context is not None: |
| 325 | + # case A: "" + "" => ["", ""] |
| 326 | + # case B: "" + {} => ["", {}] |
| 327 | + # case C: "" + [] => ["", ] |
| 328 | + # case D: [] + {} => [, {}] |
| 329 | + # case E: {} + {} => {} |
| 330 | + # case F: [] + [] => [] |
| 331 | + |
| 332 | + if is_array(existing_context) and not is_array(generated_context): |
| 333 | + generated_context = [generated_context] |
| 334 | + # case C + D |
| 335 | + elif not is_array(existing_context) and is_array(generated_context): |
| 336 | + existing_context = [existing_context] |
| 337 | + # case C + D |
| 338 | + elif not is_array(existing_context) and not is_array( |
| 339 | + generated_context |
| 340 | + ): |
| 341 | + if is_string(existing_context) or is_string( |
| 342 | + existing_context |
| 343 | + ): # case A + B |
| 344 | + generated_context = [generated_context] |
| 345 | + existing_context = [existing_context] |
| 346 | + # case E + F: nothing to do |
| 347 | + schema["@context"] = merge_deep( |
| 348 | + {"@context": existing_context}, {"@context": generated_context} |
| 349 | + )["@context"] |
| 350 | + if is_array(schema["@context"]): |
| 351 | + schema["@context"] = merge_jsonld_context_object_list( |
| 352 | + schema["@context"] |
| 353 | + ) |
| 354 | + |
| 355 | + if def_key not in schema: |
| 356 | + schema[def_key] = {} |
| 357 | + if gen_def_key not in schema[def_key]: |
| 358 | + schema[def_key][gen_def_key] = { |
| 359 | + "$comment": "Autogenerated section - do not edit. Generated from" |
| 360 | + } |
| 361 | + schema[def_key][gen_def_key]["$comment"] += " " + generated_schema_id |
| 362 | + # schema[def_key][gen_def_key] = generated_schema; # full override |
| 363 | + schema[def_key][gen_def_key] = merge_deep( |
| 364 | + schema[def_key][gen_def_key], generated_schema |
| 365 | + ) |
| 366 | + # merge |
| 367 | + |
| 368 | + if generate_root_ref: |
| 369 | + if "$ref" in schema and schema["$ref"] != gen_def_pointer: |
| 370 | + print( |
| 371 | + "Error while applying generated schema: $ref already set to " |
| 372 | + + schema["$ref"] |
| 373 | + ) |
| 374 | + else: |
| 375 | + schema["$ref"] = gen_def_pointer |
| 376 | + else: |
| 377 | + if "allOf" not in schema: |
| 378 | + schema["allOf"] = [] |
| 379 | + # check if any allOf already points to the generated schema |
| 380 | + exists = any( |
| 381 | + [allOf["$ref"] == gen_def_pointer for allOf in schema["allOf"]] |
| 382 | + ) |
| 383 | + if not exists: |
| 384 | + schema["allOf"].append({"$ref": gen_def_pointer}) |
| 385 | + if "title" in generated_schema: |
| 386 | + schema["title"] = generated_schema["title"] |
| 387 | + schema[def_key][gen_def_key]["title"] = ( |
| 388 | + "Generated" + generated_schema["title"] |
| 389 | + ) |
| 390 | + schema[def_key][gen_def_key]["description"] = ( |
| 391 | + "This is an autogenerated partial class definition of '" |
| 392 | + + generated_schema["title"] |
| 393 | + + "'" |
| 394 | + ) |
| 395 | + if "description" in generated_schema: |
| 396 | + schema["description"] = generated_schema["description"] |
| 397 | + |
| 398 | + return AggregateGeneratedSchemasResult(aggregated_schema=schema) |
0 commit comments