|
39 | 39 | from pyld import jsonld |
40 | 40 |
|
41 | 41 | import osw.model.entity as model |
42 | | - |
43 | | -try: |
44 | | - # Characteristics of the released opensemantic packages derive from this |
45 | | - # class, not from the one in osw.model.entity. |
46 | | - from opensemantic.core.v1 import ( |
47 | | - CharacteristicType as _OpensemanticCharacteristicType, |
48 | | - ) |
49 | | -except ImportError: # pragma: no cover - optional dependency |
50 | | - _OpensemanticCharacteristicType = None |
51 | 42 | from osw.defaults import params as default_params |
52 | 43 | from osw.utils.code_postprocessing import ( |
53 | 44 | remove_constraints_from_forward_refs, |
|
58 | 49 | AggregateGeneratedSchemasParamMode, |
59 | 50 | aggregate_generated_schemas, |
60 | 51 | escape_json_strings, |
| 52 | + merge_deep, |
61 | 53 | merge_generated_definitions, |
62 | 54 | ) |
63 | 55 | from osw.utils.templates import ( |
|
68 | 60 | from osw.utils.wiki import ( |
69 | 61 | get_full_title, |
70 | 62 | get_namespace, |
| 63 | + get_osw_id, |
71 | 64 | get_title, |
72 | 65 | is_empty, |
73 | 66 | namespace_from_full_title, |
@@ -233,6 +226,227 @@ def remove_generated_schema_section( |
233 | 226 | return schema |
234 | 227 |
|
235 | 228 |
|
| 229 | +CATEGORY_CATEGORY_UUID = "89aafe6d-ae5a-4f29-97ff-df7736d4cab6" |
| 230 | +CATEGORY_ENTITY_UUID = "ce353767-c628-45bd-9d88-d6eb3009aec0" |
| 231 | +# the definitions section mode is selected when this template is in the list |
| 232 | +CHARACTERISTIC_TYPE_TITLE = "Category:OSWffe74f291d354037b318c422591c5023" |
| 233 | + |
| 234 | + |
| 235 | +def _title_from_schema_ref(ref: Any) -> Optional[str]: |
| 236 | + """Page title of a $ref to a category schema, None for any other $ref. |
| 237 | +
|
| 238 | + Accepts '/wiki/Category:X?action=raw&slot=jsonschema' and |
| 239 | + '.../Special:SlotResolver/Category/X.slot_jsonschema.json'. |
| 240 | + """ |
| 241 | + if not isinstance(ref, str) or ref.startswith("#"): |
| 242 | + return None |
| 243 | + path = ref.split("?")[0] |
| 244 | + match = re.search(r"/Special:SlotResolver/([^/]+)/(.+)\.slot_[a-z]+\.json$", path) |
| 245 | + if match: |
| 246 | + return f"{match.group(1)}:{match.group(2)}" |
| 247 | + match = re.search(r"/wiki/(Category:[^/]+)$", path) |
| 248 | + if match: |
| 249 | + return match.group(1) |
| 250 | + if re.fullmatch(r"Category:[^/]+", path): |
| 251 | + return path |
| 252 | + return None |
| 253 | + |
| 254 | + |
| 255 | +def collect_category_template_titles( |
| 256 | + get_pages, meta_category_titles: List[str] |
| 257 | +) -> List[str]: |
| 258 | + """Titles of the category templates the wiki form editor applies. |
| 259 | +
|
| 260 | + Mirrors the editor: the schema of the meta categories is bundled, the uuid of |
| 261 | + every schema is collected after the schemas below it (allOf, oneOf, anyOf, |
| 262 | + definitions and $defs are followed in this order, so the most generic schema |
| 263 | + comes first) and the result starts with Category:Category. |
| 264 | + Category:Category and Category:Entity are not collected. Follows |
| 265 | + osl.util.postProcessPage (PageBot.js) with editor.jsonschema.subschemas_uuids |
| 266 | + (MwJson_schema.js, _preprocess) in mediawiki-extensions-OpenSemanticLab and |
| 267 | + mediawiki-extensions-MwJson. Schemas reached through properties are not |
| 268 | + followed. |
| 269 | +
|
| 270 | + Parameters |
| 271 | + ---------- |
| 272 | + get_pages |
| 273 | + callable that returns the page objects for a list of titles |
| 274 | + meta_category_titles |
| 275 | + titles of the meta categories, e.g. ["Category:Category"] |
| 276 | + """ |
| 277 | + templates = ["Category:Category"] |
| 278 | + schema_cache: Dict[str, dict] = {} |
| 279 | + visited = set() |
| 280 | + |
| 281 | + def schema_of(title: str) -> dict: |
| 282 | + if title not in schema_cache: |
| 283 | + schema = {} |
| 284 | + for page in get_pages([title]): |
| 285 | + if page.title == title: |
| 286 | + schema = page.get_slot_content("jsonschema") or {} |
| 287 | + if not schema: |
| 288 | + _logger.warning( |
| 289 | + f"No jsonschema found for {title}. Its templates are not " |
| 290 | + "applied, which can select the wrong schema generation mode." |
| 291 | + ) |
| 292 | + schema_cache[title] = schema |
| 293 | + return schema_cache[title] |
| 294 | + |
| 295 | + def walk(schema: Any) -> None: |
| 296 | + if not isinstance(schema, dict): |
| 297 | + return |
| 298 | + ref_title = _title_from_schema_ref(schema.get("$ref")) |
| 299 | + if ref_title is not None: |
| 300 | + # a bundled schema is included once, later refs point to it |
| 301 | + if ref_title not in visited: |
| 302 | + visited.add(ref_title) |
| 303 | + walk(schema_of(ref_title)) |
| 304 | + return |
| 305 | + for key in ("allOf", "oneOf", "anyOf"): |
| 306 | + sub = schema.get(key) |
| 307 | + for entry in sub if isinstance(sub, list) else [sub]: |
| 308 | + walk(entry) |
| 309 | + for key in ("definitions", "$defs"): |
| 310 | + if isinstance(schema.get(key), dict): |
| 311 | + for entry in schema[key].values(): |
| 312 | + walk(entry) |
| 313 | + uuid = schema.get("uuid") |
| 314 | + if uuid and uuid not in (CATEGORY_CATEGORY_UUID, CATEGORY_ENTITY_UUID): |
| 315 | + title = "Category:" + get_osw_id(uuid) |
| 316 | + if title not in templates: |
| 317 | + templates.append(title) |
| 318 | + |
| 319 | + for title in meta_category_titles: |
| 320 | + visited.add(title) |
| 321 | + walk(schema_of(title)) |
| 322 | + return templates |
| 323 | + |
| 324 | + |
| 325 | +def reset_generated_jsonld_context( |
| 326 | + schema: Optional[dict], had_generated_section: bool |
| 327 | +) -> None: |
| 328 | + """Empties the generated first object block of the @context array. |
| 329 | +
|
| 330 | + Port of osl.util.resetGeneratedJsonLdContext in PageBot.js. The generated |
| 331 | + property mappings are by convention the first object entry of an @context |
| 332 | + array, the strings before it are parent schema references. Without a generated |
| 333 | + section the first object block is hand written, so an empty block is inserted |
| 334 | + ahead of it. A string or object @context is left as it is. Modifies schema in |
| 335 | + place. |
| 336 | + """ |
| 337 | + if not schema: |
| 338 | + return |
| 339 | + context = schema.get("@context") |
| 340 | + if not isinstance(context, list): |
| 341 | + return |
| 342 | + index = next((i for i, e in enumerate(context) if isinstance(e, dict)), -1) |
| 343 | + if not had_generated_section: |
| 344 | + context.insert(len(context) if index == -1 else index, {}) |
| 345 | + return |
| 346 | + if index != -1: |
| 347 | + context[index] = {} |
| 348 | + else: |
| 349 | + context.append({}) |
| 350 | + |
| 351 | + |
| 352 | +def merge_generated_jsonld_context(schema: dict, generated_context: Any) -> None: |
| 353 | + """Merges the @context of a generated schema into the schema. |
| 354 | +
|
| 355 | + Port of the @context handling of osl.util.postProcessPage in PageBot.js. The |
| 356 | + generated mappings are merged into the first object block of the @context |
| 357 | + array only, which reset_generated_jsonld_context() reserved. oold's |
| 358 | + aggregate_generated_schemas() collapses all object blocks into one, which |
| 359 | + would fuse a hand written block with the generated one. String entries |
| 360 | + (parent schema references) of the generated context are appended when new. |
| 361 | + Modifies schema in place. |
| 362 | + """ |
| 363 | + existing = schema.get("@context") |
| 364 | + if isinstance(existing, list): |
| 365 | + context_list = existing |
| 366 | + else: |
| 367 | + context_list = [existing] if existing else [] |
| 368 | + entries = ( |
| 369 | + generated_context |
| 370 | + if isinstance(generated_context, list) |
| 371 | + else [generated_context] |
| 372 | + ) |
| 373 | + generated_object: dict = {} |
| 374 | + for entry in entries: |
| 375 | + if isinstance(entry, dict): |
| 376 | + generated_object = merge_deep(generated_object, entry) |
| 377 | + elif entry and entry not in context_list: |
| 378 | + context_list.append(entry) |
| 379 | + index = next((i for i, e in enumerate(context_list) if isinstance(e, dict)), -1) |
| 380 | + if index == -1: |
| 381 | + context_list.append(generated_object) |
| 382 | + else: |
| 383 | + context_list[index] = merge_deep(context_list[index], generated_object) |
| 384 | + schema["@context"] = context_list |
| 385 | + |
| 386 | + |
| 387 | +def strip_root_members_superseded_by_generated( |
| 388 | + root_schema: dict, generated_schema: dict |
| 389 | +) -> bool: |
| 390 | + """Removes the root members the generated section defines. |
| 391 | +
|
| 392 | + Port of osl.util.stripRootMembersSupersededByGenerated in PageBot.js. A member |
| 393 | + declared at root takes precedence over the same member pulled in through |
| 394 | + allOf, so a copy of the generated members at root would shadow the freshly |
| 395 | + generated ones. Containers are searched recursively, manual siblings stay. |
| 396 | + Members that are written to root on purpose are skipped. The allOf entries that |
| 397 | + the generated section repeats are removed from root. Modifies root_schema in |
| 398 | + place and returns True when anything was removed. |
| 399 | + """ |
| 400 | + promoted_to_root = { |
| 401 | + "title", |
| 402 | + "title*", |
| 403 | + "description", |
| 404 | + "description*", |
| 405 | + "@context", |
| 406 | + "$defs", |
| 407 | + "allOf", |
| 408 | + "$ref", |
| 409 | + "$comment", |
| 410 | + } |
| 411 | + removed = False |
| 412 | + |
| 413 | + def strip(target, generated, is_root): |
| 414 | + nonlocal removed |
| 415 | + if not isinstance(target, dict) or not isinstance(generated, dict): |
| 416 | + return |
| 417 | + for key in list(generated.keys()): |
| 418 | + if is_root and key in promoted_to_root: |
| 419 | + continue |
| 420 | + if key not in target: |
| 421 | + continue |
| 422 | + if isinstance(generated[key], dict) and isinstance(target[key], dict): |
| 423 | + strip(target[key], generated[key], False) |
| 424 | + if len(target[key]) == 0: |
| 425 | + del target[key] |
| 426 | + removed = True |
| 427 | + else: |
| 428 | + del target[key] |
| 429 | + removed = True |
| 430 | + |
| 431 | + strip(root_schema, generated_schema, True) |
| 432 | + |
| 433 | + generated_allof = generated_schema.get("allOf") |
| 434 | + generated_allof = generated_allof if isinstance(generated_allof, list) else [] |
| 435 | + if isinstance(root_schema.get("allOf"), list) and generated_allof: |
| 436 | + generated_refs = [ |
| 437 | + e["$ref"] for e in generated_allof if isinstance(e, dict) and e.get("$ref") |
| 438 | + ] |
| 439 | + kept = [ |
| 440 | + e |
| 441 | + for e in root_schema["allOf"] |
| 442 | + if not (isinstance(e, dict) and e.get("$ref") in generated_refs) |
| 443 | + ] |
| 444 | + if len(kept) != len(root_schema["allOf"]): |
| 445 | + root_schema["allOf"] = kept |
| 446 | + removed = True |
| 447 | + return removed |
| 448 | + |
| 449 | + |
236 | 450 | # Reusable type definitions |
237 | 451 | class OverwriteOptions(Enum): |
238 | 452 | """Options for overwriting properties""" |
@@ -1984,33 +2198,34 @@ def store_entity( |
1984 | 2198 | """Titles of the pages an edit was sent for, to be verified below.""" |
1985 | 2199 |
|
1986 | 2200 | meta_category_templates = {} |
| 2201 | + definitions_section = False |
1987 | 2202 | if param.namespace == "Category": |
1988 | 2203 | meta_category_titles = param.meta_category_title |
1989 | 2204 | if not isinstance(meta_category_titles, list): |
1990 | 2205 | meta_category_titles = [meta_category_titles] |
1991 | | - meta_category_template_strs = {} |
1992 | | - # We have to do this iteratively to support meta categories inheritance |
1993 | | - while meta_category_titles is not None and len(meta_category_titles) > 0: |
1994 | | - meta_categories = self.site.get_page( |
1995 | | - WtSite.GetPageParam(titles=meta_category_titles) |
1996 | | - ).pages |
1997 | | - for meta_category in meta_categories: |
1998 | | - meta_category_template_strs[meta_category.title] = ( |
1999 | | - meta_category.get_slot_content("schema_template") |
2000 | | - ) |
2001 | 2206 |
|
2002 | | - meta_category_titles = meta_category.get_slot_content("jsondata").get( |
2003 | | - "subclass_of" |
2004 | | - ) |
| 2207 | + def get_pages(titles): |
| 2208 | + return self.site.get_page(WtSite.GetPageParam(titles=titles)).pages |
2005 | 2209 |
|
2006 | | - for title in meta_category_template_strs.keys(): |
2007 | | - meta_category_template_str = meta_category_template_strs[title] |
2008 | | - if meta_category_template_str: |
2009 | | - meta_category_templates[title] = compile_handlebars_template( |
2010 | | - meta_category_template_str |
| 2210 | + # The templates follow the allOf chain of the schema of the meta |
| 2211 | + # categories, like the form editor does. subclass_of is not used: |
| 2212 | + # Category:Category has no subclass_of but an allOf to CharacteristicType |
| 2213 | + template_titles = collect_category_template_titles( |
| 2214 | + get_pages, meta_category_titles |
| 2215 | + ) |
| 2216 | + for page in get_pages(template_titles): |
| 2217 | + template_str = page.get_slot_content("schema_template") |
| 2218 | + if template_str: |
| 2219 | + meta_category_templates[page.title] = compile_handlebars_template( |
| 2220 | + template_str |
2011 | 2221 | ) |
2012 | | - # inverse order to have the most generic template first |
2013 | | - meta_category_templates = dict(reversed(meta_category_templates.items())) |
| 2222 | + definitions_section = CHARACTERISTIC_TYPE_TITLE in template_titles |
| 2223 | + # keep the order of the list, the most generic template first |
| 2224 | + meta_category_templates = { |
| 2225 | + t: meta_category_templates[t] |
| 2226 | + for t in template_titles |
| 2227 | + if t in meta_category_templates |
| 2228 | + } |
2014 | 2229 |
|
2015 | 2230 | def store_entity_( |
2016 | 2231 | entity_: model.Entity, |
@@ -2092,36 +2307,43 @@ def store_entity_( |
2092 | 2307 | ) |
2093 | 2308 |
|
2094 | 2309 | mode = AggregateGeneratedSchemasParamMode.ROOT_LEVEL |
2095 | | - # Put generated schema in definitions section, |
2096 | | - # currently only enabled for Characteristics. The class is |
2097 | | - # either the locally generated one or the one shipped by the |
2098 | | - # opensemantic packages, depending on where the model is from. |
2099 | | - characteristic_types = tuple( |
2100 | | - t |
2101 | | - for t in ( |
2102 | | - getattr(model, "CharacteristicType", None), |
2103 | | - _OpensemanticCharacteristicType, |
2104 | | - ) |
2105 | | - if t is not None |
2106 | | - ) |
2107 | | - if characteristic_types and isinstance(entity_, characteristic_types): |
| 2310 | + # Put the generated schema in the definitions section when the |
| 2311 | + # templates include CharacteristicType, like the form editor does. |
| 2312 | + if definitions_section: |
2108 | 2313 | mode = AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION |
| 2314 | + definitions = ( |
| 2315 | + mode == AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION |
| 2316 | + ) |
2109 | 2317 |
|
2110 | 2318 | stored_schema = page.get_slot_content("jsonschema") or {} |
2111 | 2319 | # Regenerate the generated section from scratch. Skip this when |
2112 | 2320 | # the generation failed, so the stored section is kept. |
2113 | | - if ( |
2114 | | - mode == AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION |
2115 | | - and len(generated_schemas) > 0 |
2116 | | - ): |
| 2321 | + if definitions and len(generated_schemas) > 0: |
| 2322 | + reset_generated_jsonld_context( |
| 2323 | + stored_schema, |
| 2324 | + isinstance(stored_schema.get("$defs"), dict) |
| 2325 | + and "generated" in stored_schema["$defs"], |
| 2326 | + ) |
2117 | 2327 | remove_generated_schema_section(stored_schema) |
| 2328 | + if definitions: |
| 2329 | + for generated_schema in generated_schemas.values(): |
| 2330 | + if "@context" in generated_schema: |
| 2331 | + merge_generated_jsonld_context( |
| 2332 | + stored_schema, generated_schema.pop("@context") |
| 2333 | + ) |
2118 | 2334 | new_schema = aggregate_generated_schemas( |
2119 | 2335 | AggregateGeneratedSchemasParam( |
2120 | 2336 | target_schema=stored_schema, |
2121 | 2337 | generated_schemas=generated_schemas, |
2122 | 2338 | mode=mode, |
2123 | 2339 | ) |
2124 | 2340 | ).aggregated_schema |
| 2341 | + if definitions and isinstance( |
| 2342 | + (new_schema.get("$defs") or {}).get("generated"), dict |
| 2343 | + ): |
| 2344 | + strip_root_members_superseded_by_generated( |
| 2345 | + new_schema, new_schema["$defs"]["generated"] |
| 2346 | + ) |
2125 | 2347 | page.set_slot_content("jsonschema", new_schema) |
2126 | 2348 | if param.offline is False and not kept_existing: |
2127 | 2349 | page.edit( |
|
0 commit comments