Skip to content

Commit 9f606f4

Browse files
committed
fix: select category templates from the schema allOf like the wiki (#220)
- collect templates from the meta category allOf chain, wiki order - pick the definitions section mode by CharacteristicType in the list - port @context reset and merge, and superseded root member removal - warn when a meta category schema is missing
1 parent 9b76a8f commit 9f606f4

13 files changed

Lines changed: 3757 additions & 55 deletions

‎src/osw/core.py‎

Lines changed: 268 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,6 @@
3939
from pyld import jsonld
4040

4141
import osw.model.entity as model
42-
43-
try:
44-
# Characteristics of the released opensemantic packages derive from this
45-
# class, not from the one in osw.model.entity.
46-
from opensemantic.core.v1 import (
47-
CharacteristicType as _OpensemanticCharacteristicType,
48-
)
49-
except ImportError: # pragma: no cover - optional dependency
50-
_OpensemanticCharacteristicType = None
5142
from osw.defaults import params as default_params
5243
from osw.utils.code_postprocessing import (
5344
remove_constraints_from_forward_refs,
@@ -58,6 +49,7 @@
5849
AggregateGeneratedSchemasParamMode,
5950
aggregate_generated_schemas,
6051
escape_json_strings,
52+
merge_deep,
6153
merge_generated_definitions,
6254
)
6355
from osw.utils.templates import (
@@ -68,6 +60,7 @@
6860
from osw.utils.wiki import (
6961
get_full_title,
7062
get_namespace,
63+
get_osw_id,
7164
get_title,
7265
is_empty,
7366
namespace_from_full_title,
@@ -233,6 +226,227 @@ def remove_generated_schema_section(
233226
return schema
234227

235228

229+
CATEGORY_CATEGORY_UUID = "89aafe6d-ae5a-4f29-97ff-df7736d4cab6"
230+
CATEGORY_ENTITY_UUID = "ce353767-c628-45bd-9d88-d6eb3009aec0"
231+
# the definitions section mode is selected when this template is in the list
232+
CHARACTERISTIC_TYPE_TITLE = "Category:OSWffe74f291d354037b318c422591c5023"
233+
234+
235+
def _title_from_schema_ref(ref: Any) -> Optional[str]:
236+
"""Page title of a $ref to a category schema, None for any other $ref.
237+
238+
Accepts '/wiki/Category:X?action=raw&slot=jsonschema' and
239+
'.../Special:SlotResolver/Category/X.slot_jsonschema.json'.
240+
"""
241+
if not isinstance(ref, str) or ref.startswith("#"):
242+
return None
243+
path = ref.split("?")[0]
244+
match = re.search(r"/Special:SlotResolver/([^/]+)/(.+)\.slot_[a-z]+\.json$", path)
245+
if match:
246+
return f"{match.group(1)}:{match.group(2)}"
247+
match = re.search(r"/wiki/(Category:[^/]+)$", path)
248+
if match:
249+
return match.group(1)
250+
if re.fullmatch(r"Category:[^/]+", path):
251+
return path
252+
return None
253+
254+
255+
def collect_category_template_titles(
256+
get_pages, meta_category_titles: List[str]
257+
) -> List[str]:
258+
"""Titles of the category templates the wiki form editor applies.
259+
260+
Mirrors the editor: the schema of the meta categories is bundled, the uuid of
261+
every schema is collected after the schemas below it (allOf, oneOf, anyOf,
262+
definitions and $defs are followed in this order, so the most generic schema
263+
comes first) and the result starts with Category:Category.
264+
Category:Category and Category:Entity are not collected. Follows
265+
osl.util.postProcessPage (PageBot.js) with editor.jsonschema.subschemas_uuids
266+
(MwJson_schema.js, _preprocess) in mediawiki-extensions-OpenSemanticLab and
267+
mediawiki-extensions-MwJson. Schemas reached through properties are not
268+
followed.
269+
270+
Parameters
271+
----------
272+
get_pages
273+
callable that returns the page objects for a list of titles
274+
meta_category_titles
275+
titles of the meta categories, e.g. ["Category:Category"]
276+
"""
277+
templates = ["Category:Category"]
278+
schema_cache: Dict[str, dict] = {}
279+
visited = set()
280+
281+
def schema_of(title: str) -> dict:
282+
if title not in schema_cache:
283+
schema = {}
284+
for page in get_pages([title]):
285+
if page.title == title:
286+
schema = page.get_slot_content("jsonschema") or {}
287+
if not schema:
288+
_logger.warning(
289+
f"No jsonschema found for {title}. Its templates are not "
290+
"applied, which can select the wrong schema generation mode."
291+
)
292+
schema_cache[title] = schema
293+
return schema_cache[title]
294+
295+
def walk(schema: Any) -> None:
296+
if not isinstance(schema, dict):
297+
return
298+
ref_title = _title_from_schema_ref(schema.get("$ref"))
299+
if ref_title is not None:
300+
# a bundled schema is included once, later refs point to it
301+
if ref_title not in visited:
302+
visited.add(ref_title)
303+
walk(schema_of(ref_title))
304+
return
305+
for key in ("allOf", "oneOf", "anyOf"):
306+
sub = schema.get(key)
307+
for entry in sub if isinstance(sub, list) else [sub]:
308+
walk(entry)
309+
for key in ("definitions", "$defs"):
310+
if isinstance(schema.get(key), dict):
311+
for entry in schema[key].values():
312+
walk(entry)
313+
uuid = schema.get("uuid")
314+
if uuid and uuid not in (CATEGORY_CATEGORY_UUID, CATEGORY_ENTITY_UUID):
315+
title = "Category:" + get_osw_id(uuid)
316+
if title not in templates:
317+
templates.append(title)
318+
319+
for title in meta_category_titles:
320+
visited.add(title)
321+
walk(schema_of(title))
322+
return templates
323+
324+
325+
def reset_generated_jsonld_context(
326+
schema: Optional[dict], had_generated_section: bool
327+
) -> None:
328+
"""Empties the generated first object block of the @context array.
329+
330+
Port of osl.util.resetGeneratedJsonLdContext in PageBot.js. The generated
331+
property mappings are by convention the first object entry of an @context
332+
array, the strings before it are parent schema references. Without a generated
333+
section the first object block is hand written, so an empty block is inserted
334+
ahead of it. A string or object @context is left as it is. Modifies schema in
335+
place.
336+
"""
337+
if not schema:
338+
return
339+
context = schema.get("@context")
340+
if not isinstance(context, list):
341+
return
342+
index = next((i for i, e in enumerate(context) if isinstance(e, dict)), -1)
343+
if not had_generated_section:
344+
context.insert(len(context) if index == -1 else index, {})
345+
return
346+
if index != -1:
347+
context[index] = {}
348+
else:
349+
context.append({})
350+
351+
352+
def merge_generated_jsonld_context(schema: dict, generated_context: Any) -> None:
353+
"""Merges the @context of a generated schema into the schema.
354+
355+
Port of the @context handling of osl.util.postProcessPage in PageBot.js. The
356+
generated mappings are merged into the first object block of the @context
357+
array only, which reset_generated_jsonld_context() reserved. oold's
358+
aggregate_generated_schemas() collapses all object blocks into one, which
359+
would fuse a hand written block with the generated one. String entries
360+
(parent schema references) of the generated context are appended when new.
361+
Modifies schema in place.
362+
"""
363+
existing = schema.get("@context")
364+
if isinstance(existing, list):
365+
context_list = existing
366+
else:
367+
context_list = [existing] if existing else []
368+
entries = (
369+
generated_context
370+
if isinstance(generated_context, list)
371+
else [generated_context]
372+
)
373+
generated_object: dict = {}
374+
for entry in entries:
375+
if isinstance(entry, dict):
376+
generated_object = merge_deep(generated_object, entry)
377+
elif entry and entry not in context_list:
378+
context_list.append(entry)
379+
index = next((i for i, e in enumerate(context_list) if isinstance(e, dict)), -1)
380+
if index == -1:
381+
context_list.append(generated_object)
382+
else:
383+
context_list[index] = merge_deep(context_list[index], generated_object)
384+
schema["@context"] = context_list
385+
386+
387+
def strip_root_members_superseded_by_generated(
388+
root_schema: dict, generated_schema: dict
389+
) -> bool:
390+
"""Removes the root members the generated section defines.
391+
392+
Port of osl.util.stripRootMembersSupersededByGenerated in PageBot.js. A member
393+
declared at root takes precedence over the same member pulled in through
394+
allOf, so a copy of the generated members at root would shadow the freshly
395+
generated ones. Containers are searched recursively, manual siblings stay.
396+
Members that are written to root on purpose are skipped. The allOf entries that
397+
the generated section repeats are removed from root. Modifies root_schema in
398+
place and returns True when anything was removed.
399+
"""
400+
promoted_to_root = {
401+
"title",
402+
"title*",
403+
"description",
404+
"description*",
405+
"@context",
406+
"$defs",
407+
"allOf",
408+
"$ref",
409+
"$comment",
410+
}
411+
removed = False
412+
413+
def strip(target, generated, is_root):
414+
nonlocal removed
415+
if not isinstance(target, dict) or not isinstance(generated, dict):
416+
return
417+
for key in list(generated.keys()):
418+
if is_root and key in promoted_to_root:
419+
continue
420+
if key not in target:
421+
continue
422+
if isinstance(generated[key], dict) and isinstance(target[key], dict):
423+
strip(target[key], generated[key], False)
424+
if len(target[key]) == 0:
425+
del target[key]
426+
removed = True
427+
else:
428+
del target[key]
429+
removed = True
430+
431+
strip(root_schema, generated_schema, True)
432+
433+
generated_allof = generated_schema.get("allOf")
434+
generated_allof = generated_allof if isinstance(generated_allof, list) else []
435+
if isinstance(root_schema.get("allOf"), list) and generated_allof:
436+
generated_refs = [
437+
e["$ref"] for e in generated_allof if isinstance(e, dict) and e.get("$ref")
438+
]
439+
kept = [
440+
e
441+
for e in root_schema["allOf"]
442+
if not (isinstance(e, dict) and e.get("$ref") in generated_refs)
443+
]
444+
if len(kept) != len(root_schema["allOf"]):
445+
root_schema["allOf"] = kept
446+
removed = True
447+
return removed
448+
449+
236450
# Reusable type definitions
237451
class OverwriteOptions(Enum):
238452
"""Options for overwriting properties"""
@@ -1984,33 +2198,34 @@ def store_entity(
19842198
"""Titles of the pages an edit was sent for, to be verified below."""
19852199

19862200
meta_category_templates = {}
2201+
definitions_section = False
19872202
if param.namespace == "Category":
19882203
meta_category_titles = param.meta_category_title
19892204
if not isinstance(meta_category_titles, list):
19902205
meta_category_titles = [meta_category_titles]
1991-
meta_category_template_strs = {}
1992-
# We have to do this iteratively to support meta categories inheritance
1993-
while meta_category_titles is not None and len(meta_category_titles) > 0:
1994-
meta_categories = self.site.get_page(
1995-
WtSite.GetPageParam(titles=meta_category_titles)
1996-
).pages
1997-
for meta_category in meta_categories:
1998-
meta_category_template_strs[meta_category.title] = (
1999-
meta_category.get_slot_content("schema_template")
2000-
)
20012206

2002-
meta_category_titles = meta_category.get_slot_content("jsondata").get(
2003-
"subclass_of"
2004-
)
2207+
def get_pages(titles):
2208+
return self.site.get_page(WtSite.GetPageParam(titles=titles)).pages
20052209

2006-
for title in meta_category_template_strs.keys():
2007-
meta_category_template_str = meta_category_template_strs[title]
2008-
if meta_category_template_str:
2009-
meta_category_templates[title] = compile_handlebars_template(
2010-
meta_category_template_str
2210+
# The templates follow the allOf chain of the schema of the meta
2211+
# categories, like the form editor does. subclass_of is not used:
2212+
# Category:Category has no subclass_of but an allOf to CharacteristicType
2213+
template_titles = collect_category_template_titles(
2214+
get_pages, meta_category_titles
2215+
)
2216+
for page in get_pages(template_titles):
2217+
template_str = page.get_slot_content("schema_template")
2218+
if template_str:
2219+
meta_category_templates[page.title] = compile_handlebars_template(
2220+
template_str
20112221
)
2012-
# inverse order to have the most generic template first
2013-
meta_category_templates = dict(reversed(meta_category_templates.items()))
2222+
definitions_section = CHARACTERISTIC_TYPE_TITLE in template_titles
2223+
# keep the order of the list, the most generic template first
2224+
meta_category_templates = {
2225+
t: meta_category_templates[t]
2226+
for t in template_titles
2227+
if t in meta_category_templates
2228+
}
20142229

20152230
def store_entity_(
20162231
entity_: model.Entity,
@@ -2092,36 +2307,43 @@ def store_entity_(
20922307
)
20932308

20942309
mode = AggregateGeneratedSchemasParamMode.ROOT_LEVEL
2095-
# Put generated schema in definitions section,
2096-
# currently only enabled for Characteristics. The class is
2097-
# either the locally generated one or the one shipped by the
2098-
# opensemantic packages, depending on where the model is from.
2099-
characteristic_types = tuple(
2100-
t
2101-
for t in (
2102-
getattr(model, "CharacteristicType", None),
2103-
_OpensemanticCharacteristicType,
2104-
)
2105-
if t is not None
2106-
)
2107-
if characteristic_types and isinstance(entity_, characteristic_types):
2310+
# Put the generated schema in the definitions section when the
2311+
# templates include CharacteristicType, like the form editor does.
2312+
if definitions_section:
21082313
mode = AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION
2314+
definitions = (
2315+
mode == AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION
2316+
)
21092317

21102318
stored_schema = page.get_slot_content("jsonschema") or {}
21112319
# Regenerate the generated section from scratch. Skip this when
21122320
# the generation failed, so the stored section is kept.
2113-
if (
2114-
mode == AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION
2115-
and len(generated_schemas) > 0
2116-
):
2321+
if definitions and len(generated_schemas) > 0:
2322+
reset_generated_jsonld_context(
2323+
stored_schema,
2324+
isinstance(stored_schema.get("$defs"), dict)
2325+
and "generated" in stored_schema["$defs"],
2326+
)
21172327
remove_generated_schema_section(stored_schema)
2328+
if definitions:
2329+
for generated_schema in generated_schemas.values():
2330+
if "@context" in generated_schema:
2331+
merge_generated_jsonld_context(
2332+
stored_schema, generated_schema.pop("@context")
2333+
)
21182334
new_schema = aggregate_generated_schemas(
21192335
AggregateGeneratedSchemasParam(
21202336
target_schema=stored_schema,
21212337
generated_schemas=generated_schemas,
21222338
mode=mode,
21232339
)
21242340
).aggregated_schema
2341+
if definitions and isinstance(
2342+
(new_schema.get("$defs") or {}).get("generated"), dict
2343+
):
2344+
strip_root_members_superseded_by_generated(
2345+
new_schema, new_schema["$defs"]["generated"]
2346+
)
21252347
page.set_slot_content("jsonschema", new_schema)
21262348
if param.offline is False and not kept_existing:
21272349
page.edit(

0 commit comments

Comments
 (0)