Skip to content

Commit d00ac77

Browse files
committed
feat: improve ontology import
- name imported properties camelCase - add support for change_id - set policy override=true - catch more errors - catch unsupported language
1 parent 572c6eb commit d00ac77

2 files changed

Lines changed: 86 additions & 29 deletions

File tree

‎src/osw/ontology.py‎

Lines changed: 83 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from osw.core import OSW, model
1313
from osw.model.static import OswBaseModel
14-
from osw.utils.strings import pascal_case
14+
from osw.utils.strings import camel_case, pascal_case
1515
from osw.utils.wiki import get_namespace
1616
from osw.wtsite import WtSite
1717

@@ -46,6 +46,7 @@ class ParserSettings(OswBaseModel):
4646
"subclass_of",
4747
"restrictions",
4848
"range",
49+
"subproperty_of",
4950
]
5051
"""List of properties that should be arrays. If the property is not an array, the
5152
parser will create an array with the value as single element."""
@@ -99,6 +100,8 @@ class ImportConfig(OswBaseModel):
99100
"""Path to the directory where the parsed ontology will be dumped"""
100101
dry_run: Optional[bool] = False
101102
"""If True, the parsed ontology will not be imported into the wiki"""
103+
change_id: Optional[str] = None
104+
"""The change id to keep track of the import, stored at meta.change_id"""
102105
property_naming_policy: Optional[Literal["UUID", "label", "prefixed_label"]] = (
103106
"prefixed_label"
104107
)
@@ -596,14 +599,52 @@ def _apply_osw_structure(self):
596599
if key in node:
597600
if isinstance(node[key], str):
598601
node[key] = {"text": node[key], "lang": "en"}
599-
elif "text" in node[key] and "lang" not in node[key]:
600-
node[key]["lang"] = "en"
602+
elif "text" in node[key]:
603+
if "lang" not in node[key]:
604+
node[key]["lang"] = "en"
605+
if "lang" in node[key] and node[key]["lang"] not in [
606+
"en",
607+
"de",
608+
]:
609+
# ToDo: Support all/more languages
610+
print(
611+
(
612+
"Warning: remove value with unsupported language: ",
613+
f"{node[key]['lang']}",
614+
)
615+
)
616+
del node[key]
617+
continue
601618
elif isinstance(node[key], list):
602619
for i, _val in enumerate(node[key]):
603620
if isinstance(node[key][i], str):
604621
node[key][i] = {"text": node[key][i], "lang": "en"}
605-
elif "text" in node[key][i] and "lang" not in node[key][i]:
606-
node[key][i]["lang"] = "en"
622+
elif "text" in node[key][i]:
623+
if "lang" not in node[key][i]:
624+
node[key][i]["lang"] = "en"
625+
elif "lang" in node[key][i] and node[key][i][
626+
"lang"
627+
] not in ["en", "de"]:
628+
# ToDo: Support all/more languages
629+
print(
630+
(
631+
"Warning: remove value with unsupported language: ",
632+
f"{node[key][i]['lang']}",
633+
)
634+
)
635+
del node[key][i]
636+
continue
637+
else:
638+
print(
639+
(
640+
"Warning: remove invalide multilang value: "
641+
f"{node[key][i]}"
642+
)
643+
)
644+
del node[key][i]
645+
else:
646+
print(f"Warning: remove invalide multilang value: {node[key]}")
647+
del node[key]
607648
for key in ps.ensure_array:
608649
if key in node and not isinstance(node[key], list):
609650
node[key] = [node[key]]
@@ -628,6 +669,8 @@ def _apply_osw_structure(self):
628669
):
629670
node["uuid"] = str(self._get_uuid_from_iri(node["iri"]))
630671

672+
namespace = self._get_page_name(node).split(":")[0]
673+
631674
# name = ""
632675
if "prefLabel" in node:
633676
# if isinstance(node["prefLabel"], str):
@@ -640,13 +683,18 @@ def _apply_osw_structure(self):
640683
node["name"] = node["label"][0]["text"]
641684
else:
642685
print("No label: ", node["iri"])
686+
# node["name"] = node["iri"].split("/")[-1].split("#")[-1]
687+
# warning(f"Fallback to name from iri: {node['name']}")
643688
if "name" in node:
644-
node["name"] = pascal_case(node["name"])
689+
# camel case for properties, else pascal case
690+
if namespace == "Property":
691+
node["name"] = camel_case(node["name"])
692+
else:
693+
node["name"] = pascal_case(node["name"])
645694

646695
# update iri dict
647696
self._iri_dict[node["iri"]] = node
648697

649-
namespace = self._get_page_name(node).split(":")[0]
650698
title = self._get_page_name(node).replace(namespace + ":", "")
651699
node["meta"] = {
652700
"wiki_page": {
@@ -683,8 +731,8 @@ def _apply_osw_structure(self):
683731
label = self._iri_dict[key]["label"][0]["text"]
684732
self._prop_dict[key] = label
685733
# print the inverted property_dict
686-
for key, value in self._prop_dict.items():
687-
print(f'"{value}": "{key}"')
734+
# for key, value in self._prop_dict.items():
735+
# print(f'"{value}": "{key}"')
688736

689737
def _map_iris(self, node_array):
690738
if not isinstance(node_array, list):
@@ -698,14 +746,12 @@ def _map_iris(self, node_array):
698746
node[key][i] = self._map_iri_to_osw(node[key][i])
699747
if isinstance(node[key], str):
700748
node[key] = self._map_iri_to_osw(node[key])
701-
if key == "restrictions":
702-
print(key)
703749
if isinstance(node[key], dict) or isinstance(node[key], list):
704750
self._map_iris(node[key])
705751

706752
def _create_entities(self):
707753
# create OSW entities
708-
limit = 3000 # choose a smaller number for tests
754+
limit = 10000 # choose a smaller number for tests
709755
counter = 0
710756
self._entities = []
711757
self._entities_json = []
@@ -716,21 +762,28 @@ def _create_entities(self):
716762

717763
if counter < limit:
718764
if "rdf_type" in node and "label" in node:
765+
719766
e = None
720-
if "owl:Class" in node["rdf_type"]:
721-
e = self.import_config.base_class(**node)
767+
try:
768+
if "owl:Class" in node["rdf_type"]:
769+
e = self.import_config.base_class(**node)
722770

723-
if "owl:ObjectProperty" in node["rdf_type"]:
724-
e = model.ObjectProperty(**node)
771+
if "owl:ObjectProperty" in node["rdf_type"]:
772+
e = model.ObjectProperty(**node)
725773

726-
if "owl:DatatypeProperty" in node["rdf_type"]:
727-
e = model.DataProperty(**node)
774+
if "owl:DatatypeProperty" in node["rdf_type"]:
775+
e = model.DataProperty(**node)
728776

729-
if "owl:AnnotationProperty" in node["rdf_type"]:
730-
e = model.AnnotationProperty(**node)
777+
if "owl:AnnotationProperty" in node["rdf_type"]:
778+
e = model.AnnotationProperty(**node)
731779

732-
if "owl:NamedIndividual" in node["rdf_type"]:
733-
e = model.OwlIndividual(**node)
780+
if "owl:NamedIndividual" in node["rdf_type"]:
781+
e = model.OwlIndividual(**node)
782+
783+
except Exception as ex:
784+
# print the validation error
785+
print("Exception while generating entity with ", node)
786+
print(ex)
734787

735788
if e:
736789
self._entities.append(e)
@@ -756,7 +809,7 @@ def _store_ontology(self, param: StoreOntologyParam):
756809
# see https://www.semantic-mediawiki.org/wiki/Help:Import_vocabulary
757810
if namespace == "Category":
758811
smw_import_type = "Category"
759-
if not hasattr(e, "subclass_of"):
812+
if not hasattr(e, "subclass_of") or e.subclass_of is None:
760813
e.subclass_of = []
761814
if len(e.subclass_of) == 0:
762815
e.subclass_of.append(self.import_config.meta_class_title)
@@ -780,7 +833,13 @@ def _store_ontology(self, param: StoreOntologyParam):
780833
import_page.set_slot_content("main", text)
781834
import_page.edit("import ontology")
782835

783-
self.osw.store_entity(OSW.StoreEntityParam(entities=param.entities))
836+
self.osw.store_entity(
837+
OSW.StoreEntityParam(
838+
entities=param.entities,
839+
overwrite=True,
840+
change_id=self.import_config.change_id,
841+
)
842+
)
784843

785844
@deprecated("use ontology.OntologyImporter.StoreOntologiesParam instead")
786845
class StoreOntologiesParam(model.OswBaseModel):

‎tests/integration/ontology_import_test.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,7 @@ def test_ontology_import(wiki_domain, wiki_username, wiki_password):
6767
)
6868

6969
# import ontologies
70-
importer = OntologyImporter(
71-
osw=osw_obj,
72-
)
70+
importer = OntologyImporter(osw=osw_obj)
7371
importer.import_ontology(import_config)
7472

7573
smw_import_page = wtsite.get_page(
@@ -91,10 +89,10 @@ def test_ontology_import(wiki_domain, wiki_username, wiki_password):
9189

9290
property_name = None
9391
if import_config.property_naming_policy == "label":
94-
property_name = "ObjectPropertyA"
92+
property_name = "objectPropertyA"
9593
elif import_config.property_naming_policy == "prefixed_label":
9694
property_name = (
97-
f"example{import_config.property_naming_prefix_delimiter}ObjectPropertyA"
95+
f"example{import_config.property_naming_prefix_delimiter}objectPropertyA"
9896
)
9997
search_param = SearchParam(
10098
query=f"[[{property_name}::Category:OSWb1e6910f1e3d567aaed30b83ac887708]]"

0 commit comments

Comments
 (0)