Skip to content

Commit 0e5f321

Browse files
authored
Merge pull request #93 from OpenSemanticLab/feat-add-rdf-generation
Feat: add rdf generation
2 parents 0d68805 + 6676a39 commit 0e5f321

6 files changed

Lines changed: 510 additions & 32 deletions

File tree

‎examples/generate_rdf.py‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import os
2+
from uuid import UUID, uuid5
3+
4+
import osw.model.entity as model
5+
from osw.core import OSW
6+
from osw.express import OswExpress
7+
from osw.utils.wiki import get_full_title
8+
from osw.wtsite import WtSite
9+
10+
# Create/update the password file under examples/accounts.pwd.yaml
11+
pwd_file_path = os.path.join(
12+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "accounts.pwd.yaml"
13+
)
14+
# pwd_file_path = "./accounts.pwd.yaml"
15+
osw_obj = OswExpress(
16+
domain="wiki-dev.open-semantic-lab.org", cred_filepath=pwd_file_path
17+
)
18+
19+
# load dependencies
20+
DEPENDENCIES = {
21+
"Tool": "Category:OSWe427aafafbac4262955b9f690a83405d",
22+
}
23+
osw_obj.install_dependencies(DEPENDENCIES, mode="append", policy="if-missing")
24+
25+
# generate a UUID namespace
26+
EXAMPLE_UUID_NAMESPACE = UUID("bd65611d-8669-4903-8a14-af88203acc38")
27+
28+
sensor1 = model.Device(
29+
uuid=uuid5(EXAMPLE_UUID_NAMESPACE, "Sensor1"),
30+
name="Sensor1",
31+
label=[model.Label(text="Sensor 1")],
32+
description=[model.Description(text="A sensor")],
33+
)
34+
35+
sensor2 = model.Device(
36+
uuid=uuid5(EXAMPLE_UUID_NAMESPACE, "Sensor2"),
37+
name="Sensor2",
38+
label=[model.Label(text="Sensor 2")],
39+
description=[model.Description(text="Another sensor")],
40+
)
41+
42+
example_machine = model.Device(
43+
uuid=uuid5(EXAMPLE_UUID_NAMESPACE, "ExampleMachine"),
44+
name="ExampleMachine",
45+
label=[model.Label(text="Example Machine")],
46+
# note: components are modelled as statements to define
47+
# the relationship between the machine and the components
48+
components=[
49+
model.Component(
50+
# note: uuid format needs to be fixed in schema
51+
uuid=str(uuid5(EXAMPLE_UUID_NAMESPACE, "sensor_at_position_1")),
52+
component_id="sensor_at_position_1",
53+
label=[model.Label(text="Sensor at position 1")],
54+
component_instance=get_full_title(sensor1),
55+
),
56+
model.Component(
57+
uuid=str(uuid5(EXAMPLE_UUID_NAMESPACE, "sensor_at_position_2")),
58+
component_id="sensor_at_position_2",
59+
label=[model.Label(text="Sensor at position 2")],
60+
component_instance=get_full_title(sensor2),
61+
),
62+
],
63+
)
64+
65+
result = osw_obj.export_jsonld(
66+
params=OSW.ExportJsonLdParams(
67+
entities=[example_machine, sensor1, sensor2],
68+
mode=OSW.JsonLdMode.expand,
69+
build_rdf_graph=True,
70+
context_loader_config=WtSite.JsonLdContextLoaderParams(
71+
prefer_external_vocal=False
72+
),
73+
)
74+
)
75+
graph = result.graph
76+
# print all triples in the graph
77+
qres = graph.query(
78+
"""
79+
SELECT ?s ?p ?o
80+
WHERE {
81+
?s ?p ?o .
82+
}
83+
"""
84+
)
85+
86+
print("\nAll triples in the graph:")
87+
for row in qres:
88+
print(row.s, row.p, row.o)
89+
90+
# query all components of example_machine
91+
qres = graph.query(
92+
"""
93+
SELECT ?component ?clabel
94+
WHERE {
95+
?s Property:HasStatement ?statement .
96+
?statement Property:HasProperty Property:HasPart .
97+
?statement Property:HasObject ?component .
98+
?component Property:HasLabel ?clabel .
99+
}
100+
"""
101+
)
102+
print("\nComponents of example_machine:")
103+
for row in qres:
104+
print(row.clabel)
105+
106+
print("\n\nDefine a custom context:")
107+
108+
result = osw_obj.export_jsonld(
109+
params=OSW.ExportJsonLdParams(
110+
entities=[example_machine, sensor1, sensor2],
111+
mode=OSW.JsonLdMode.expand,
112+
build_rdf_graph=True,
113+
context={
114+
"ex": "http://example.org/",
115+
"components": "@nest",
116+
"component_instance": {"@id": "ex:hasPart", "@type": "@id"},
117+
"name": {"@id": "ex:name"},
118+
},
119+
)
120+
)
121+
graph = result.graph
122+
# print all triples in the graph
123+
qres = graph.query(
124+
"""
125+
SELECT ?s ?p ?o
126+
WHERE {
127+
?s ?p ?o .
128+
}
129+
"""
130+
)
131+
print("\nAll triples in the graph:")
132+
for row in qres:
133+
print(row.s, row.p, row.o)
134+
135+
# query all components of example_machine
136+
qres = graph.query(
137+
"""
138+
SELECT ?clabel
139+
WHERE {
140+
?s <http://example.org/hasPart> ?component .
141+
?component <http://example.org/name> ?clabel
142+
}
143+
"""
144+
)
145+
print("\nComponents of example_machine:")
146+
for row in qres:
147+
print(row.clabel)
148+
149+
150+
print(
151+
"\n\ncreate a custom class with an additional property mapped in a context extension"
152+
)
153+
154+
155+
class MyCustomDeviceClass(model.Device):
156+
my_property: str
157+
158+
159+
my_custom_device = example_machine.cast(MyCustomDeviceClass, my_property="test")
160+
161+
result = osw_obj.export_jsonld(
162+
params=OSW.ExportJsonLdParams(
163+
entities=[my_custom_device, sensor1, sensor2],
164+
mode=OSW.JsonLdMode.expand,
165+
build_rdf_graph=True,
166+
context_loader_config=WtSite.JsonLdContextLoaderParams(
167+
prefer_external_vocal=False
168+
),
169+
additional_context={"my_property": "http://example.org/my_property"},
170+
)
171+
)
172+
173+
graph = result.graph
174+
# print all triples in the graph
175+
qres = graph.query(
176+
"""
177+
SELECT ?s ?p ?o
178+
WHERE {
179+
?s ?p ?o .
180+
}
181+
"""
182+
)
183+
print("\nAll triples in the graph:")
184+
for row in qres:
185+
print(row.s, row.p, row.o)

‎setup.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ testing =
113113
sqlalchemy
114114
psycopg2-binary # see: https://www.psycopg.org/docs/install.html#psycopg-vs-psycopg-binary
115115
mwparserfromhell
116+
jsondiff
116117

117118
[options.entry_points]
118119
# Add here console scripts like:

‎src/osw/core.py‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88
import sys
99
from copy import deepcopy
1010
from enum import Enum
11-
from typing import Dict, List, Optional, Type, Union
11+
from typing import Any, Dict, List, Optional, Type, Union
1212
from uuid import UUID
1313
from warnings import warn
1414

15+
import rdflib
1516
from jsonpath_ng.ext import parse
1617
from pydantic.v1 import BaseModel, PrivateAttr, create_model, validator
18+
from pyld import jsonld
1719

1820
import osw.model.entity as model
1921
from osw.model.static import OswBaseModel
@@ -23,6 +25,7 @@
2325
)
2426
from osw.utils.util import parallelize
2527
from osw.utils.wiki import (
28+
get_full_title,
2629
get_namespace,
2730
get_title,
2831
get_uuid,
@@ -1300,6 +1303,126 @@ def query_instances(
13001303
full_page_titles = self.site.semantic_search(search_param)
13011304
return full_page_titles
13021305

1306+
class JsonLdMode(str, Enum):
1307+
"""enum for jsonld processing mode"""
1308+
1309+
expand = "expand"
1310+
flatten = "flatten"
1311+
compact = "compact"
1312+
frame = "frame"
1313+
1314+
class ExportJsonLdParams(OswBaseModel):
1315+
context_loader_config: Optional[WtSite.JsonLdContextLoaderParams] = None
1316+
"""The configuration for the JSON-LD context loader."""
1317+
entities: Union[OswBaseModel, List[OswBaseModel]]
1318+
"""The entities to convert to JSON-LD. Can be a single entity or a list of
1319+
entities."""
1320+
resolve_context: Optional[bool] = True
1321+
"""If True, remote context URLs are resolved."""
1322+
mode: Optional[OSW.JsonLdMode] = "expand"
1323+
"""The JSON-LD processing mode to apply if resolve_context is True."""
1324+
context: Optional[Union[str, list, Dict[str, Any]]] = None
1325+
"""The JSON-LD context to apply. Replaces any existing context."""
1326+
additional_context: Optional[Union[str, list, Dict[str, Any]]] = None
1327+
"""The JSON-LD context to apply on top of the existing context."""
1328+
frame: Optional[Dict[str, Any]] = None
1329+
"""The JSON-LD frame to use for framed mode. If not set, the existing context is used"""
1330+
build_rdf_graph: Optional[bool] = False
1331+
"""If True, the output is a graph."""
1332+
debug: Optional[bool] = False
1333+
1334+
def __init__(self, **data):
1335+
super().__init__(**data)
1336+
if not isinstance(self.entities, list):
1337+
self.entities = [self.entities]
1338+
1339+
class ExportJsonLdResult(OswBaseModel):
1340+
documents: List[Union[Dict[str, Any]]]
1341+
"""A single JSON-LD document per entity"""
1342+
graph_document: Dict[str, Any] = None
1343+
"""A single JSON-LD document with a @graph element containing all entities"""
1344+
graph: rdflib.Graph = None
1345+
"""RDF graph containing all entities. Build only if build_rdf_graph is True"""
1346+
1347+
class Config:
1348+
arbitrary_types_allowed = True
1349+
1350+
def export_jsonld(self, params: ExportJsonLdParams) -> ExportJsonLdResult:
1351+
"""Exports the given entity/entities as JSON-LD."""
1352+
1353+
if params.resolve_context:
1354+
jsonld.set_document_loader(
1355+
self.site.get_jsonld_context_loader(params.context_loader_config)
1356+
)
1357+
1358+
documents = []
1359+
graph_document = {"@graph": []}
1360+
graph = None
1361+
if params.build_rdf_graph:
1362+
graph = rdflib.Graph()
1363+
prefixes = self.site.get_prefix_dict()
1364+
for prefix in prefixes:
1365+
graph.bind(prefix, prefixes[prefix])
1366+
1367+
for e in params.entities:
1368+
data = json.loads(e.json(exclude_none=True, indent=4, ensure_ascii=False))
1369+
1370+
data["@context"] = []
1371+
if params.context is None:
1372+
for t in e.type:
1373+
data["@context"].append("/wiki/" + t)
1374+
if params.context is not None:
1375+
data["@context"].append(params.context)
1376+
else:
1377+
data["@context"] = {
1378+
**self.site.get_jsonld_context_prefixes(),
1379+
**params.context,
1380+
}
1381+
if params.additional_context is not None:
1382+
if data["@context"] is None:
1383+
data["@context"] = []
1384+
elif not isinstance(data["@context"], list):
1385+
data["@context"] = [data["@context"]]
1386+
data["@context"].append(params.additional_context)
1387+
1388+
data["@id"] = get_full_title(e)
1389+
1390+
if params.resolve_context:
1391+
graph_document["@graph"].append(jsonld.expand(data))
1392+
if params.mode == "expand":
1393+
data = jsonld.expand(data)
1394+
if isinstance(data, list):
1395+
data = data[0]
1396+
elif params.mode == "flatten":
1397+
data = jsonld.flatten(data)
1398+
elif params.mode == "compact":
1399+
# data = jsonld.expand(data)
1400+
# if isinstance(data, list): data = data[0]
1401+
data = jsonld.compact(
1402+
data,
1403+
data["@context"] if params.context is None else params.context,
1404+
)
1405+
elif params.mode == "frame":
1406+
data = jsonld.frame(
1407+
data,
1408+
(
1409+
{"@context": data["@context"]}
1410+
if params.frame is None
1411+
else params.frame
1412+
),
1413+
)
1414+
1415+
if params.build_rdf_graph:
1416+
graph.parse(data=json.dumps(data), format="json-ld")
1417+
1418+
documents.append(data)
1419+
1420+
result = OSW.ExportJsonLdResult(
1421+
documents=documents, graph_document=graph_document, graph=graph
1422+
)
1423+
return result
1424+
13031425

13041426
OSW._ApplyOverwriteParam.update_forward_refs()
13051427
OSW.StoreEntityParam.update_forward_refs()
1428+
OSW.ExportJsonLdParams.update_forward_refs()

‎src/osw/ontology.py‎

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import uuid
55
from typing import Dict, List, Literal, Optional, Type
66

7-
import pyld
87
from pydantic.v1 import PrivateAttr
98
from pyld import jsonld
109
from rdflib import Graph
@@ -214,36 +213,7 @@ def import_ontology(self, config: ImportConfig):
214213
"""
215214

216215
# overwrite the default document loader to load relative context from the wiki
217-
def myloader(*args, **kwargs):
218-
requests_loader = pyld.documentloader.requests.requests_document_loader(
219-
*args, **kwargs
220-
)
221-
222-
def loader(url, options=None):
223-
if options is None:
224-
options = {}
225-
if "/wiki/" in url:
226-
title = url.replace("/wiki/", "").split("?")[0]
227-
page = self.osw.site.get_page(
228-
WtSite.GetPageParam(titles=[title])
229-
).pages[0]
230-
schema = page.get_slot_content("jsonschema")
231-
if isinstance(schema, str):
232-
schema = json.loads(schema)
233-
doc = {
234-
"contentType": "application/json",
235-
"contextUrl": None,
236-
"documentUrl": url,
237-
"document": schema,
238-
}
239-
return doc
240-
241-
else:
242-
return requests_loader(url, options)
243-
244-
return loader
245-
246-
jsonld.set_document_loader(myloader())
216+
jsonld.set_document_loader(self.osw.site.get_jsonld_context_loader())
247217

248218
self.import_config = config
249219

0 commit comments

Comments
 (0)