feat(generators): add --deterministic flag with diff-stable WL hashing

Layers Weisfeiler-Lehman structural hashing on top of upstream RDFC-1.0
canonicalization (linkml#3407) to produce diff-stable blank node
identifiers. RDFC-1.0 remains always-on as the default serialization;
the --deterministic flag adds WL hashing for version-controlled artifacts.

Three-phase pipeline (deterministic_turtle):
1. RDFC-1.0 via pyoxigraph — canonical triple ordering
2. WL structural hashing — content-based blank node IDs
3. rdflib re-serialization — idiomatic Turtle syntax

Additional --deterministic behaviours:
- deterministic_json() for JSON-LD context output
- Sorted owl:oneOf, sh:in, sh:ignoredProperties members
- Sorted any_of/exactly_one_of expression members in OWL

Fixes from review (#1):
- Remove trailing newline from context generator return values
  (avoids double-newline when CLI prints output)
- Sort WL collision counter by signature, not c14n ID
  (prevents unrelated triples from swapping _0/_1 suffixes)
- Remove dead code: well_known_prefix_map(), normalize_prefixes
  (belong in separate --normalize-prefixes PR)

Signed-off-by: Carlo van Driesten <carlo.van-driesten@bmw.de>

Author: jdsika

packages/linkml/src/linkml/generators/jsonldcontextgen.py

@@ -236,7 +236,61 @@ def end_schema(
             with open(frame_path, "w", encoding="UTF-8") as f:
                 json.dump(frame, f, indent=2, ensure_ascii=False)
 
-        return str(as_json(context)) + "\n"
+        if self.deterministic:
+            return self._deterministic_context_json(json.loads(str(as_json(context))), indent=3)
+        return str(as_json(context))
+
+    @staticmethod
+    def _deterministic_context_json(data: dict, indent: int = 3) -> str:
+        """Serialize a JSON-LD context with deterministic key ordering.
+
+        Preserves the conventional JSON-LD context structure:
+        1. ``comments`` block first (metadata)
+        2. ``@context`` block second, with:
+           a. ``@``-prefixed directives (``@vocab``, ``@base``) first
+           b. Prefix declarations (string values) second
+           c. Class/property term entries (object values) last
+        3. Each group sorted alphabetically within itself
+
+        Unlike :func:`deterministic_json`, this understands JSON-LD
+        conventions so that the output remains human-readable while
+        still being byte-identical across invocations.
+        """
+        from linkml.utils.generator import deterministic_json
+
+        ordered = {}
+
+        # 1. "comments" first (if present)
+        if "comments" in data:
+            ordered["comments"] = data["comments"]
+
+        # 2. "@context" with structured internal ordering
+        if "@context" in data:
+            ctx = data["@context"]
+            ordered_ctx = {}
+
+            # 2a. @-prefixed directives (@vocab, @base, etc.)
+            for k in sorted(k for k in ctx if k.startswith("@")):
+                ordered_ctx[k] = ctx[k]
+
+            # 2b. Prefix declarations (string values — short namespace URIs)
+            for k in sorted(k for k in ctx if not k.startswith("@") and isinstance(ctx[k], str)):
+                ordered_ctx[k] = ctx[k]
+
+            # 2c. Term definitions (object values) — deep-sorted for determinism
+            term_entries = {k: v for k, v in ctx.items() if not k.startswith("@") and not isinstance(v, str)}
+            sorted_terms = json.loads(deterministic_json(term_entries))
+            for k in sorted(sorted_terms):
+                ordered_ctx[k] = sorted_terms[k]
+
+            ordered["@context"] = ordered_ctx
+
+        # 3. Any remaining top-level keys
+        for k in sorted(data):
+            if k not in ordered:
+                ordered[k] = data[k]
+
+        return json.dumps(ordered, indent=indent, ensure_ascii=False)
 
     def visit_class(self, cls: ClassDefinition) -> bool:
         if self.exclude_imports and cls.name not in self._local_classes:

packages/linkml/src/linkml/generators/jsonldgen.py

@@ -1,5 +1,6 @@
 """Generate JSONld from a LinkML schema."""
 
+import json
 import os
 from collections.abc import Sequence
 from copy import deepcopy
@@ -203,6 +204,10 @@ def end_schema(self, context: str | Sequence[str] | None = None, context_kwargs:
                 self.schema["@context"].append({"@base": base_prefix})
         # json_obj["@id"] = self.schema.id
         out = str(as_json(self.schema, indent="  ")) + "\n"
+        if self.deterministic:
+            from linkml.utils.generator import deterministic_json
+
+            out = deterministic_json(json.loads(out), indent=2) + "\n"
         self.schema = self.original_schema
         return out
 

packages/linkml/src/linkml/generators/owlgen.py

@@ -43,6 +43,7 @@
 from linkml_runtime.utils.formatutils import camelcase, underscore
 from linkml_runtime.utils.introspection import package_schemaview
 from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph
+from linkml_runtime.utils.yamlutils import YAMLRoot
 
 logger = logging.getLogger(__name__)
 
@@ -55,6 +56,21 @@
 SWRLB = rdflib.Namespace("http://www.w3.org/2003/11/swrlb#")
 
 
+def _expression_sort_key(expr: YAMLRoot) -> str:
+    """Return a stable sort key for LinkML anonymous expressions.
+
+    Used by ``--deterministic`` to order ``any_of``, ``all_of``,
+    ``none_of``, and ``exactly_one_of`` members reproducibly.
+
+    This relies on ``YAMLRoot.__repr__()`` which formats objects using
+    their **field values** (not memory addresses).  All anonymous
+    expression dataclasses in ``linkml_runtime.linkml_model.meta``
+    use ``@dataclass(repr=False)`` and inherit this field-based repr,
+    so the output is deterministic across runs.
+    """
+    return repr(expr)
+
+
 @unique
 class MetadataProfile(Enum):
     """
@@ -300,7 +316,14 @@ def serialize(self, **kwargs: Any) -> str:
         """
         self.as_graph()
         fmt = "turtle" if self.format in ["owl", "ttl"] else self.format
-        return canonicalize_rdf_graph(self.graph, output_format=fmt)
+        if self.deterministic and fmt == "turtle":
+            # Deferred to avoid circular import (generator.py imports from this package)
+            from linkml.utils.generator import deterministic_turtle
+
+            data = deterministic_turtle(self.graph)
+        else:
+            data = canonicalize_rdf_graph(self.graph, output_format=fmt)
+        return data
 
     def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None:
         """
@@ -568,25 +591,29 @@ def transform_class_expression(
         own_slots = self.get_own_slots(cls)
         owl_exprs: list[OWL_EXPRESSION] = []
         if cls.any_of:
-            any_of_expr = self._union_of([self.transform_class_expression(x) for x in cls.any_of])
+            members = list(cls.any_of)
+            if self.deterministic:
+                members = sorted(members, key=_expression_sort_key)
+            any_of_expr = self._union_of([self.transform_class_expression(x) for x in members])
             if any_of_expr:
                 owl_exprs.append(any_of_expr)
         if cls.exactly_one_of:
-            sub_exprs: list[OWL_EXPRESSION] = self._present(
-                self.transform_class_expression(x) for x in cls.exactly_one_of
-            )
+            members = list(cls.exactly_one_of)
+            if self.deterministic:
+                members = sorted(members, key=_expression_sort_key)
+            sub_exprs: list[OWL_EXPRESSION] = self._present(self.transform_class_expression(x) for x in members)
             if isinstance(cls, ClassDefinition):
                 cls_uri = self._class_uri(cls.name)
                 listnode = BNode()
                 Collection(graph, listnode, sub_exprs)
                 graph.add((cls_uri, OWL.disjointUnionOf, listnode))
             else:
                 sub_sub_exprs: list[OWL_EXPRESSION] = []
-                for i, x in enumerate(cls.exactly_one_of):
+                for i, x in enumerate(members):
                     operand_expr = self.transform_class_expression(x)
                     if not operand_expr:
                         continue
-                    rest = cls.exactly_one_of[0:i] + cls.exactly_one_of[i + 1 :]
+                    rest = members[0:i] + members[i + 1 :]
                     neg_expr = self._complement_of_union_of([self.transform_class_expression(nx) for nx in rest])
                     pos_expr = self._intersection_of([operand_expr, neg_expr])
                     if pos_expr:
@@ -596,11 +623,17 @@ def transform_class_expression(
                     owl_exprs.append(union_expr)
                 # owl_exprs.extend(sub_exprs)
         if cls.all_of:
-            all_of_expr = self._intersection_of([self.transform_class_expression(x) for x in cls.all_of])
+            members = list(cls.all_of)
+            if self.deterministic:
+                members = sorted(members, key=_expression_sort_key)
+            all_of_expr = self._intersection_of([self.transform_class_expression(x) for x in members])
             if all_of_expr:
                 owl_exprs.append(all_of_expr)
         if cls.none_of:
-            none_of_expr = self._complement_of_union_of([self.transform_class_expression(x) for x in cls.none_of])
+            members = list(cls.none_of)
+            if self.deterministic:
+                members = sorted(members, key=_expression_sort_key)
+            none_of_expr = self._complement_of_union_of([self.transform_class_expression(x) for x in members])
             if none_of_expr:
                 owl_exprs.append(none_of_expr)
         for slot in own_slots:
@@ -773,19 +806,29 @@ def _get_slot_nodes(
             )
             return rdflib_nodes or None
 
-        if any_of_rdflib_nodes := _get_slot_nodes(slot.any_of):
+        def _maybe_sort_slots(
+            slot_definitions: Sequence[SlotDefinition | AnonymousSlotExpression] | None,
+        ) -> Sequence[SlotDefinition | AnonymousSlotExpression] | None:
+            if slot_definitions and self.deterministic:
+                return sorted(slot_definitions, key=_expression_sort_key)
+            return slot_definitions
+
+        if any_of_rdflib_nodes := _get_slot_nodes(_maybe_sort_slots(slot.any_of)):
             owl_exprs.append(self._union_of(any_of_rdflib_nodes))
-        if all_of_rdflib_nodes := _get_slot_nodes(slot.all_of):
+        if all_of_rdflib_nodes := _get_slot_nodes(_maybe_sort_slots(slot.all_of)):
             owl_exprs.append(self._intersection_of(all_of_rdflib_nodes))
-        if none_of_rdflib_nodes := _get_slot_nodes(slot.none_of):
+        if none_of_rdflib_nodes := _get_slot_nodes(_maybe_sort_slots(slot.none_of)):
             owl_exprs.append(self._complement_of_union_of(none_of_rdflib_nodes))
         if slot.exactly_one_of:
+            members = list(slot.exactly_one_of)
+            if self.deterministic:
+                members = sorted(members, key=_expression_sort_key)
             disj_exprs: list[OWL_EXPRESSION] = []
-            for i, operand in enumerate(slot.exactly_one_of):
+            for i, operand in enumerate(members):
                 operand_expr = self.transform_class_slot_expression(cls, operand, main_slot, owl_types)
                 if not operand_expr:
                     continue
-                rest = slot.exactly_one_of[0:i] + slot.exactly_one_of[i + 1 :]
+                rest = members[0:i] + members[i + 1 :]
                 neg_expr = self._complement_of_union_of(
                     [self.transform_class_slot_expression(cls, x, main_slot, owl_types) for x in rest],
                     owl_types=owl_types,
@@ -1059,7 +1102,10 @@ def add_enum(self, e: EnumDefinition) -> None:
         owl_types: list[URIRef | None] = []
         enum_owl_type = self._get_metatype(e, self.default_permissible_value_type)
 
-        for pv in e.permissible_values.values():
+        pvs = e.permissible_values.values()
+        if self.deterministic:
+            pvs = sorted(pvs, key=lambda x: x.text)
+        for pv in pvs:
             pv_owl_type = self._get_metatype(pv, enum_owl_type)
             owl_types.append(pv_owl_type)
             if pv_owl_type == RDFS.Literal:

packages/linkml/src/linkml/generators/shaclgen.py

@@ -95,7 +95,13 @@ def generate_header(self) -> str:
     def serialize(self, **args) -> str:
         g = self.as_graph()
         fmt = "turtle" if self.format in ["owl", "ttl"] else self.format
-        return canonicalize_rdf_graph(g, output_format=fmt)
+        if self.deterministic and fmt == "turtle":
+            from linkml.utils.generator import deterministic_turtle
+
+            data = deterministic_turtle(g)
+        else:
+            data = canonicalize_rdf_graph(g, output_format=fmt)
+        return data
 
     def as_graph(self) -> Graph:
         sv = self.schemaview
@@ -313,13 +319,13 @@ def _add_enum(self, g: Graph, func: Callable, r: ElementName) -> None:
         sv = self.schemaview
         enum = sv.get_enum(r)
         pv_node = BNode()
+        pv_items = list(enum.permissible_values.items())
+        if self.deterministic:
+            pv_items = sorted(pv_items, key=lambda x: x[0])
         Collection(
             g,
             pv_node,
-            [
-                URIRef(sv.expand_curie(pv.meaning)) if pv.meaning else Literal(pv_name)
-                for pv_name, pv in enum.permissible_values.items()
-            ],
+            [URIRef(sv.expand_curie(pv.meaning)) if pv.meaning else Literal(pv_name) for pv_name, pv in pv_items],
         )
         func(SH["in"], pv_node)
 
@@ -473,7 +479,10 @@ def collect_child_properties(class_name: str, output: set) -> None:
 
         list_node = BNode()
         ignored_properties.add(RDF.type)
-        Collection(g, list_node, list(ignored_properties))
+        props = list(ignored_properties)
+        if self.deterministic:
+            props = sorted(props, key=str)
+        Collection(g, list_node, props)
 
         return list_node