+ ShExMap: schema-to-schema mapping via Map/# semantic actions

Implement http://shex.io/extensions/Map/# (lib/ShExMap.{hpp,cpp}),
ported from shex.js's @shexjs/extension-map: validating data against a
source schema whose triple constraints carry %Map:{ pfx:var %} actions
binds each matched value to the (prefix-expanded) variable, and
materialization walks a target schema emitting those bindings as a new
graph. Lift/lower functions are supported on both sides:
  %Map:{ regex(/(?<bp:family>[a-zA-Z]+), (?<bp:given>[a-zA-Z]+)/) %}
  %Map:{ hashmap(bp:code, {"D": "Divorced", ...}) %}
with named-group regexes rewritten for Boost.Regex and hashmaps read
via MiniJSON.

The validator gains a SemActHandler interface (in addition to the
built-in Test language): handlers see each action with its focus node
and the triples the accepted matching assigned to its constraint, and
mark()/rollback() scope handler state to successful validation
subtrees -- without that, probing a sibling shape (e.g. a systolic
component against the diastolic shape) leaks spurious bindings.

Bindings are an ordered log frozen into per-variable FIFO queues:
variables bound once (and manifest staticVars) stay available, while
multi-valued queues pop, and only pops count as "consumption". The
materializer uses that to decide repetition: a repeated triple
constraint (@<Shape>* etc.) keeps emitting fresh bnodes only while
each repetition consumes bindings, which reproduces shex.js's
multi-bindings behavior without its solution-tree binder.

Manifest entries (JSON or YAML) drive mapping with outputSchema[URL],
outputShape, createRoot and staticVars, plus optional
expectedBindingsURL and outputDataURL checks (materialized graphs
compared bnode-isomorphically); bin/sparql --shex-manifest prints the
materialized graph. tests/ShExMap/ carries the BP FHIR<->DAM round
trip and the multi-bindings case from shex.js's examples; the ShEx
suite is now 1736 cases, all green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Eric Prud'hommeaux

CMakeLists.txt

@@ -165,6 +165,7 @@ add_library(SWObjects STATIC
   lib/ShExSchema.cpp
   lib/ShExShapeMap.cpp
   lib/ShExManifest.cpp
+  lib/ShExMap.cpp
   lib/SWObjects.cpp
   ${SWOBJ_GENERATED_SOURCES}
 )

README

@@ -60,6 +60,18 @@ see lib/ShExManifest.hpp and tests/ShExManifest/):
 --shex-run selects 0-based entries: ranges like "1", "2-5", "5-2"
 (descending) or "*" (everything), comma-separated.
 
+lib/ShExMap.{hpp,cpp} implements the http://shex.io/extensions/Map/#
+semantic-action language for mapping between two schemas: validating
+against a source schema whose constraints carry %Map:{ var %} (or
+regex(...)/hashmap(...) lift functions) collects variable bindings,
+and materialization walks a target schema emitting the bound (or
+lowered) values. Manifest entries drive it with outputSchema[URL],
+outputShape, createRoot, staticVars and optional expectedBindingsURL /
+outputDataURL comparisons (see tests/ShExMap/, ported from shex.js's
+packages/extension-map examples):
+
+  bin/sparql --shex-manifest tests/ShExMap/manifest.yaml
+
 # More info
 
 To learn more about SWObjects, checkout some tutorials given at SWAT4LS in 2010 and 2011:

bin/sparql.cpp

@@ -1075,7 +1075,7 @@ int main(int ac, char* av[])
 		std::cout << (outcome.allAsAsserted ? "conformant" : "nonconformant")
 			  << ", expected "
 			  << (entry.expectConformant ? "conformant" : "nonconformant")
-			  << (outcome.statusMatched ? " -- PASS" : " -- FAIL") << "\n";
+			  << (outcome.passed() ? " -- PASS" : " -- FAIL") << "\n";
 		for (std::vector<sw::ShEx::AssociationResult>::const_iterator r
 			 = outcome.results.begin(); r != outcome.results.end(); ++r)
 		    std::cout << "  " << r->assoc.node->toString()
@@ -1084,7 +1084,13 @@ int main(int ac, char* av[])
 				 : std::string("START"))
 			      << ": " << (r->conformant ? "conformant" : "nonconformant")
 			      << "\n";
-		if (!outcome.statusMatched)
+		for (std::vector<std::string>::const_iterator p = outcome.problems.begin();
+		     p != outcome.problems.end(); ++p)
+		    std::cout << "  ! " << *p << "\n";
+		if (entry.isMapTest() && !outcome.outputGraph.empty()
+		    && outcome.problems.empty())
+		    std::cout << outcome.outputGraph;
+		if (!outcome.passed())
 		    ++failures;
 	    }
 	    return failures > 125 ? 125 : failures;

lib/SWObjects.hpp

@@ -4487,6 +4487,7 @@ class NamespaceMap {
 	    return vi->second;
     }
     virtual void set (std::string prefix, const URI* uri) { ns[prefix] = uri; }
+    const std::map<std::string, const URI*>& raw () const { return ns; }
     virtual std::string unmap (std::string mapped);
 
     typedef std::map<std::string, const URI*>::iterator iterator;

lib/ShExCParser.ypp

@@ -1425,6 +1425,12 @@ void ShExDriver::parse (IStreamContext& in, ShEx::Schema* intoSchema) {
     parser.set_debug_level(trace_parsing);
     parser.parse();
     checkErrors();
+
+    // keep the prefixes for semantic-action languages (e.g. ShExMap)
+    const std::map<std::string, const URI*>& ns = getNamespaceMap()->raw();
+    for (std::map<std::string, const URI*>::const_iterator it = ns.begin();
+	 it != ns.end(); ++it)
+	intoSchema->prefixes[it->first] = it->second->getLexicalValue();
 }
 
 std::string ShExDriver::facetKey (const ShEx::NCComponent* c) {

lib/ShExManifest.cpp

@@ -2,6 +2,7 @@
 
 #include "ShExManifest.hpp"
 #include "MiniJSON.hpp"
+#include "ShExMap.hpp"
 #include "ShExCParser.hpp"
 #include "TurtleParser.hpp"
 #include "RdfDB.hpp"
@@ -137,6 +138,32 @@ namespace ShEx {
 		    std::string key = strip(cur.substr(indent, colon - indent));
 		    std::string value = strip(cur.substr(colon + 1));
 		    ++i;
+		    if (value.empty()) {
+			// a one-level nested mapping (e.g. staticVars:)
+			while (i < yl.lines.size()) {
+			    std::string nl = yl.lines[i];
+			    if (strip(nl).empty()) {
+				++i;
+				continue;
+			    }
+			    size_t nIndent = YamlLines::indentOf(nl);
+			    if (nIndent <= indent || nl.compare(0, 2, "- ") == 0)
+				break;
+			    size_t nColon = nl.find(':', nIndent);
+			    if (nColon == std::string::npos)
+				throw ManifestError("expected nested \"key: value\": " + nl);
+			    std::string nKey = strip(nl.substr(nIndent, nColon - nIndent));
+			    std::string nValue = strip(nl.substr(nColon + 1));
+			    if (!nValue.empty() && nValue[0] == '\'')
+				nValue = nValue.size() >= 2 && nValue[nValue.size()-1] == '\''
+				    ? nValue.substr(1, nValue.size()-2) : nValue;
+			    else if (!nValue.empty() && nValue[0] == '"')
+				nValue = unescapeDoubleQuoted(nValue.substr(1, nValue.size()-2), i);
+			    item[key + "\t" + nKey] = nValue;
+			    ++i;
+			}
+			continue;
+		    }
 		    if (value == "|" || value == "|-") {
 			bool clip = value == "|-";
 			// literal block scalar: lines indented past the key
@@ -201,6 +228,15 @@ namespace ShEx {
 	    if ((it = kv.find("dataURL")) != kv.end()) e.dataURL = it->second;
 	    if ((it = kv.find("queryMap")) != kv.end()) e.queryMap = it->second;
 	    if ((it = kv.find("queryMapURL")) != kv.end()) e.queryMapURL = it->second;
+	    if ((it = kv.find("outputSchema")) != kv.end()) e.outputSchema = it->second;
+	    if ((it = kv.find("outputSchemaURL")) != kv.end()) e.outputSchemaURL = it->second;
+	    if ((it = kv.find("outputShape")) != kv.end()) e.outputShape = it->second;
+	    if ((it = kv.find("createRoot")) != kv.end()) e.createRoot = it->second;
+	    if ((it = kv.find("expectedBindingsURL")) != kv.end()) e.expectedBindingsURL = it->second;
+	    if ((it = kv.find("outputDataURL")) != kv.end()) e.outputDataURL = it->second;
+	    for (it = kv.begin(); it != kv.end(); ++it)
+		if (it->first.compare(0, 11, "staticVars\t") == 0)
+		    e.staticVars[it->first.substr(11)] = it->second;
 	    std::string status;
 	    if ((it = kv.find("status")) != kv.end()) status = it->second;
 	    std::stringstream where;
@@ -222,6 +258,114 @@ namespace ShEx {
 	}
     } // namespace
 
+    namespace {
+
+	/** Parse a term from a manifest string: "<iri>", a quoted literal, or
+	 * a bare string (taken as a plain literal). */
+	const TTerm* parseTermString (const std::string& s, AtomFactory& atomFactory) {
+	    std::string t = s;
+	    if (!t.empty() && t[0] == '<' && t[t.size()-1] == '>')
+		return atomFactory.getURI(t.substr(1, t.size() - 2));
+	    if (t.size() >= 2 && t[0] == '"' && t[t.size()-1] == '"')
+		return atomFactory.getRDFLiteral(t.substr(1, t.size() - 2), NULL, NULL);
+	    return atomFactory.getRDFLiteral(t, NULL, NULL);
+	}
+
+	/** JSON binding value {value, type?, language?} -> a term. */
+	const TTerm* bindingValueTerm (const MiniJSON::Value& v, AtomFactory& atomFactory) {
+	    std::string value = v.getString("value");
+	    std::string type = v.getString("type");
+	    std::string language = v.getString("language");
+	    if (!language.empty())
+		return atomFactory.getRDFLiteral(value, NULL, new LANGTAG(language));
+	    if (!type.empty())
+		return atomFactory.getRDFLiteral(value, atomFactory.getURI(type), NULL);
+	    return atomFactory.getRDFLiteral(value, NULL, NULL);
+	}
+
+	void compareBindings (const ManifestEntry& entry, const Manifest& manifest,
+			      const ShExMap::Bindings& bindings, AtomFactory& atomFactory,
+			      EntryOutcome& outcome) {
+	    MiniJSON::Value expected
+		= MiniJSON::parse(readFileOrThrow(manifest.directory + entry.expectedBindingsURL));
+	    if (!expected.isObject())
+		throw ManifestError(entry.expectedBindingsURL + " should be a JSON object");
+	    for (std::vector<std::pair<std::string, MiniJSON::Value> >::const_iterator it
+		     = expected.object.begin(); it != expected.object.end(); ++it) {
+		const TTerm* want = bindingValueTerm(it->second, atomFactory);
+		std::map<std::string, std::deque<const TTerm*> >::const_iterator q
+		    = bindings.queues.find(it->first);
+		bool found = false;
+		if (q != bindings.queues.end())
+		    for (std::deque<const TTerm*>::const_iterator b = q->second.begin();
+			 !found && b != q->second.end(); ++b)
+			found = *b == want;
+		if (!found)
+		    outcome.problems.push_back("expected binding " + it->first + " = "
+					       + want->toString());
+	    }
+	}
+
+	void materializeEntry (const ManifestEntry& entry, const Manifest& manifest,
+			       ShExMap::Bindings& bindings, AtomFactory& atomFactory,
+			       EntryOutcome& outcome) {
+	    // target schema
+	    std::string schemaText = entry.outputSchema;
+	    std::string schemaBase = manifest.baseURI;
+	    if (!entry.outputSchemaURL.empty()) {
+		schemaText = readFileOrThrow(manifest.directory + entry.outputSchemaURL);
+		schemaBase = libwww::GetAbsoluteURIstring(entry.outputSchemaURL, manifest.baseURI);
+	    }
+	    ShEx::Schema target;
+	    ShExDriver targetDriver(schemaBase, &atomFactory);
+	    {
+		IStreamContext istr(schemaText, IStreamContext::STRING);
+		istr.nameStr = schemaBase;
+		targetDriver.parse(istr, &target);
+	    }
+	    target.checkStructure();
+
+	    const TTerm* root = NULL;
+	    if (!entry.createRoot.empty()) {
+		std::string t = entry.createRoot;
+		if (!t.empty() && t[0] == '<' && t[t.size()-1] == '>')
+		    root = atomFactory.getURI(
+			libwww::GetAbsoluteURIstring(t.substr(1, t.size()-2), schemaBase));
+		else
+		    root = parseTermString(t, atomFactory);
+	    }
+	    const TTerm* shapeLabel = NULL;
+	    if (!entry.outputShape.empty()) {
+		std::string t = entry.outputShape;
+		if (!t.empty() && t[0] == '<' && t[t.size()-1] == '>')
+		    shapeLabel = atomFactory.getURI(
+			libwww::GetAbsoluteURIstring(t.substr(1, t.size()-2), schemaBase));
+		else
+		    shapeLabel = atomFactory.getURI(t);
+	    }
+
+	    RdfDB outDB;
+	    BasicGraphPattern* outGraph = outDB.ensureGraph(DefaultGraph);
+	    ShExMap::materialize(target, bindings, atomFactory, outGraph, root, shapeLabel);
+	    outcome.outputGraph = outDB.toString(MediaType("text/turtle"));
+
+	    if (!entry.outputDataURL.empty()) {
+		std::string expectedText = readFileOrThrow(manifest.directory + entry.outputDataURL);
+		std::string expectedBase
+		    = libwww::GetAbsoluteURIstring(entry.outputDataURL, manifest.baseURI);
+		RdfDB expectedDB;
+		TurtleDriver expectedParser(expectedBase, &atomFactory);
+		IStreamContext istr(expectedText, IStreamContext::STRING);
+		istr.nameStr = expectedBase;
+		expectedParser.parse(istr, expectedDB.ensureGraph(DefaultGraph));
+		if (!(outDB == expectedDB))
+		    outcome.problems.push_back("materialized graph differs from "
+					       + entry.outputDataURL + ":\n"
+					       + outcome.outputGraph);
+	    }
+	}
+    } // namespace
+
     Manifest Manifest::parse (const std::string& text, const std::string& path) {
 	Manifest m;
 	size_t slash = path.find_last_of('/');
@@ -253,6 +397,11 @@ namespace ShEx {
 			 f = o.object.begin(); f != o.object.end(); ++f)
 		    if (f->second.isString())
 			kv[f->first] = f->second.string;
+		    else if (f->second.isObject()) // e.g. staticVars
+			for (std::vector<std::pair<std::string, MiniJSON::Value> >::const_iterator
+				 n = f->second.object.begin(); n != f->second.object.end(); ++n)
+			    if (n->second.isString())
+				kv[f->first + "\t" + n->first] = n->second.string;
 		m.entries.push_back(entryFromKeyValues(kv, i));
 	    }
 	} else {
@@ -312,14 +461,28 @@ namespace ShEx {
 	    std::vector<Association> associations
 		= parseQueryMap(mapText, &atomFactory, nodeEnv, shapeEnv, *graph);
 
-	    // ---- validate
+	    // ---- validate, collecting ShExMap bindings
+	    ShExMap::Bindings bindings;
+	    for (std::map<std::string, std::string>::const_iterator sv = entry.staticVars.begin();
+		 sv != entry.staticVars.end(); ++sv)
+		bindings.setStatic(sv->first, parseTermString(sv->second, atomFactory));
+	    ShExMap::BindingCollector collector(bindings, &atomFactory, schema.prefixes);
 	    Validator validator(schema, *graph);
+	    validator.setSemActHandler(&collector);
 	    outcome.results = evaluate(validator, associations);
 	    outcome.allAsAsserted = true;
 	    for (std::vector<AssociationResult>::const_iterator it = outcome.results.begin();
 		 it != outcome.results.end(); ++it)
 		outcome.allAsAsserted = outcome.allAsAsserted && it->asAsserted;
 	    outcome.statusMatched = outcome.allAsAsserted == entry.expectConformant;
+	    bindings.freeze();
+	    outcome.bindings = bindings.str();
+
+	    // ---- ShExMap: check bindings, materialize into the target schema
+	    if (!entry.expectedBindingsURL.empty())
+		compareBindings(entry, manifest, bindings, atomFactory, outcome);
+	    if (entry.isMapTest() && outcome.statusMatched && outcome.allAsAsserted)
+		materializeEntry(entry, manifest, bindings, atomFactory, outcome);
 	} catch (std::exception& e) {
 	    outcome.error = e.what();
 	} catch (std::string& e) {

lib/ShExManifest.hpp

@@ -44,6 +44,17 @@ namespace ShEx {
 	std::string data, dataURL;
 	std::string queryMap, queryMapURL;
 	bool expectConformant;
+
+	/* ShExMap entries additionally name a target schema to materialize
+	 * into (see lib/ShExMap.hpp): */
+	std::string outputSchema, outputSchemaURL;
+	std::string outputShape;     // "<label>" in the output schema; "" = start
+	std::string createRoot;      // subject of the materialized root; "" = bnode
+	std::map<std::string, std::string> staticVars; // var IRI -> value
+	std::string expectedBindingsURL; // JSON var -> {value, type?, language?}
+	std::string outputDataURL;   // expected materialized graph (Turtle)
+
+	bool isMapTest () const { return !outputSchema.empty() || !outputSchemaURL.empty(); }
     };
 
     struct Manifest {
@@ -62,7 +73,11 @@ namespace ShEx {
 	std::vector<AssociationResult> results;
 	bool allAsAsserted;
 	bool statusMatched; // allAsAsserted == entry.expectConformant
+	std::string bindings;    // collected ShExMap bindings (diagnostic)
+	std::string outputGraph; // materialized graph as Turtle, when mapping
+	std::vector<std::string> problems; // bindings/output mismatches
 	EntryOutcome () : allAsAsserted(true), statusMatched(false) {  }
+	bool passed () const { return error.empty() && statusMatched && problems.empty(); }
     };
 
     /** Parse the entry's schema, data and query map, validate every