From 1818fdabf5abaa7d1cd93010a9263b61e75a8e57 Mon Sep 17 00:00:00 2001 From: Xavi Aracil Date: Wed, 14 Dec 2022 16:12:49 +0100 Subject: [PATCH] initial extension probe, only checks contexts --- .../oneedtech/inspect/vc/OB20Inspector.java | 20 ++- .../org/oneedtech/inspect/vc/VCInspector.java | 51 ++++++ .../vc/jsonld/probe/ExtensionProbe.java | 152 ++++++++++++++++-- .../jsonld/probe/JsonLDCompactionProve.java | 6 +- .../vc/util/CachingDocumentLoader.java | 3 + .../resources/contexts/obv2x-extensions.json | 12 ++ .../assertion-with-extension-node-basic.json | 19 ++- ...assertion-with-extension-node-invalid.json | 19 ++- 8 files changed, 245 insertions(+), 37 deletions(-) create mode 100644 inspector-vc/src/main/resources/contexts/obv2x-extensions.json diff --git a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/OB20Inspector.java b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/OB20Inspector.java index 327b1a9..6c4a5cb 100644 --- a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/OB20Inspector.java +++ b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/OB20Inspector.java @@ -1,6 +1,7 @@ package org.oneedtech.inspect.vc; import static java.lang.Boolean.TRUE; +import static java.util.stream.Collectors.toList; import static org.oneedtech.inspect.core.Inspector.Behavior.RESET_CACHES_ON_RUN; import static org.oneedtech.inspect.core.report.ReportUtil.onProbeException; import static org.oneedtech.inspect.util.json.ObjectMapperCache.Config.DEFAULT; @@ -13,9 +14,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.oneedtech.inspect.core.Inspector; import org.oneedtech.inspect.core.probe.GeneratedObject; @@ -26,6 +24,7 @@ import org.oneedtech.inspect.core.probe.json.JsonPathEvaluator; import org.oneedtech.inspect.core.report.Report; import org.oneedtech.inspect.core.report.ReportItems; import org.oneedtech.inspect.schema.JsonSchemaCache; +import org.oneedtech.inspect.util.code.Tuple; import org.oneedtech.inspect.util.json.ObjectMapperCache; import org.oneedtech.inspect.util.resource.Resource; import org.oneedtech.inspect.util.resource.ResourceType; @@ -166,7 +165,7 @@ public class OB20Inspector extends VCInspector { if(broken(accumulator)) return abort(ctx, accumulator, probeCount); } - // extension validations + // get all json-ld generated objects for both extension and endorsements validation List jsonLdGeneratedObjects = ctx.getGeneratedObjects().values().stream() .filter(generatedObject -> generatedObject instanceof JsonLdGeneratedObject) .map(obj -> { @@ -177,10 +176,15 @@ public class OB20Inspector extends VCInspector { throw new IllegalArgumentException("Couldn't not parse " + obj.getId() + ": contains invalid JSON"); } }) - .collect(Collectors.toList()); - for (JsonNode generatedObject : jsonLdGeneratedObjects) { + .collect(toList()); + + // validate extensions + List> extensionProbeTuples = jsonLdGeneratedObjects.stream() + .flatMap(node -> getExtensionProbes(node, "id").stream()) + .collect(toList()); + for (Tuple extensionProbeTuple : extensionProbeTuples) { probeCount++; - accumulator.add(new ExtensionProbe().run(generatedObject, ctx)); + accumulator.add(extensionProbeTuple.t1.run(extensionProbeTuple.t2, ctx)); if(broken(accumulator)) return abort(ctx, accumulator, probeCount); } @@ -192,7 +196,7 @@ public class OB20Inspector extends VCInspector { // return endorsement node, filtering out the on inside @context return asNodeList(node, "$..endorsement", jsonPath).stream().filter(endorsementNode -> !endorsementNode.isObject()); }) - .collect(Collectors.toList()); + .collect(toList()); for(JsonNode node : endorsements) { probeCount++; diff --git a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/VCInspector.java b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/VCInspector.java index b4cf13a..3e29fd7 100644 --- a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/VCInspector.java +++ b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/VCInspector.java @@ -1,8 +1,16 @@ package org.oneedtech.inspect.vc; +import static java.util.stream.Collectors.toList; +import static org.oneedtech.inspect.vc.util.JsonNodeUtil.asStringList; + +import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.oneedtech.inspect.core.Inspector; import org.oneedtech.inspect.core.probe.Outcome; @@ -10,7 +18,10 @@ import org.oneedtech.inspect.core.probe.Probe; import org.oneedtech.inspect.core.probe.RunContext; import org.oneedtech.inspect.core.report.Report; import org.oneedtech.inspect.core.report.ReportItems; +import org.oneedtech.inspect.util.code.Tuple; +import org.oneedtech.inspect.vc.jsonld.probe.ExtensionProbe; import org.oneedtech.inspect.vc.util.CachingDocumentLoader; +import org.oneedtech.inspect.vc.util.JsonNodeUtil; import com.apicatalog.jsonld.loader.DocumentLoader; import com.fasterxml.jackson.databind.JsonNode; @@ -71,6 +82,46 @@ public abstract class VCInspector extends Inspector { return new CachingDocumentLoader(); } + protected List> getExtensionProbes(JsonNode node, String entryPath) { + List> probes = new ArrayList<>(); + if (!node.isObject()) { + return probes; + } + + if (node.has("type")) { + List types = asStringList(node.get("type")); + + // only validate extension types + if (types.contains("Extension")) { + List typesToTest = types.stream().filter(type -> !type.equals("Extension")).collect(toList()); + // add an extension Probe + probes.add(new Tuple(new ExtensionProbe(entryPath, typesToTest), node)); + } + } + + + probes.addAll(StreamSupport + .stream(Spliterators.spliteratorUnknownSize(node.fields(), 0), false) + .filter(e -> !e.getKey().equals("id") && !e.getKey().equals("type")) + .flatMap(entry -> { + if (entry.getValue().isArray()) { + // recursive call + List childNodes = JsonNodeUtil.asNodeList(entry.getValue()); + List> subProbes = new ArrayList<>(); + for (int i = 0; i < childNodes.size(); i++) { + JsonNode childNode = childNodes.get(i); + subProbes.addAll(getExtensionProbes(childNode, entryPath + "." + entry.getKey() + "[" + i + "]")); + } + return subProbes.stream(); + } else { + return getExtensionProbes(entry.getValue(), entryPath + "." + entry.getKey()).stream(); + } + }) + .collect(Collectors.toList()) + ); + return probes; + } + protected static final String REFRESHED = "is.refreshed.credential"; public abstract static class Builder> extends Inspector.Builder { diff --git a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/ExtensionProbe.java b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/ExtensionProbe.java index a801222..c1be1ef 100644 --- a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/ExtensionProbe.java +++ b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/ExtensionProbe.java @@ -1,37 +1,169 @@ package org.oneedtech.inspect.vc.jsonld.probe; +import static java.util.stream.Collectors.joining; + +import java.io.IOException; +import java.io.StringReader; import java.net.URI; +import java.util.HashSet; +import java.util.List; import java.util.Set; import org.oneedtech.inspect.core.probe.Probe; import org.oneedtech.inspect.core.probe.RunContext; import org.oneedtech.inspect.core.probe.RunContext.Key; +import org.oneedtech.inspect.core.probe.json.JsonSchemaProbe; import org.oneedtech.inspect.core.report.ReportItems; import org.oneedtech.inspect.vc.util.CachingDocumentLoader; +import org.oneedtech.inspect.vc.util.JsonNodeUtil; +import com.apicatalog.jsonld.JsonLd; +import com.apicatalog.jsonld.JsonLdError; +import com.apicatalog.jsonld.JsonLdOptions; +import com.apicatalog.jsonld.document.Document; +import com.apicatalog.jsonld.document.JsonDocument; +import com.apicatalog.jsonld.loader.DocumentLoader; +import com.apicatalog.jsonld.loader.DocumentLoaderOptions; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.util.TokenBuffer; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion.VersionFlag; +import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + +/** + * Probe for extensions in OB 2.0 + * Maps to task "VALIDATE_EXTENSION_NODE" in python implementation + * @author xaracil + */ public class ExtensionProbe extends Probe { + private final List typesToTest; + + public ExtensionProbe(String entryPath, List typesToTest) { + super(ID, entryPath, typesToTest.stream().collect(joining())); + this.typesToTest = typesToTest; + } @Override public ReportItems run(JsonNode node, RunContext ctx) throws Exception { - if (!node.isObject()) { - return success(ctx); - } - - Object documentLoader = ctx.get(Key.JSON_DOCUMENT_LOADER); - Set contexts; + ReportItems reportItems = new ReportItems(); + DocumentLoader documentLoader = (DocumentLoader) ctx.get(Key.JSON_DOCUMENT_LOADER); + Set contexts = null; if (documentLoader instanceof CachingDocumentLoader) { - contexts = ((CachingDocumentLoader) documentLoader).getContexts(); + contexts = new HashSet<>(((CachingDocumentLoader) documentLoader).getContexts()); } else { contexts = Set.of(); } - // TODO Auto-generated method stub - return null; + // compact contexts + URI ob20contextUri = new URI(CONTEXT_URI_STRING); + ObjectMapper mapper = (ObjectMapper) ctx.get(Key.JACKSON_OBJECTMAPPER); + for (URI uri : contexts) { + if (!uri.equals(ob20contextUri)) { + JsonLdOptions options = new JsonLdOptions(documentLoader); + Document contextDocument = documentLoader.loadDocument(uri, new DocumentLoaderOptions()); + JsonNode contextJson = mapper.readTree(contextDocument.getJsonContent().orElseThrow().toString()); + + JsonObject compactedContext = JsonLd.compact(uri, "https://w3id.org/openbadges/v2") + .options(options) + .get(); + JsonNode context = mapper.readTree(compactedContext.toString()); + List validations = JsonNodeUtil.asNodeList(context.get("validation")); + for (JsonNode validation : validations) { + if (isLdTermInList(validation.get("validatesType"), options)) { + JsonNode schemaJson = null; + URI schemaUri = null; + try { + schemaUri = new URI(validation.get("validationSchema").asText().strip()); + // check schema is valid + Document schemaDocument = documentLoader.loadDocument(schemaUri, new DocumentLoaderOptions()); + schemaJson = mapper.readTree(schemaDocument.getJsonContent().orElseThrow().toString()); + } catch (Exception e) { + return fatal("Could not load JSON-schema from URL " + schemaUri, ctx); + } + + reportItems = new ReportItems(List.of(reportItems, validateSingleExtension(node, uri, contextJson, validation.get("validatesType").asText().strip(), schemaJson, schemaUri, options, ctx))); + } + } + } + } + + if (reportItems.size() == 0) { + return error("Could not determine extension type to test", ctx); + } + + return reportItems; + } + + private boolean isLdTermInList(JsonNode termNode, JsonLdOptions options) throws JsonLdError { + JsonDocument jsonDocument = JsonDocument.of(Json.createObjectBuilder() + .add("@context", CONTEXT_URI_STRING) + .add("_:term", Json.createObjectBuilder() + .add("@type", termNode.asText().strip())) + .add("_:list", Json.createObjectBuilder() + .add("@type", Json.createArrayBuilder(typesToTest))) + .build()); + JsonArray expandedDocument = JsonLd.expand(jsonDocument) + .options(options) + .get(); + + JsonArray list = expandedDocument.getJsonObject(0).getJsonArray("_:list").getJsonObject(0).getJsonArray("@type"); + JsonValue term = expandedDocument.getJsonObject(0).getJsonArray("_:term").getJsonObject(0).getJsonArray("@type").get(0); + + return list.contains(term); } - private void getValidations(JsonNode node, String entryPath, Set contexts) { + private ReportItems validateSingleExtension(JsonNode node, URI uri, JsonNode context, String string, JsonNode schemaJson, URI schemaUri, JsonLdOptions options, RunContext ctx) throws JsonGenerationException, JsonMappingException, IOException, JsonLdError { + ObjectMapper mapper = (ObjectMapper) ctx.get(Key.JACKSON_OBJECTMAPPER); + // validate against JSON schema, using a copy of the node + TokenBuffer tb = new TokenBuffer(mapper, false); + mapper.writeValue(tb, node); + JsonNode auxNode = mapper.readTree(tb.asParser()); + ObjectReader readerForUpdating = mapper.readerForUpdating(auxNode); + JsonNode merged = readerForUpdating.readValue("{\"@context\": \"" + CONTEXT_URI_STRING + "\"}"); + + // combine contexts + JsonDocument contextsDocument = combineContexts(context); + + JsonObject compactedObject = JsonLd.compact(JsonDocument.of(new StringReader(merged.toString())), contextsDocument) + .options(options) + .get(); + + // schema probe on compactedObject and schema + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(VersionFlag.V4); + JsonSchema schema = factory.getSchema(schemaUri, schemaJson); + return new JsonSchemaProbe(schema).run(mapper.readTree(compactedObject.toString()), ctx); } + + private JsonDocument combineContexts(JsonNode context) { + List contexts = JsonNodeUtil.asNodeList(context); + JsonArrayBuilder contextArrayBuilder = Json.createArrayBuilder(); + contextArrayBuilder.add(CONTEXT_URI_STRING); // add OB context to the list + for (JsonNode contextNode : contexts) { + if (contextNode.isTextual()) { + contextArrayBuilder.add(contextNode.asText().strip()); + } else if (contextNode.isObject() && contextNode.hasNonNull("@context")) { + contextArrayBuilder.add(Json.createReader(new StringReader(contextNode.get("@context").toString())).readObject()); + + } + } + + JsonDocument contextsDocument = JsonDocument.of(Json.createObjectBuilder() + .add("@context", contextArrayBuilder.build()) + .build()); + return contextsDocument; + } + + public static final String ID = ExtensionProbe.class.getSimpleName(); + private static final String CONTEXT_URI_STRING = "https://w3id.org/openbadges/v2"; } diff --git a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/JsonLDCompactionProve.java b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/JsonLDCompactionProve.java index 92c4a51..a934d46 100644 --- a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/JsonLDCompactionProve.java +++ b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/jsonld/probe/JsonLDCompactionProve.java @@ -36,10 +36,10 @@ public class JsonLDCompactionProve extends Probe { try { // compact JSON JsonDocument jsonDocument = JsonDocument.of(new StringReader(crd.getJson().toString())); - CompactionApi compactApi = JsonLd.compact(jsonDocument, context); - compactApi.options(new JsonLdOptions((DocumentLoader) ctx.get(Key.JSON_DOCUMENT_LOADER))); + JsonObject compactedObject = JsonLd.compact(jsonDocument, context) + .options(new JsonLdOptions((DocumentLoader) ctx.get(Key.JSON_DOCUMENT_LOADER))) + .get(); - JsonObject compactedObject = compactApi.get(); ctx.addGeneratedObject(new JsonLdGeneratedObject(getId(crd), compactedObject.toString())); // Handle mismatch between URL node source and declared ID. diff --git a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/util/CachingDocumentLoader.java b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/util/CachingDocumentLoader.java index 1136aea..18251b9 100644 --- a/inspector-vc/src/main/java/org/oneedtech/inspect/vc/util/CachingDocumentLoader.java +++ b/inspector-vc/src/main/java/org/oneedtech/inspect/vc/util/CachingDocumentLoader.java @@ -126,6 +126,8 @@ public class CachingDocumentLoader extends ConfigurableDocumentLoader { .put("https://w3id.org/security/suites/jws-2020/v1", Resources.getResource("contexts/suites-jws-2020.jsonld")) .put("https://openbadgespec.org/v2/context.json", Resources.getResource("contexts/ob-v2p0.json")) .put("https://w3id.org/openbadges/v2", Resources.getResource("contexts/obv2x.jsonld")) + .put("https://w3id.org/openbadges/extensions/exampleExtension/context.json", Resources.getResource("contexts/obv2x-extensions.json")) + .put("https://openbadgespec.org/extensions/exampleExtension/schema.json", Resources.getResource("catalog/openbadgespec.org/extensions/exampleExtension/schema.json")) .build(); @@ -133,6 +135,7 @@ public class CachingDocumentLoader extends ConfigurableDocumentLoader { .initialCapacity(32).maximumSize(64).expireAfterAccess(Duration.ofHours(24)) .build(new CacheLoader, Document>() { public Document load(final Tuple id) throws Exception { + System.out.println("CachingDocumentLoader " + id.t1 + ": " + bundled.containsKey(id.t1)); try (InputStream is = bundled.containsKey(id.t1) ? bundled.get(id.t1).openStream() : new URI(id.t1).toURL().openStream();) { diff --git a/inspector-vc/src/main/resources/contexts/obv2x-extensions.json b/inspector-vc/src/main/resources/contexts/obv2x-extensions.json new file mode 100644 index 0000000..d759111 --- /dev/null +++ b/inspector-vc/src/main/resources/contexts/obv2x-extensions.json @@ -0,0 +1,12 @@ +{ + "@context": { + "obi": "https://w3id.org/openbadges#", + "exampleProperty": "http://schema.org/text" + }, + "obi:validation": [ + { + "obi:validatesType": "obi:extensions/#ExampleExtension", + "obi:validationSchema": "https://openbadgespec.org/extensions/exampleExtension/schema.json" + } + ] + } \ No newline at end of file diff --git a/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-basic.json b/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-basic.json index 0ef8631..9861d08 100644 --- a/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-basic.json +++ b/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-basic.json @@ -1,5 +1,8 @@ { - "@context": ["https://w3id.org/openbadges/v2","https://w3id.org/openbadges/extensions/exampleExtension/context.json"], + "@context": [ + "https://w3id.org/openbadges/v2", + "https://w3id.org/openbadges/extensions/exampleExtension/context.json" + ], "id": "http://example.org/assertion", "type": "Assertion", "recipient": { @@ -9,21 +12,21 @@ "identity": "sha256$ecf5409f3f4b91ab60cc5ef4c02aef7032354375e70cf4d8e43f6a1d29891942" }, "image": "https://example.org/beths-robot-badge.png", - "issuedOn": "2016-12-31T23:59:59Z", "badge": "https://example.org/robotics-badge.json", + "issuedOn": "2016-12-31T23:59:59Z", "verification": { "type": "hosted" }, - "evidence": { - "id": "_:b1", - "narrative": "Rocked the free world" - }, "extensions:exampleExtension": { "id": "_:b0", "type": [ "Extension", - "extensions:ExampleExtension" + "obi:extensions/#ExampleExtension" ], - "schema:text": "I'm a property, short and sweet" + "http://schema.org/text": "I'm a property, short and sweet" + }, + "evidence": { + "id": "_:b1", + "narrative": "Rocked the free world" } } \ No newline at end of file diff --git a/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-invalid.json b/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-invalid.json index af49df5..76194b3 100644 --- a/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-invalid.json +++ b/inspector-vc/src/test/resources/ob20/assertion-with-extension-node-invalid.json @@ -1,5 +1,8 @@ { - "@context": ["https://w3id.org/openbadges/v2","https://w3id.org/openbadges/extensions/exampleExtension/context.json"], + "@context": [ + "https://w3id.org/openbadges/v2", + "https://w3id.org/openbadges/extensions/exampleExtension/context.json" + ], "id": "http://example.org/assertion", "type": "Assertion", "recipient": { @@ -9,21 +12,21 @@ "identity": "sha256$ecf5409f3f4b91ab60cc5ef4c02aef7032354375e70cf4d8e43f6a1d29891942" }, "image": "https://example.org/beths-robot-badge.png", - "issuedOn": "2016-12-31T23:59:59Z", "badge": "https://example.org/robotics-badge.json", + "issuedOn": "2016-12-31T23:59:59Z", "verification": { "type": "hosted" }, - "evidence": { - "id": "_:b1", - "narrative": "Rocked the free world" - }, "extensions:exampleExtension": { "id": "_:b0", "type": [ "Extension", - "extensions:ExampleExtension" + "obi:extensions/#ExampleExtension" ], - "schema:text": 1337 + "http://schema.org/text": 1337 + }, + "evidence": { + "id": "_:b1", + "narrative": "Rocked the free world" } } \ No newline at end of file