diff --git a/src/main/java/org/topbraid/shacl/validation/ValidationEngine.java b/src/main/java/org/topbraid/shacl/validation/ValidationEngine.java index d5c4b992..9abc308b 100644 --- a/src/main/java/org/topbraid/shacl/validation/ValidationEngine.java +++ b/src/main/java/org/topbraid/shacl/validation/ValidationEngine.java @@ -16,6 +16,9 @@ */ package org.topbraid.shacl.validation; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.URI; import java.util.Collection; import java.util.Collections; @@ -26,12 +29,17 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; +import org.apache.commons.lang3.concurrent.ConcurrentRuntimeException; +import org.apache.jena.ext.com.google.common.cache.Cache; +import org.apache.jena.ext.com.google.common.cache.CacheBuilder; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; @@ -86,601 +94,634 @@ * @author Holger Knublauch */ public class ValidationEngine extends AbstractEngine { - - // The currently active ValidationEngine for cases where no direct pointer can be acquired, e.g. from HasShapeFunction - private static ThreadLocal current = new ThreadLocal<>(); - - public static ValidationEngine getCurrent() { - return current.get(); - } - - public static void setCurrent(ValidationEngine value) { - current.set(value); - } - - - // Avoids repeatedly walking up/down the class hierarchy for sh:class constraints - private ClassesCache classesCache; - - private ValidationEngineConfiguration configuration; - - // Can be used to drop certain focus nodes from validation - private Predicate focusNodeFilter; - - // The inferred triples if the shapes graph declares an entailment regime - private Model inferencesModel; - - // The label function for rendering nodes in validation results (message templates etc) - private Function labelFunction = (node -> RDFLabels.get().getNodeLabel(node)); - - // Avoids repeatedly fetching labels - private Map labelsCache = new ConcurrentHashMap<>(); - - // Can be used to collect statistical data about execution time of constraint components and shapes - private ValidationProfile profile; - - // The resulting validation report instance - private Resource report; - - // Number of created results, e.g. for progress monitor - private int resultsCount = 0; - - // Avoids repeatedly fetching the value nodes of a focus node / path combination - private Map> valueNodes = new WeakHashMap<>(); - - // Number of created violations, e.g. for progress monitor - private int violationsCount = 0; - - - /** - * Constructs a new ValidationEngine. - * @param dataset the Dataset to operate on - * @param shapesGraphURI the URI of the shapes graph (must be in the dataset) - * @param shapesGraph the ShapesGraph with the shapes to validate against - * @param report the sh:ValidationReport object in the results Model, or null to create a new one - */ - protected ValidationEngine(Dataset dataset, URI shapesGraphURI, ShapesGraph shapesGraph, Resource report) { - super(dataset, shapesGraph, shapesGraphURI); - setConfiguration(new ValidationEngineConfiguration()); - if(report == null) { - Model reportModel = JenaUtil.createMemoryModel(); - reportModel.setNsPrefixes(dataset.getDefaultModel()); // This can be very expensive in some databases - reportModel.withDefaultMappings(shapesGraph.getShapesModel()); - this.report = reportModel.createResource(SH.ValidationReport); - } - else { - this.report = report; - } - } - - - /** - * Checks if entailments are active for the current shapes graph and applies them for a given focus node. - * This will only work for the sh:Rules entailment, e.g. to compute sh:values and sh:defaultValue. - * If any inferred triples exist, the focus node will be returned attached to the model that includes those inferences. - * The dataset used internally will also be switched to use that new model as its default model, so that if - * a node gets validated it will "see" the inferred triples too. - * @param focusNode the focus node - * @return the focus node, possibly in a different Model than originally - */ - public RDFNode applyEntailments(Resource focusNode) { - Model shapesModel = dataset.getNamedModel(shapesGraphURI.toString()); - if(shapesModel.contains(null, SH.entailment, SH.Rules)) { - - // Create union of data model and inferences if called for the first time - if(inferencesModel == null) { - inferencesModel = JenaUtil.createDefaultModel(); - Model dataModel = dataset.getDefaultModel(); - MultiUnion multiUnion = new MultiUnion(new Graph[]{ - dataModel.getGraph(), - inferencesModel.getGraph() - }); - multiUnion.setBaseGraph(dataModel.getGraph()); - dataset.setDefaultModel(ModelFactory.createModelForGraph(multiUnion)); - } - - // Apply sh:values rules - Map defaultValueMap = new HashMap<>(); - for(SHNodeShape nodeShape : SHACLUtil.getAllShapesAtNode(focusNode)) { - if(!nodeShape.hasProperty(SH.deactivated, JenaDatatypes.TRUE)) { - for(SHPropertyShape ps : nodeShape.getPropertyShapes()) { - if(!ps.hasProperty(SH.deactivated, JenaDatatypes.TRUE)) { - Resource path = ps.getPath(); - if(path instanceof Resource) { - Statement values = ps.getProperty(SH.values); - if(values != null) { - NodeExpression ne = NodeExpressionFactory.get().create(values.getObject()); - ne.eval(focusNode, this).forEachRemaining(v -> inferencesModel.getGraph().add(Triple.create(focusNode.asNode(), path.asNode(), v.asNode()))); - } - Statement defaultValue = ps.getProperty(SH.defaultValue); - if(defaultValue != null) { - defaultValueMap.put(JenaUtil.asProperty(path), defaultValue.getObject()); - } - } - } - } - } - } - - // Add sh:defaultValue where needed - Model dataModel = dataset.getDefaultModel(); // This is now the union model - Resource newFocusNode = focusNode.inModel(dataModel); - for(Property predicate : defaultValueMap.keySet()) { - if(!newFocusNode.hasProperty(predicate)) { - NodeExpression ne = NodeExpressionFactory.get().create(defaultValueMap.get(predicate)); - ne.eval(focusNode, this).forEachRemaining(v -> inferencesModel.add(focusNode, predicate, v)); - } - } - return newFocusNode; - } - return focusNode; - } - - - public void addResultMessage(Resource result, Literal message, QuerySolution bindings) { - result.addProperty(SH.resultMessage, SPARQLSubstitutions.withSubstitutions(message, bindings, getLabelFunction())); - } - - - // Note: does not set sh:path - public Resource createResult(Resource type, Constraint constraint, RDFNode focusNode) { - Resource result = report.getModel().createResource(type); - report.addProperty(SH.result, result); - result.addProperty(SH.resultSeverity, constraint.getSeverity()); - result.addProperty(SH.sourceConstraintComponent, constraint.getComponent()); - result.addProperty(SH.sourceShape, constraint.getShapeResource()); - if(focusNode != null) { - result.addProperty(SH.focusNode, focusNode); - } - - checkMaximumNumberFailures(constraint); - - resultsCount++; - - return result; - } - - - public Resource createValidationResult(Constraint constraint, RDFNode focusNode, RDFNode value, Supplier defaultMessage) { - Resource result = createResult(SH.ValidationResult, constraint, focusNode); - if(value != null) { - result.addProperty(SH.value, value); - } - if(!constraint.getShape().isNodeShape()) { - result.addProperty(SH.resultPath, SHACLPaths.clonePath(constraint.getShapeResource().getPath(), result.getModel())); - } - Collection messages = constraint.getMessages(); - if(messages.size() > 0) { - messages.stream().forEach(message -> result.addProperty(SH.resultMessage, message)); - } - else if(defaultMessage != null) { - String m = defaultMessage.get(); - if(m != null) { - result.addProperty(SH.resultMessage, m); - } - } - return result; - } - - - private void checkMaximumNumberFailures(Constraint constraint) { - if (SH.Violation.equals(constraint.getShape().getSeverity())) { - this.violationsCount++; - if (configuration.getValidationErrorBatch() != -1 && violationsCount >= configuration.getValidationErrorBatch()) { - throw new MaximumNumberViolations(violationsCount); - } - } - } - - - public ClassesCache getClassesCache() { - return classesCache; - } - - - public ValidationEngineConfiguration getConfiguration() { - return configuration; - } - - - public String getLabel(RDFNode node) { - return labelsCache.computeIfAbsent(node, n -> getLabelFunction().apply(n)); - } - - - public Function getLabelFunction() { - return labelFunction; - } - - - public ValidationProfile getProfile() { - return profile; - } - - - /** - * Gets the validation report as a Resource in the report Model. - * @return the report Resource - */ - public Resource getReport() { - return report; - } - - - /** - * Gets a Set of all shapes that should be evaluated for a given resource. - * @param focusNode the focus node to get the shapes for - * @param dataset the Dataset containing the resource - * @param shapesModel the shapes Model - * @return a Set of shape resources - */ - private Set getShapesForNode(RDFNode focusNode, Dataset dataset, Model shapesModel) { - - Set shapes = new HashSet<>(); - - for(Shape rootShape : shapesGraph.getRootShapes()) { - for(Target target : rootShape.getTargets()) { - if(!(target instanceof InstancesTarget)) { - if(target.contains(dataset, focusNode)) { - shapes.add(rootShape.getShapeResource()); - } - } - } - } - - // rdf:type / sh:targetClass - if(focusNode instanceof Resource) { - for(Resource type : JenaUtil.getAllTypes((Resource)focusNode)) { - if(JenaUtil.hasIndirectType(type.inModel(shapesModel), SH.Shape)) { - shapes.add(type); - } - for(Statement s : shapesModel.listStatements(null, SH.targetClass, type).toList()) { - shapes.add(s.getSubject()); - } - } - } - - return shapes; - } - - - public ValidationReport getValidationReport() { - return new ResourceValidationReport(report); - } - - - public Collection getValueNodes(Constraint constraint, RDFNode focusNode) { - if(constraint.getShape().isNodeShape()) { - return Collections.singletonList(focusNode); - } - else { - // We use a cache here because many shapes contains for example both sh:datatype and sh:minCount, and fetching - // the value nodes each time may be expensive, esp for sh:minCount/maxCount constraints. - ValueNodesCacheKey key = new ValueNodesCacheKey(focusNode, constraint.getShape().getPath()); - return valueNodes.computeIfAbsent(key, k -> getValueNodesHelper(focusNode, constraint)); - } - } - - - private Collection getValueNodesHelper(RDFNode focusNode, Constraint constraint) { - Property predicate = constraint.getShape().getPredicate(); - if(predicate != null) { - List results = new LinkedList<>(); - if(focusNode instanceof Resource) { - Iterator it = ((Resource)focusNode).listProperties(predicate); - while(it.hasNext()) { - results.add(it.next().getObject()); - } - } - return results; - } - else { - Path jenaPath = constraint.getShape().getJenaPath(); - if(jenaPath instanceof P_Inverse && ((P_Inverse)jenaPath).getSubPath() instanceof P_Link) { - List results = new LinkedList<>(); - Property inversePredicate = ResourceFactory.createProperty(((P_Link)((P_Inverse)jenaPath).getSubPath()).getNode().getURI()); - Iterator it = focusNode.getModel().listStatements(null, inversePredicate, focusNode); - while(it.hasNext()) { - results.add(it.next().getSubject()); - } - return results; - } - Set results = new HashSet<>(); - Iterator it = PathEval.eval(focusNode.getModel().getGraph(), focusNode.asNode(), jenaPath, Context.emptyContext()); - while(it.hasNext()) { - Node node = it.next(); - results.add(focusNode.getModel().asRDFNode(node)); - } - return results; - } - } - - - /** - * Validates a given list of focus nodes against a given Shape, and stops as soon - * as one validation result is reported. No results are recorded. - * @param focusNodes the nodes to validate - * @param shape the sh:Shape to validate against - * @return true if there were no validation results, false for violations - */ - public boolean nodesConformToShape(List focusNodes, Node shape) { - if(!shapesGraph.isIgnored(shape)) { - Resource oldReport = report; - report = JenaUtil.createMemoryModel().createResource(); - try { - Shape vs = shapesGraph.getShape(shape); - if(!vs.isDeactivated()) { - boolean nested = SHACLScriptEngineManager.get().begin(); - try { - for(Constraint constraint : vs.getConstraints()) { - validateNodesAgainstConstraint(focusNodes, constraint); - if(report.hasProperty(SH.result)) { - return false; - } - } - } - finally { - SHACLScriptEngineManager.get().end(nested); - } - } - } - finally { - this.report = oldReport; - } - } - return true; - } - - - public void setClassesCache(ClassesCache value) { - this.classesCache = value; - } - - - /** - * Sets a filter that can be used to skip certain focus node from validation. - * The filter must return true if the given candidate focus node shall be validated, - * and false to skip it. - * @param value the new filter - */ - public void setFocusNodeFilter(Predicate value) { - this.focusNodeFilter = value; - } - - - public void setLabelFunction(Function value) { - this.labelFunction = value; - } - - - public void updateConforms() { - boolean conforms = true; - StmtIterator it = report.listProperties(SH.result); - while(it.hasNext()) { - Statement s = it.next(); - if(s.getResource().hasProperty(RDF.type, SH.ValidationResult)) { - conforms = false; - it.close(); - break; - } - } - if(report.hasProperty(SH.conforms)) { - report.removeAll(SH.conforms); - } - report.addProperty(SH.conforms, conforms ? JenaDatatypes.TRUE : JenaDatatypes.FALSE); - } - - - /** - * Validates all target nodes against all of their shapes. - * To further narrow down which nodes to validate, use {@link #setFocusNodeFilter(Predicate)}. - * @return an instance of sh:ValidationReport in the results Model - * @throws InterruptedException if the monitor has canceled this - */ - public Resource validateAll() throws InterruptedException { - List rootShapes = shapesGraph.getRootShapes(); - return validateShapes(rootShapes); - } - - - /** - * Validates a given focus node against all of the shapes that have matching targets. - * @param focusNode the node to validate - * @return an instance of sh:ValidationReport in the results Model - * @throws InterruptedException if the monitor has canceled this - */ - public Resource validateNode(Node focusNode) throws InterruptedException { - - Model shapesModel = dataset.getNamedModel(shapesGraphURI.toString()); - - RDFNode focusRDFNode = dataset.getDefaultModel().asRDFNode(focusNode); - Set shapes = getShapesForNode(focusRDFNode, dataset, shapesModel); - boolean nested = SHACLScriptEngineManager.get().begin(); - try { - for(Resource shape : shapes) { - if(monitor != null && monitor.isCanceled()) { - throw new InterruptedException(); - } - validateNodesAgainstShape(Collections.singletonList(focusRDFNode), shape.asNode()); - } - } - finally { - SHACLScriptEngineManager.get().end(nested); - } - - return report; - } - - - /** - * Validates a given list of focus node against a given Shape. - * @param focusNodes the nodes to validate - * @param shape the sh:Shape to validate against - * @return an instance of sh:ValidationReport in the results Model - */ - public Resource validateNodesAgainstShape(List focusNodes, Node shape) { - if(!shapesGraph.isIgnored(shape)) { - Shape vs = shapesGraph.getShape(shape); - if(!vs.isDeactivated()) { - boolean nested = SHACLScriptEngineManager.get().begin(); - ValidationEngine oldEngine = current.get(); - current.set(this); - try { - for(Constraint constraint : vs.getConstraints()) { - validateNodesAgainstConstraint(focusNodes, constraint); - } - } - finally { - current.set(oldEngine); - SHACLScriptEngineManager.get().end(nested); - } - } - } - return report; - } - - - /** - * Validates all target nodes of a given collection of shapes against these shapes. - * To further narrow down which nodes to validate, use {@link #setFocusNodeFilter(Predicate)}. - * @return an instance of sh:ValidationReport in the results Model - * @throws InterruptedException if the monitor has canceled this - */ - public Resource validateShapes(Collection shapes) throws InterruptedException { - boolean nested = SHACLScriptEngineManager.get().begin(); - try { - if(monitor != null) { - monitor.beginTask("Validating " + shapes.size() + " shapes", shapes.size()); - } - if(classesCache == null) { - // If we are doing everything then the cache should be used, but not for validation of individual focus nodes - classesCache = new ClassesCache(); - } - int i = 0; - for(Shape shape : shapes) { - - if(monitor != null) { - String label = "Shape " + (++i) + ": " + getLabelFunction().apply(shape.getShapeResource()); - if(resultsCount > 0) { - label = "" + resultsCount + " results. " + label; - } - monitor.subTask(label); - } - - Collection focusNodes = shape.getTargetNodes(dataset); - if(focusNodeFilter != null) { - List filteredFocusNodes = new LinkedList<>(); - for(RDFNode focusNode : focusNodes) { - if(focusNodeFilter.test(focusNode)) { - filteredFocusNodes.add(focusNode); - } - } - focusNodes = filteredFocusNodes; - } - if(!focusNodes.isEmpty()) { - for(Constraint constraint : shape.getConstraints()) { - validateNodesAgainstConstraint(focusNodes, constraint); - } - } - if(monitor != null) { - monitor.worked(1); - if(monitor.isCanceled()) { - throw new InterruptedException(); - } - } - } - } - catch(MaximumNumberViolations ex) { - // Ignore as this is just our way to stop validation when max number of violations is reached - } - finally { - SHACLScriptEngineManager.get().end(nested); - } - updateConforms(); - return report; - } - - - protected void validateNodesAgainstConstraint(Collection focusNodes, Constraint constraint) { - if(configuration != null && configuration.isSkippedConstraintComponent(constraint.getComponent())) { - return; - } - - ConstraintExecutor executor; - try { - executor = constraint.getExecutor(); - } - catch(Exception ex) { - Resource result = createResult(DASH.FailureResult, constraint, constraint.getShapeResource()); - result.addProperty(SH.resultMessage, "Failed to create validator: " + ExceptionUtil.getStackTrace(ex)); - return; - } - if(executor != null) { - if(SHACLPreferences.isProduceFailuresMode()) { - try { - executor.executeConstraint(constraint, this, focusNodes); - } - catch(Exception ex) { - Resource result = createResult(DASH.FailureResult, constraint, constraint.getShapeResource()); - result.addProperty(SH.resultMessage, "Exception during validation: " + ExceptionUtil.getStackTrace(ex)); - } - } - else { - executor.executeConstraint(constraint, this, focusNodes); - } - } - else { - FailureLog.get().logWarning("No suitable validator found for constraint " + constraint); - } - } - - - public void setConfiguration(ValidationEngineConfiguration configuration) { - this.configuration = configuration; - if(!configuration.getValidateShapes()) { - shapesGraph.setShapeFilter(new ExcludeMetaShapesFilter()); - } - } - - - public void setProfile(ValidationProfile profile) { - this.profile = profile; - } - - - // Used to avoid repeated computation of value nodes for a focus node / path combination - private static class ValueNodesCacheKey { - - Resource path; - - RDFNode focusNode; - - - ValueNodesCacheKey(RDFNode focusNode, Resource path) { - this.path = path; - this.focusNode = focusNode; - } - - - public boolean equals(Object o) { - if(o instanceof ValueNodesCacheKey) { - return path.equals(((ValueNodesCacheKey)o).path) && focusNode.equals(((ValueNodesCacheKey)o).focusNode); - } - else { - return false; - } - } - - - @Override - public int hashCode() { - return path.hashCode() + focusNode.hashCode(); - } - - - @Override - public String toString() { - return focusNode.toString() + " . " + path; - } - } + + // The currently active ValidationEngine for cases where no direct pointer can be acquired, e.g. from HasShapeFunction + private static ThreadLocal current = new ThreadLocal<>(); + + public static ValidationEngine getCurrent() { + return current.get(); + } + + public static void setCurrent(ValidationEngine value) { + current.set(value); + } + + + // Avoids repeatedly walking up/down the class hierarchy for sh:class constraints + private ClassesCache classesCache; + + private ValidationEngineConfiguration configuration; + + // Can be used to drop certain focus nodes from validation + private Predicate focusNodeFilter; + + // The inferred triples if the shapes graph declares an entailment regime + private Model inferencesModel; + + // The label function for rendering nodes in validation results (message templates etc) + private Function labelFunction = (node -> RDFLabels.get().getNodeLabel(node)); + + // Avoids repeatedly fetching labels + private Map labelsCache = new ConcurrentHashMap<>(); + + // Can be used to collect statistical data about execution time of constraint components and shapes + private ValidationProfile profile; + + // The resulting validation report instance + private Resource report; + + // Number of created results, e.g. for progress monitor + private int resultsCount = 0; + + // Avoids repeatedly fetching the value nodes of a focus node / path combination + // cache builder is thread-safe whereas WeakHashMap is not + private Cache> valueNodes = CacheBuilder.newBuilder().weakKeys().build(); + + // Number of created violations, e.g. for progress monitor + private int violationsCount = 0; + + + /** + * Constructs a new ValidationEngine. + * @param dataset the Dataset to operate on + * @param shapesGraphURI the URI of the shapes graph (must be in the dataset) + * @param shapesGraph the ShapesGraph with the shapes to validate against + * @param report the sh:ValidationReport object in the results Model, or null to create a new one + */ + protected ValidationEngine(Dataset dataset, URI shapesGraphURI, ShapesGraph shapesGraph, Resource report) { + super(dataset, shapesGraph, shapesGraphURI); + setConfiguration(new ValidationEngineConfiguration()); + if(report == null) { + Model reportModel = JenaUtil.createMemoryModel(); + reportModel.setNsPrefixes(dataset.getDefaultModel()); // This can be very expensive in some databases + reportModel.withDefaultMappings(shapesGraph.getShapesModel()); + this.report = reportModel.createResource(SH.ValidationReport); + } + else { + this.report = report; + } + } + + + /** + * Checks if entailments are active for the current shapes graph and applies them for a given focus node. + * This will only work for the sh:Rules entailment, e.g. to compute sh:values and sh:defaultValue. + * If any inferred triples exist, the focus node will be returned attached to the model that includes those inferences. + * The dataset used internally will also be switched to use that new model as its default model, so that if + * a node gets validated it will "see" the inferred triples too. + * @param focusNode the focus node + * @return the focus node, possibly in a different Model than originally + */ + public RDFNode applyEntailments(Resource focusNode) { + Model shapesModel = dataset.getNamedModel(shapesGraphURI.toString()); + if(shapesModel.contains(null, SH.entailment, SH.Rules)) { + + // Create union of data model and inferences if called for the first time + if(inferencesModel == null) { + inferencesModel = JenaUtil.createDefaultModel(); + Model dataModel = dataset.getDefaultModel(); + MultiUnion multiUnion = new MultiUnion(new Graph[]{ + dataModel.getGraph(), + inferencesModel.getGraph() + }); + multiUnion.setBaseGraph(dataModel.getGraph()); + dataset.setDefaultModel(ModelFactory.createModelForGraph(multiUnion)); + } + + // Apply sh:values rules + Map defaultValueMap = new HashMap<>(); + for(SHNodeShape nodeShape : SHACLUtil.getAllShapesAtNode(focusNode)) { + if(!nodeShape.hasProperty(SH.deactivated, JenaDatatypes.TRUE)) { + for(SHPropertyShape ps : nodeShape.getPropertyShapes()) { + if(!ps.hasProperty(SH.deactivated, JenaDatatypes.TRUE)) { + Resource path = ps.getPath(); + if(path instanceof Resource) { + Statement values = ps.getProperty(SH.values); + if(values != null) { + NodeExpression ne = NodeExpressionFactory.get().create(values.getObject()); + ne.eval(focusNode, this).forEachRemaining(v -> inferencesModel.getGraph().add(Triple.create(focusNode.asNode(), path.asNode(), v.asNode()))); + } + Statement defaultValue = ps.getProperty(SH.defaultValue); + if(defaultValue != null) { + defaultValueMap.put(JenaUtil.asProperty(path), defaultValue.getObject()); + } + } + } + } + } + } + + // Add sh:defaultValue where needed + Model dataModel = dataset.getDefaultModel(); // This is now the union model + Resource newFocusNode = focusNode.inModel(dataModel); + for(Property predicate : defaultValueMap.keySet()) { + if(!newFocusNode.hasProperty(predicate)) { + NodeExpression ne = NodeExpressionFactory.get().create(defaultValueMap.get(predicate)); + ne.eval(focusNode, this).forEachRemaining(v -> inferencesModel.add(focusNode, predicate, v)); + } + } + return newFocusNode; + } + return focusNode; + } + + + synchronized public void addResultMessage(Resource result, Literal message, QuerySolution bindings) { + result.addProperty(SH.resultMessage, SPARQLSubstitutions.withSubstitutions(message, bindings, getLabelFunction())); + } + + + // Note: does not set sh:path + synchronized public Resource createResult(Resource type, Constraint constraint, RDFNode focusNode) { + Resource result = report.getModel().createResource(type); + report.addProperty(SH.result, result); + result.addProperty(SH.resultSeverity, constraint.getSeverity()); + result.addProperty(SH.sourceConstraintComponent, constraint.getComponent()); + result.addProperty(SH.sourceShape, constraint.getShapeResource()); + if(focusNode != null) { + result.addProperty(SH.focusNode, focusNode); + } + + checkMaximumNumberFailures(constraint); + + resultsCount++; + + return result; + } + + + synchronized public Resource createValidationResult(Constraint constraint, RDFNode focusNode, RDFNode value, Supplier defaultMessage) { + Resource result = createResult(SH.ValidationResult, constraint, focusNode); + if(value != null) { + result.addProperty(SH.value, value); + } + if(!constraint.getShape().isNodeShape()) { + result.addProperty(SH.resultPath, SHACLPaths.clonePath(constraint.getShapeResource().getPath(), result.getModel())); + } + Collection messages = constraint.getMessages(); + if(messages.size() > 0) { + messages.stream().forEach(message -> result.addProperty(SH.resultMessage, message)); + } + else if(defaultMessage != null) { + String m = defaultMessage.get(); + if(m != null) { + result.addProperty(SH.resultMessage, m); + } + } + return result; + } + + + private void checkMaximumNumberFailures(Constraint constraint) { + if (SH.Violation.equals(constraint.getShape().getSeverity())) { + this.violationsCount++; + if (configuration.getValidationErrorBatch() != -1 && violationsCount >= configuration.getValidationErrorBatch()) { + throw new MaximumNumberViolations(violationsCount); + } + } + } + + + public ClassesCache getClassesCache() { + return classesCache; + } + + + public ValidationEngineConfiguration getConfiguration() { + return configuration; + } + + + public String getLabel(RDFNode node) { + return labelsCache.computeIfAbsent(node, n -> getLabelFunction().apply(n)); + } + + + public Function getLabelFunction() { + return labelFunction; + } + + + public ValidationProfile getProfile() { + return profile; + } + + + /** + * Gets the validation report as a Resource in the report Model. + * @return the report Resource + */ + public Resource getReport() { + return report; + } + + + /** + * Gets a Set of all shapes that should be evaluated for a given resource. + * @param focusNode the focus node to get the shapes for + * @param dataset the Dataset containing the resource + * @param shapesModel the shapes Model + * @return a Set of shape resources + */ + private Set getShapesForNode(RDFNode focusNode, Dataset dataset, Model shapesModel) { + + Set shapes = new HashSet<>(); + + for(Shape rootShape : shapesGraph.getRootShapes()) { + for(Target target : rootShape.getTargets()) { + if(!(target instanceof InstancesTarget)) { + if(target.contains(dataset, focusNode)) { + shapes.add(rootShape.getShapeResource()); + } + } + } + } + + // rdf:type / sh:targetClass + if(focusNode instanceof Resource) { + for(Resource type : JenaUtil.getAllTypes((Resource)focusNode)) { + if(JenaUtil.hasIndirectType(type.inModel(shapesModel), SH.Shape)) { + shapes.add(type); + } + for(Statement s : shapesModel.listStatements(null, SH.targetClass, type).toList()) { + shapes.add(s.getSubject()); + } + } + } + + return shapes; + } + + + public ValidationReport getValidationReport() { + return new ResourceValidationReport(report); + } + + + public Collection getValueNodes(Constraint constraint, RDFNode focusNode) { + if(constraint.getShape().isNodeShape()) { + return Collections.singletonList(focusNode); + } + else { + // We use a cache here because many shapes contains for example both sh:datatype and sh:minCount, and fetching + // the value nodes each time may be expensive, esp for sh:minCount/maxCount constraints. + ValueNodesCacheKey key = new ValueNodesCacheKey(focusNode, constraint.getShape().getPath()); + return valueNodes.asMap().computeIfAbsent(key, k -> getValueNodesHelper(focusNode, constraint)); + } + } + + + private Collection getValueNodesHelper(RDFNode focusNode, Constraint constraint) { + Property predicate = constraint.getShape().getPredicate(); + if(predicate != null) { + List results = new LinkedList<>(); + if(focusNode instanceof Resource) { + Iterator it = ((Resource)focusNode).listProperties(predicate); + while(it.hasNext()) { + results.add(it.next().getObject()); + } + } + return results; + } + else { + Path jenaPath = constraint.getShape().getJenaPath(); + if(jenaPath instanceof P_Inverse && ((P_Inverse)jenaPath).getSubPath() instanceof P_Link) { + List results = new LinkedList<>(); + Property inversePredicate = ResourceFactory.createProperty(((P_Link)((P_Inverse)jenaPath).getSubPath()).getNode().getURI()); + Iterator it = focusNode.getModel().listStatements(null, inversePredicate, focusNode); + while(it.hasNext()) { + results.add(it.next().getSubject()); + } + return results; + } + Set results = new HashSet<>(); + Iterator it = PathEval.eval(focusNode.getModel().getGraph(), focusNode.asNode(), jenaPath, Context.emptyContext()); + while(it.hasNext()) { + Node node = it.next(); + results.add(focusNode.getModel().asRDFNode(node)); + } + return results; + } + } + + + /** + * Validates a given list of focus nodes against a given Shape, and stops as soon + * as one validation result is reported. No results are recorded. + * @param focusNodes the nodes to validate + * @param shape the sh:Shape to validate against + * @return true if there were no validation results, false for violations + */ + public boolean nodesConformToShape(List focusNodes, Node shape) { + if(!shapesGraph.isIgnored(shape)) { + Resource oldReport = report; + report = JenaUtil.createMemoryModel().createResource(); + try { + Shape vs = shapesGraph.getShape(shape); + if(!vs.isDeactivated()) { + boolean nested = SHACLScriptEngineManager.get().begin(); + try { + for(Constraint constraint : vs.getConstraints()) { + validateNodesAgainstConstraint(focusNodes, constraint); + if(report.hasProperty(SH.result)) { + return false; + } + } + } + finally { + SHACLScriptEngineManager.get().end(nested); + } + } + } + finally { + this.report = oldReport; + } + } + return true; + } + + + public void setClassesCache(ClassesCache value) { + this.classesCache = value; + } + + + /** + * Sets a filter that can be used to skip certain focus node from validation. + * The filter must return true if the given candidate focus node shall be validated, + * and false to skip it. + * @param value the new filter + */ + public void setFocusNodeFilter(Predicate value) { + this.focusNodeFilter = value; + } + + + public void setLabelFunction(Function value) { + this.labelFunction = value; + } + + + synchronized public void updateConforms() { + boolean conforms = true; + StmtIterator it = report.listProperties(SH.result); + while(it.hasNext()) { + Statement s = it.next(); + if(s.getResource().hasProperty(RDF.type, SH.ValidationResult)) { + conforms = false; + it.close(); + break; + } + } + if(report.hasProperty(SH.conforms)) { + report.removeAll(SH.conforms); + } + report.addProperty(SH.conforms, conforms ? JenaDatatypes.TRUE : JenaDatatypes.FALSE); + } + + + /** + * Validates all target nodes against all of their shapes. + * To further narrow down which nodes to validate, use {@link #setFocusNodeFilter(Predicate)}. + * @return an instance of sh:ValidationReport in the results Model + * @throws InterruptedException if the monitor has canceled this + */ + public Resource validateAll() throws InterruptedException { + List rootShapes = shapesGraph.getRootShapes(); + return validateShapes(rootShapes); + } + + + /** + * Validates a given focus node against all of the shapes that have matching targets. + * @param focusNode the node to validate + * @return an instance of sh:ValidationReport in the results Model + * @throws InterruptedException if the monitor has canceled this + */ + public Resource validateNode(Node focusNode) throws InterruptedException { + + Model shapesModel = dataset.getNamedModel(shapesGraphURI.toString()); + + RDFNode focusRDFNode = dataset.getDefaultModel().asRDFNode(focusNode); + Set shapes = getShapesForNode(focusRDFNode, dataset, shapesModel); + boolean nested = SHACLScriptEngineManager.get().begin(); + try { + for(Resource shape : shapes) { + if(monitor != null && monitor.isCanceled()) { + throw new InterruptedException(); + } + validateNodesAgainstShape(Collections.singletonList(focusRDFNode), shape.asNode()); + } + } + finally { + SHACLScriptEngineManager.get().end(nested); + } + + return report; + } + + + /** + * Validates a given list of focus node against a given Shape. + * @param focusNodes the nodes to validate + * @param shape the sh:Shape to validate against + * @return an instance of sh:ValidationReport in the results Model + */ + public Resource validateNodesAgainstShape(List focusNodes, Node shape) { + if(!shapesGraph.isIgnored(shape)) { + Shape vs = shapesGraph.getShape(shape); + if(!vs.isDeactivated()) { + boolean nested = SHACLScriptEngineManager.get().begin(); + ValidationEngine oldEngine = current.get(); + current.set(this); + try { + for(Constraint constraint : vs.getConstraints()) { + validateNodesAgainstConstraint(focusNodes, constraint); + } + } + finally { + current.set(oldEngine); + SHACLScriptEngineManager.get().end(nested); + } + } + } + return report; + } + + /** + * Validates all target nodes of a given collection of shapes against these shapes. + * To further narrow down which nodes to validate, use {@link #setFocusNodeFilter(Predicate)}. + * @return an instance of sh:ValidationReport in the results Model + * @throws InterruptedException if the monitor has canceled this + */ + public Resource validateShapes(Collection shapes) throws InterruptedException { + int threads = Runtime.getRuntime().availableProcessors(); + threads = threads > 1 ? threads - 1 : 1; + ExecutorService executor = Executors.newFixedThreadPool(threads); + + boolean nested = SHACLScriptEngineManager.get().begin(); + try { + if(monitor != null) { + monitor.beginTask("Validating " + shapes.size() + " shapes", shapes.size()); + } + if(classesCache == null) { + // If we are doing everything then the cache should be used, but not for validation of individual focus nodes + classesCache = new ClassesCache(); + } + + for(Shape shape : shapes) { + + + if(monitor != null) { + String label = "Shape: " + getLabelFunction().apply(shape.getShapeResource()); + if(resultsCount > 0) { + label = "" + resultsCount + " results. " + label; + } + monitor.subTask(label); + } + + Collection focusNodes = shape.getTargetNodes(dataset); + if(focusNodeFilter != null) { + List filteredFocusNodes = new LinkedList<>(); + for(RDFNode focusNode : focusNodes) { + if(focusNodeFilter.test(focusNode)) { + filteredFocusNodes.add(focusNode); + } + } + focusNodes = filteredFocusNodes; + } + final Collection finalFocusNodes = new LinkedList<>(focusNodes); + + // thread the shapes matching for better performance assuming everything is thread-safe + // which is a very big assumption so this is more of a test really + if(!focusNodes.isEmpty()) { + for(Constraint constraint : shape.getConstraints()) { + executor.submit(() -> { + validateNodesAgainstConstraint(finalFocusNodes, constraint); + }); + } + } + if(monitor != null) { + monitor.worked(1); + if(monitor.isCanceled()) { + throw new ConcurrentRuntimeException(new InterruptedException()); + } + } + } + } + catch(MaximumNumberViolations ex) { + // Ignore as this is just our way to stop validation when max number of violations is reached + } + finally { + SHACLScriptEngineManager.get().end(nested); + + // shut down executor and wait for any threads to terminate + executor.shutdown(); + try { + if (!executor.awaitTermination(30, TimeUnit.SECONDS)) { + // report potentially useful information about why threads have not finished yet + StringBuffer threadDump = new StringBuffer(System.lineSeparator()); + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + for(ThreadInfo threadInfo : threadMXBean.dumpAllThreads(true, true)) { + threadDump.append(threadInfo.toString()); + } + FailureLog.get().logWarning("Still waiting for threads to complete - check for BLOCKED threads in thread dump that follows"); + FailureLog.get().logWarning(threadDump.toString()); + + executor.shutdownNow(); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + } + } + updateConforms(); + return report; + } + + protected void validateNodesAgainstConstraint(Collection focusNodes, Constraint constraint) { + if(configuration != null && configuration.isSkippedConstraintComponent(constraint.getComponent())) { + return; + } + + ConstraintExecutor executor; + try { + executor = constraint.getExecutor(); + } + catch(Exception ex) { + Resource result = createResult(DASH.FailureResult, constraint, constraint.getShapeResource()); + synchronized (result) { + result.addProperty(SH.resultMessage, "Failed to create validator: " + ExceptionUtil.getStackTrace(ex)); + } + return; + } + if(executor != null) { + if(SHACLPreferences.isProduceFailuresMode()) { + try { + executor.executeConstraint(constraint, this, focusNodes); + } + catch(Exception ex) { + Resource result = createResult(DASH.FailureResult, constraint, constraint.getShapeResource()); + synchronized (result) { + result.addProperty(SH.resultMessage, "Exception during validation: " + ExceptionUtil.getStackTrace(ex)); + } + } + } + else { + executor.executeConstraint(constraint, this, focusNodes); + } + } + else { + FailureLog.get().logWarning("No suitable validator found for constraint " + constraint); + } + } + + + public void setConfiguration(ValidationEngineConfiguration configuration) { + this.configuration = configuration; + if(!configuration.getValidateShapes()) { + shapesGraph.setShapeFilter(new ExcludeMetaShapesFilter()); + } + } + + + public void setProfile(ValidationProfile profile) { + this.profile = profile; + } + + + // Used to avoid repeated computation of value nodes for a focus node / path combination + private static class ValueNodesCacheKey { + + Resource path; + + RDFNode focusNode; + + + ValueNodesCacheKey(RDFNode focusNode, Resource path) { + this.path = path; + this.focusNode = focusNode; + } + + + public boolean equals(Object o) { + if(o instanceof ValueNodesCacheKey) { + return path.equals(((ValueNodesCacheKey)o).path) && focusNode.equals(((ValueNodesCacheKey)o).focusNode); + } + else { + return false; + } + } + + + @Override + public int hashCode() { + return path.hashCode() + focusNode.hashCode(); + } + + + @Override + public String toString() { + return focusNode.toString() + " . " + path; + } + } } diff --git a/src/main/java/org/topbraid/shacl/validation/sparql/AbstractSPARQLExecutor.java b/src/main/java/org/topbraid/shacl/validation/sparql/AbstractSPARQLExecutor.java index 3fbcf17d..060588bb 100644 --- a/src/main/java/org/topbraid/shacl/validation/sparql/AbstractSPARQLExecutor.java +++ b/src/main/java/org/topbraid/shacl/validation/sparql/AbstractSPARQLExecutor.java @@ -189,39 +189,41 @@ private void executeSelectQuery(ValidationEngine engine, Constraint constraint, } Resource result = engine.createResult(resultType, constraint, thisValue); - if(SH.SPARQLConstraintComponent.equals(constraint.getComponent())) { - result.addProperty(SH.sourceConstraint, constraint.getParameterValue()); - } - - if(selectMessage != null) { - result.addProperty(SH.resultMessage, selectMessage); - } - else if(constraint.getShapeResource().hasProperty(SH.message)) { - for(Statement s : constraint.getShapeResource().listProperties(SH.message).toList()) { - result.addProperty(SH.resultMessage, s.getObject()); - } - } - else { - addDefaultMessages(engine, messageHolder, constraint.getComponent(), result, bindings, sol); - } - - RDFNode pathValue = sol.get(SH.pathVar.getVarName()); - if(pathValue != null && pathValue.isURIResource()) { - result.addProperty(SH.resultPath, pathValue); - } - else if(constraint.getShapeResource().isPropertyShape()) { - Resource basePath = constraint.getShapeResource().getPropertyResourceValue(SH.path); - result.addProperty(SH.resultPath, SHACLPaths.clonePath(basePath, result.getModel())); - } - - if(!SH.HasValueConstraintComponent.equals(constraint.getComponent())) { // See https://github.com/w3c/data-shapes/issues/111 - RDFNode selectValue = sol.get(SH.valueVar.getVarName()); - if(selectValue != null) { - result.addProperty(SH.value, selectValue); - } - else if(SH.NodeShape.equals(constraint.getContext())) { - result.addProperty(SH.value, focusNode); - } + synchronized (result) { + if(SH.SPARQLConstraintComponent.equals(constraint.getComponent())) { + result.addProperty(SH.sourceConstraint, constraint.getParameterValue()); + } + + if(selectMessage != null) { + result.addProperty(SH.resultMessage, selectMessage); + } + else if(constraint.getShapeResource().hasProperty(SH.message)) { + for(Statement s : constraint.getShapeResource().listProperties(SH.message).toList()) { + result.addProperty(SH.resultMessage, s.getObject()); + } + } + else { + addDefaultMessages(engine, messageHolder, constraint.getComponent(), result, bindings, sol); + } + + RDFNode pathValue = sol.get(SH.pathVar.getVarName()); + if(pathValue != null && pathValue.isURIResource()) { + result.addProperty(SH.resultPath, pathValue); + } + else if(constraint.getShapeResource().isPropertyShape()) { + Resource basePath = constraint.getShapeResource().getPropertyResourceValue(SH.path); + result.addProperty(SH.resultPath, SHACLPaths.clonePath(basePath, result.getModel())); + } + + if(!SH.HasValueConstraintComponent.equals(constraint.getComponent())) { // See https://github.com/w3c/data-shapes/issues/111 + RDFNode selectValue = sol.get(SH.valueVar.getVarName()); + if(selectValue != null) { + result.addProperty(SH.value, selectValue); + } + else if(SH.NodeShape.equals(constraint.getContext())) { + result.addProperty(SH.value, focusNode); + } + } } if(engine.getConfiguration().getReportDetails()) { @@ -232,11 +234,13 @@ else if(SH.NodeShape.equals(constraint.getContext())) { } else if(createSuccessResults) { Resource success = engine.createResult(DASH.SuccessResult, constraint, focusNode); - if(SH.SPARQLConstraintComponent.equals(constraint.getComponent())) { - success.addProperty(SH.sourceConstraint, constraint.getParameterValue()); - } - if(engine.getConfiguration().getReportDetails()) { - addDetails(success, nestedResults); + synchronized (success) { + if(SH.SPARQLConstraintComponent.equals(constraint.getComponent())) { + success.addProperty(SH.sourceConstraint, constraint.getParameterValue()); + } + if(engine.getConfiguration().getReportDetails()) { + addDetails(success, nestedResults); + } } } } @@ -264,7 +268,7 @@ private void addDefaultMessages(ValidationEngine engine, Resource messageHolder, } - public static void addDetails(Resource parentResult, Model nestedResults) { + synchronized public static void addDetails(Resource parentResult, Model nestedResults) { if(!nestedResults.isEmpty()) { parentResult.getModel().add(nestedResults); for(Resource type : SHACLUtil.RESULT_TYPES) {