runtime-v2: plugin to assert flow call input parameters

cli/src/main/java/com/walmartlabs/concord/cli/Run.java

@@ -31,20 +31,19 @@
 import com.walmartlabs.concord.dependencymanager.DependencyManager;
 import com.walmartlabs.concord.dependencymanager.DependencyManagerConfiguration;
 import com.walmartlabs.concord.dependencymanager.DependencyManagerRepositories;
-import com.walmartlabs.concord.imports.ImportManager;
-import com.walmartlabs.concord.imports.ImportManagerFactory;
-import com.walmartlabs.concord.imports.ImportProcessingException;
-import com.walmartlabs.concord.imports.Imports;
+import com.walmartlabs.concord.imports.*;
 import com.walmartlabs.concord.process.loader.model.ProcessDefinitionUtils;
 import com.walmartlabs.concord.process.loader.v2.ProcessDefinitionV2;
 import com.walmartlabs.concord.runtime.common.cfg.RunnerConfiguration;
+import com.walmartlabs.concord.runtime.v2.NoopImportsNormalizer;
 import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;
 import com.walmartlabs.concord.runtime.v2.ProjectSerializerV2;
 import com.walmartlabs.concord.runtime.v2.model.ProcessDefinition;
 import com.walmartlabs.concord.runtime.v2.model.ProcessDefinitionConfiguration;
 import com.walmartlabs.concord.runtime.v2.model.Profile;
 import com.walmartlabs.concord.runtime.v2.model.Step;
 import com.walmartlabs.concord.runtime.v2.runner.InjectorFactory;
+import com.walmartlabs.concord.runtime.v2.runner.ProjectLoadListeners;
 import com.walmartlabs.concord.runtime.v2.runner.Runner;
 import com.walmartlabs.concord.runtime.v2.runner.guice.ProcessDependenciesModule;
 import com.walmartlabs.concord.runtime.v2.runner.tasks.TaskProviders;
@@ -256,6 +255,11 @@ public Integer call() throws Exception {
                 new CliServicesModule(secretStoreDir, targetDir, new VaultProvider(vaultDir, vaultId), dependencyManager, verbosity))
                 .create();
 
+        // Just to notify listeners
+        ProjectLoadListeners loadListeners = injector.getInstance(ProjectLoadListeners.class);
+        ProjectLoaderV2 loader = new ProjectLoaderV2(new NoopImportManager());
+        loader.load(targetDir, new NoopImportsNormalizer(), ImportsListener.NOP_LISTENER, loadListeners);
+
         Runner runner = injector.getInstance(Runner.class);
 
         if (cfg.debug()) {

plugins/pom.xml

@@ -36,5 +36,6 @@
         <module>tasks/variables</module>
         <module>templates/ansible</module>
         <module>tasks/files</module>
+        <module>tasks/input-params-assert</module>
     </modules>
 </project>

plugins/tasks/input-params-assert/pom.xml

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.walmartlabs.concord.plugins.basic</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.102.1-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>input-params-assert</artifactId>
+    <packaging>takari-jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.walmartlabs.concord.runtime.v2</groupId>
+            <artifactId>concord-runtime-sdk-v2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord.runtime.v2</groupId>
+            <artifactId>concord-runner-v2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord.runtime.v2</groupId>
+            <artifactId>concord-runtime-vm-v2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord.runtime.v2</groupId>
+            <artifactId>concord-runtime-model-v2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>json-schema-validator</artifactId>
+        </dependency>
+
+        <!-- Immutables -->
+        <dependency>
+            <groupId>org.immutables</groupId>
+            <artifactId>value</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.errorprone</groupId>
+            <artifactId>error_prone_annotations</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.inject</groupId>
+            <artifactId>javax.inject</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>3.24.2</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>net.bytebuddy</groupId>
+                    <artifactId>byte-buddy</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.eclipse.sisu</groupId>
+                <artifactId>sisu-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>io.takari.maven.plugins</groupId>
+                <artifactId>takari-lifecycle-plugin</artifactId>
+                <extensions>true</extensions>
+                <configuration>
+                    <proc>proc</proc>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

plugins/tasks/input-params-assert/src/main/java/com/walmartlabs/concord/plugins/input/ConcordYamlProcessor.java

@@ -0,0 +1,70 @@
+package com.walmartlabs.concord.plugins.input;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2023 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.walmartlabs.concord.plugins.input.parser.CommentParser;
+import com.walmartlabs.concord.plugins.input.parser.CommentsGrabber;
+import com.walmartlabs.concord.runtime.v2.ProjectLoadListener;
+import com.walmartlabs.concord.runtime.v2.runner.PersistenceService;
+
+import javax.inject.Inject;
+import java.nio.file.Path;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ConcordYamlProcessor implements ProjectLoadListener {
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private final PersistenceService persistenceService;
+
+    private static final FlowCallSchemaGenerator schemaGenerator = new FlowCallSchemaGenerator(new CommentsGrabber(), new CommentParser());
+
+    private final Map<String, Object> definitions = new LinkedHashMap<>();
+
+    @Inject
+    public ConcordYamlProcessor(PersistenceService persistenceService) {
+        this.persistenceService = persistenceService;
+    }
+
+    @Override
+    public void afterFlowDefinitionLoaded(Path filename) {
+        if (schemaExists()) {
+            return;
+        }
+
+        definitions.putAll(schemaGenerator.generate(filename));
+    }
+
+    @Override
+    public void afterProjectLoaded() {
+        if (schemaExists()) {
+            return;
+        }
+
+        persistenceService.persistFile(InputParamsDefinitionProvider.DEFAULT_SCHEMA_FILENAME, out -> mapper.writeValue(out, definitions));
+    }
+
+    private boolean schemaExists() {
+        return persistenceService.loadPersistedFile(InputParamsDefinitionProvider.DEFAULT_SCHEMA_FILENAME, inputStream -> true) != null;
+    }
+}

plugins/tasks/input-params-assert/src/main/java/com/walmartlabs/concord/plugins/input/FlowCallInputParamsAssert.java

@@ -0,0 +1,75 @@
+package com.walmartlabs.concord.plugins.input;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2023 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.networknt.schema.JsonSchema;
+import com.networknt.schema.JsonSchemaFactory;
+import com.networknt.schema.SpecVersion;
+import com.networknt.schema.ValidationMessage;
+import com.walmartlabs.concord.runtime.v2.sdk.ProcessConfiguration;
+import com.walmartlabs.concord.runtime.v2.sdk.UserDefinedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.Map;
+import java.util.Set;
+
+public class FlowCallInputParamsAssert {
+
+    private static final Logger log = LoggerFactory.getLogger(FlowCallInputParamsAssert.class);
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private final InputParamsDefinitionProvider definitionProvider;
+
+    private final boolean debug;
+
+    @Inject
+    public FlowCallInputParamsAssert(InputParamsDefinitionProvider definitionProvider, ProcessConfiguration cfg) {
+        this.definitionProvider = definitionProvider;
+        this.debug = cfg.debug();
+    }
+
+    public void process(String flowName, Map<String, Object> input) {
+        JsonNode definition = definitionProvider.get(flowName);
+        if (definition == null) {
+            if (debug) {
+                log.info("Can't find input params definition for '{}' flow", flowName);
+            }
+            return;
+        }
+
+        if (debug) {
+            log.info("Validating input params for '{}' flow", flowName);
+        }
+
+        JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4);
+        JsonSchema schema = schemaFactory.getSchema(definition);
+
+        Set<ValidationMessage> validationResult = schema.validate(mapper.valueToTree(input));
+        if (!validationResult.isEmpty()) {
+            throw new UserDefinedException("Input params validation failed: " + validationResult);
+        }
+    }
+}

plugins/tasks/input-params-assert/src/main/java/com/walmartlabs/concord/plugins/input/FlowCallListener.java

@@ -0,0 +1,59 @@
+package com.walmartlabs.concord.plugins.input;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2023 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.runtime.v2.model.Location;
+import com.walmartlabs.concord.runtime.v2.runner.vm.FlowCallCommand;
+import com.walmartlabs.concord.runtime.v2.runner.vm.VMUtils;
+import com.walmartlabs.concord.runtime.v2.sdk.UserDefinedException;
+import com.walmartlabs.concord.svm.*;
+import com.walmartlabs.concord.svm.Runtime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+
+public class FlowCallListener implements ExecutionListener {
+
+    private static final Logger log = LoggerFactory.getLogger(FlowCallListener.class);
+
+    private final FlowCallInputParamsAssert inputParamsAssert;
+
+    @Inject
+    public FlowCallListener(FlowCallInputParamsAssert inputParamsAssert) {
+        this.inputParamsAssert = inputParamsAssert;
+    }
+
+    @Override
+    public Result afterCommand(Runtime runtime, VM vm, State state, ThreadId threadId, Command cmd) {
+        if (cmd instanceof FlowCallCommand) {
+            String flowName = VMUtils.getLocal(state, threadId, "flowName");
+            try {
+                inputParamsAssert.process(flowName, VMUtils.getCombinedLocals(state, threadId));
+            } catch (UserDefinedException e) {
+                log.error("{} {}", Location.toErrorPrefix(((FlowCallCommand) cmd).getStep().getLocation()), e.getMessage());
+                throw e;
+            }
+        }
+
+        return ExecutionListener.super.afterCommand(runtime, vm, state, threadId, cmd);
+    }
+}