Merge remote-tracking branch 'origin/staging' into NU.2152.loose.redundant.parameters.validation

Author: arkadius

build.sbt

@@ -857,7 +857,8 @@ lazy val scenarioCompiler = (project in file("scenario-compiler"))
         "org.apache.avro"         % "avro"                          % avroV          % Test,
         "org.scalacheck"         %% "scalacheck"                    % scalaCheckV    % Test,
         "com.cronutils"           % "cron-utils"                    % cronParserV    % Test,
-        "org.scalatestplus"      %% s"scalacheck-$scalaCheckVshort" % scalaTestPlusV % Test
+        "org.scalatestplus"      %% s"scalacheck-$scalaCheckVshort" % scalaTestPlusV % Test,
+        "org.apache.flink"        % "flink-core"                    % flinkV         % Test,
       )
     }
   )
@@ -1160,7 +1161,12 @@ lazy val mathUtils = (project in utils("math-utils"))
 lazy val defaultHelpers = (project in utils("default-helpers"))
   .settings(commonSettings)
   .settings(
-    name := "nussknacker-default-helpers"
+    name := "nussknacker-default-helpers",
+    libraryDependencies ++= {
+      Seq(
+        "org.apache.flink" % "flink-core" % flinkV % Test,
+      )
+    }
   )
   .dependsOn(mathUtils, commonUtils, testUtils % Test, scenarioCompiler % "test->test;test->compile")
 

common-api/src/main/scala/pl/touk/nussknacker/engine/graph/expression/Expression.scala

@@ -7,6 +7,7 @@ import pl.touk.nussknacker.engine.graph.expression.Expression.Language
 import pl.touk.nussknacker.engine.graph.expression.Expression.Language.{
   DictKeyWithLabel,
   Json,
+  JsonTemplate,
   Spel,
   SpelTemplate,
   TabularDataDefinition
@@ -25,6 +26,7 @@ object Expression {
       case DictKeyWithLabel      => "dictKeyWithLabel"
       case TabularDataDefinition => "tabularDataDefinition"
       case Json                  => "json"
+      case JsonTemplate          => "jsonTemplate"
     }
 
   }
@@ -35,6 +37,7 @@ object Expression {
     object DictKeyWithLabel      extends Language
     object TabularDataDefinition extends Language
     object Json                  extends Language
+    object JsonTemplate          extends Language
 
     implicit val encoder: Encoder[Language] = Encoder.encodeString.contramap(_.toString)
 
@@ -44,6 +47,7 @@ object Expression {
       case "dictKeyWithLabel"      => Right(DictKeyWithLabel)
       case "tabularDataDefinition" => Right(TabularDataDefinition)
       case "json"                  => Right(Json)
+      case "jsonTemplate"          => Right(JsonTemplate)
       case unknown                 => Left(s"Unknown language [$unknown]")
     }
 
@@ -61,4 +65,6 @@ object Expression {
   def tabularDataDefinition(definition: String): Expression = Expression(Language.TabularDataDefinition, definition)
 
   def json(jsonString: String): Expression = Expression(Language.Json, jsonString)
+
+  def jsonTemplate(jsonTemplateString: String): Expression = Expression(Language.JsonTemplate, jsonTemplateString)
 }

components-api/src/main/java/pl/touk/nussknacker/engine/api/DefaultValue.java

@@ -9,4 +9,5 @@
 @Retention(RetentionPolicy.RUNTIME)
 public @interface DefaultValue {
     String value();
-}
+    ExpressionLanguage language() default ExpressionLanguage.SPEL;
+}

components-api/src/main/java/pl/touk/nussknacker/engine/api/ExpressionLanguage.java

@@ -0,0 +1,10 @@
+package pl.touk.nussknacker.engine.api;
+
+public enum ExpressionLanguage {
+    SPEL,
+    SPEL_TEMPLATE,
+    DICT_KEY_WITH_LABEL,
+    TABULAR_DATA_DEFINITION,
+    JSON,
+    JSON_TEMPLATE,
+}

components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/EditorType.java

@@ -16,4 +16,5 @@ public enum EditorType {
     DICT_EDITOR,
     TYPED_TABULAR_DATA_EDITOR,
     SPEL_EDITOR,
+    JSON_TEMPLATE_EDITOR,
 }

components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala

@@ -29,6 +29,8 @@ case object TextareaParameterEditor extends ParameterEditor with StaticParameter
 
 case object JsonParameterEditor extends ParameterEditor with StaticParameterEditor
 
+case object JsonTemplateParameterEditor extends ParameterEditor with StaticParameterEditor
+
 case object SqlParameterEditor extends ParameterEditor
 
 case object SpelTemplateParameterEditor extends ParameterEditor

components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/TypeConversionHandler.scala

@@ -22,11 +22,11 @@ import scala.util.Try
   */
 private[engine] object TypeConversionHandler {
 
-  private val javaListClass              = classOf[java.util.List[_]]
-  private val javaCollectionClass        = classOf[java.util.Collection[_]]
-  private val javaMapClass               = classOf[java.util.Map[_, _]]
-  private val arrayOfAnyRefClass         = classOf[Array[AnyRef]]
-  private val avroIndexedRecordClassName = "org.apache.avro.generic.IndexedRecord"
+  private val javaListClass            = classOf[java.util.List[_]]
+  private val javaCollectionClass      = classOf[java.util.Collection[_]]
+  private val javaMapClass             = classOf[java.util.Map[_, _]]
+  private val arrayOfAnyRefClass       = classOf[Array[AnyRef]]
+  private val mapConvertableClassNames = List("org.apache.avro.generic.IndexedRecord", "org.apache.flink.types.Row")
 
   /**
     * java.math.BigDecimal is quite often returned as a wrapper for all kind of numbers (floating and without floating point).
@@ -93,7 +93,7 @@ private[engine] object TypeConversionHandler {
       case Loose =>
         Option.when(handleStringToValueClassConversions(from, to))(to) orElse
           handleArrayToListConversions(from.runtimeObjType, to) orElse
-          handleIndexedRecordToMapConversion(from, to)
+          handleMapConversions(from, to)
     }
   }
 
@@ -153,7 +153,7 @@ private[engine] object TypeConversionHandler {
     }
   }
 
-  private def handleIndexedRecordToMapConversion(
+  private def handleMapConversions(
       from: SingleTypingResult,
       to: TypedClass
   )(
@@ -167,7 +167,9 @@ private[engine] object TypeConversionHandler {
         lazy val indexedRecordValueType = superTypeOfTypes(fromFields.values)
 
         Option.when(
-          AssignabilityUtil.isAssignableToLoadableClass(fromRuntimeObjClass, avroIndexedRecordClassName) &&
+          mapConvertableClassNames.exists(className =>
+            AssignabilityUtil.isAssignableToLoadableClass(fromRuntimeObjClass, className)
+          ) &&
             AssignabilityDeterminer.isAssignable(Typed[String], mapKeyParam).isValid &&
             AssignabilityDeterminer.isAssignable(indexedRecordValueType, mapValueParam).isValid
         )(

defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/migrations/SpelToSpelTemplateNodeMigration.scala

@@ -5,7 +5,7 @@ import pl.touk.nussknacker.engine.graph.evaluatedparam.Parameter
 import pl.touk.nussknacker.engine.graph.expression.Expression
 import pl.touk.nussknacker.engine.graph.expression.Expression.Language
 import pl.touk.nussknacker.engine.graph.node
-import pl.touk.nussknacker.engine.migration.NodeMigration
+import pl.touk.nussknacker.engine.migration.{NodeMigration, ParametersMigration}
 
 object SpelToSpelTemplateNodeMigration extends NodeMigration {
   private lazy val spelStringPattern   = "^'(.*)'$".r

designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Node window should display event-generator source #0.png

@@ -4,7 +4,6 @@ export type ActionTypes =
     | "LOGGED_USER"
     | "UI_SETTINGS"
     | "AVAILABLE_QUERY_STATES"
-    | "SWITCH_TOOL_TIPS_HIGHLIGHT"
     | "ZOOM_IN"
     | "ZOOM_OUT"
     | "DELETE_NODES"
@@ -21,14 +20,11 @@ export type ActionTypes =
     | "EDIT_LABELS"
     | "SHOW_METRICS"
     | "UPDATE_TEST_CAPABILITIES"
-    | "UPDATE_TEST_TYPE"
     | "DISPLAY_PROCESS"
     | "GET_SCENARIO_ACTIVITIES"
     | "UPDATE_SCENARIO_ACTIVITIES"
     | "PROCESS_FETCH"
     | "PROCESS_LOADING"
-    | "TEST_RESULTS_LOADING"
-    | "TEST_RESULTS_FAILED"
     | "LOADING_FAILED"
     | "CLEAR_PROCESS"
     | "TOGGLE_PROCESS_ACTION_MODAL"

designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process initially clean should import JSON and save #0.png

@@ -3,71 +3,87 @@ import type { ProcessName } from "src/components/Process/types";
 import type { TestResults } from "../../common/TestResultUtils";
 import type { SourceWithParametersTest, TestProcessResponse } from "../../http/HttpService";
 import HttpService from "../../http/HttpService";
+import { getProcessName, getScenarioGraph } from "../../reducers/selectors/graph";
 import type { ScenarioGraph } from "../../types";
-import type { ThunkAction } from "../reduxTypes";
+import type { Action, ThunkAction } from "../reduxTypes";
 import { displayProcessCounts } from "./displayProcessCounts";
 
-export function testProcessFromFile(processName: ProcessName, testDataFile: File, scenarioGraph: ScenarioGraph): ThunkAction {
-    return (dispatch) => {
-        dispatch({
-            type: "PROCESS_LOADING",
-        });
+export function testProcessFromFile(testDataFile: File): ThunkAction {
+    return wrapWithTestAction((processName, scenarioGraph) =>
+        HttpService.testProcess(processName, testDataFile, scenarioGraph).then(({ data }) => ({
+            testResults: data,
+        })),
+    );
+}
 
-        HttpService.testProcess(processName, testDataFile, scenarioGraph)
-            .then((response) => dispatch(displayTestResults(response.data)))
-            .catch(() => dispatch({ type: "LOADING_FAILED" }));
-    };
+export function testProcessWithParameters(testData: SourceWithParametersTest): ThunkAction {
+    return wrapWithTestAction((processName, scenarioGraph) =>
+        HttpService.testProcessWithParameters(processName, testData, scenarioGraph).then(({ data }) => ({
+            testResults: data,
+            testData,
+        })),
+    );
+}
+
+export function testScenarioWithGeneratedData(testSampleSize: string): ThunkAction {
+    return wrapWithTestAction((processName, scenarioGraph) =>
+        HttpService.testScenarioWithGeneratedData(processName, parseInt(testSampleSize), scenarioGraph).then(({ data }) => ({
+            testResults: data,
+        })),
+    );
 }
 
-export function testProcessWithParameters(
-    processName: ProcessName,
-    testData: SourceWithParametersTest,
-    scenarioGraph: ScenarioGraph,
+export type TestsActions =
+    | { type: "TEST_RESULTS_LOADING" }
+    | { type: "TEST_RESULTS_FAILED" }
+    | {
+          type: "DISPLAY_TEST_RESULTS_DETAILS";
+          testResults: TestResults;
+          testData?: SourceWithParametersTest;
+      }
+    | {
+          type: "UPDATE_TEST_TYPE";
+          testType: string;
+      };
+
+function wrapWithTestAction(
+    fn: (
+        processName: ProcessName,
+        scenarioGraph: ScenarioGraph,
+    ) => Promise<{
+        testResults: TestProcessResponse;
+        testData?: SourceWithParametersTest;
+    }>,
 ): ThunkAction {
-    return (dispatch) => {
+    return (dispatch, getState) => {
         dispatch({ type: "TEST_RESULTS_LOADING" });
-
-        HttpService.testProcessWithParameters(processName, testData, scenarioGraph)
-            .then((response) => {
-                dispatch(displayTestResults(response.data, testData));
-            })
+        const state = getState();
+        const scenarioGraph = getScenarioGraph(state);
+        const processName = getProcessName(state);
+        fn(processName, scenarioGraph)
+            .then(({ testResults, testData }) => dispatch(displayTestResults(testResults, testData)))
             .catch(() => dispatch({ type: "TEST_RESULTS_FAILED" }));
     };
 }
 
-export function testScenarioWithGeneratedData(testSampleSize: string, processName: ProcessName, scenarioGraph: ScenarioGraph): ThunkAction {
-    return (dispatch) => {
-        dispatch({
-            type: "PROCESS_LOADING",
-        });
-        dispatch({ type: "TEST_RESULTS_LOADING" });
-
-        HttpService.testScenarioWithGeneratedData(processName, +testSampleSize, scenarioGraph)
-            .then((response) => dispatch(displayTestResults(response.data)))
-            .catch(() => {
-                dispatch({ type: "LOADING_FAILED" });
-                dispatch({ type: "TEST_RESULTS_FAILED" });
-            });
+function displayTestResultsDetails(testResults: TestResults, testData?: SourceWithParametersTest): Action {
+    return {
+        type: "DISPLAY_TEST_RESULTS_DETAILS",
+        testResults,
+        testData,
     };
 }
 
-export interface DisplayTestResultsDetailsAction {
-    type: "DISPLAY_TEST_RESULTS_DETAILS";
-    testResults: TestResults;
-    testData?: SourceWithParametersTest;
-}
-
-function displayTestResultsDetails(testResults: TestProcessResponse, testData?: SourceWithParametersTest): DisplayTestResultsDetailsAction {
+export function updateTestType(testType: string): Action {
     return {
-        type: "DISPLAY_TEST_RESULTS_DETAILS",
-        testResults: testResults.results,
-        testData,
+        type: "UPDATE_TEST_TYPE",
+        testType,
     };
 }
 
-function displayTestResults(testResults: TestProcessResponse, testData?: SourceWithParametersTest) {
+function displayTestResults({ counts, results }: TestProcessResponse, testData?: SourceWithParametersTest): ThunkAction {
     return (dispatch) => {
-        dispatch(displayTestResultsDetails(testResults, testData));
-        dispatch(displayProcessCounts(testResults.counts));
+        dispatch(displayTestResultsDetails(results, testData));
+        dispatch(displayProcessCounts(counts));
     };
 }

designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process initially clean should import JSON and save #1.png

@@ -1,4 +1,13 @@
-import { SwitchToolTipsHighlightAction } from "../tooltips";
-import { LayoutChangedAction, PanelActions } from "./layout";
+import type { EditState } from "../../../components/graph/node-modal/node/useNodeState";
+import type { SwitchToolTipsHighlightAction } from "../tooltips";
+import type { LayoutChangedAction, PanelActions } from "./layout";
 
-export type UiActions = SwitchToolTipsHighlightAction | PanelActions | LayoutChangedAction;
+export type UiActions =
+    | SwitchToolTipsHighlightAction
+    | PanelActions
+    | LayoutChangedAction
+    | {
+          type: "SET_PENDING_CHANGES";
+          id: string;
+          pendingChanges?: EditState;
+      };

designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Sticky notes should add text to note and display it as markdown #0.png

@@ -7,7 +7,7 @@ import type { SquashHistoryActions } from "../reducers/graph/historySquash";
 import type { ScenariosActions } from "../reducers/scenarios";
 import type { ActionTypes } from "./actionTypes";
 import type { CountsActions, NodeActions, NodeDetailsActions, PropertiesActions, ScenarioActions, SelectionActions } from "./nk";
-import type { DisplayTestResultsDetailsAction } from "./nk/displayTestResults";
+import type { TestsActions } from "./nk/displayTestResults";
 import type { NotificationActions } from "./nk/notifications";
 import type { GetScenarioActivitiesAction, UpdateScenarioActivitiesAction } from "./nk/scenarioActivities";
 import type { ToolbarActions } from "./nk/toolbars";
@@ -18,7 +18,6 @@ import type { SettingsActions } from "./settingsActions";
 type TypedAction =
     | CloudDataActions
     | CountsActions
-    | DisplayTestResultsDetailsAction
     | GetScenarioActivitiesAction
     | NodeActions
     | NodeDetailsActions
@@ -29,6 +28,7 @@ type TypedAction =
     | SelectionActions
     | SettingsActions
     | SquashHistoryActions
+    | TestsActions
     | ToolbarActions
     | UiActions
     | UpdateScenarioActivitiesAction