From 042e84b09a8e006c249a045d3161c03fac38a420 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Thu, 2 Jan 2025 12:30:39 +0100 Subject: [PATCH] refactor: migrate to dynamic properties --- .../plugin/dataform/cli/DataformCLI.java | 25 ++++++++++--------- .../plugin/dataform/cli/DataformCLITest.java | 5 ++-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/kestra/plugin/dataform/cli/DataformCLI.java b/src/main/java/io/kestra/plugin/dataform/cli/DataformCLI.java index 996dc1e..1d1713d 100644 --- a/src/main/java/io/kestra/plugin/dataform/cli/DataformCLI.java +++ b/src/main/java/io/kestra/plugin/dataform/cli/DataformCLI.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.*; import io.kestra.core.models.tasks.runners.ScriptService; import io.kestra.core.models.tasks.runners.TaskRunner; @@ -49,7 +50,7 @@ tasks: - id: clone_repo type: io.kestra.plugin.git.Clone - url: https://github.com/dataform-co/dataform-example-project-bigquery + url: https://github.com/dataform-co/dataform-example-project-bigquery - id: transform type: io.kestra.plugin.dataform.cli.DataformCLI @@ -78,8 +79,7 @@ public class DataformCLI extends Task implements RunnableTask, Nam @Schema( title = "The commands to run before main list of commands" ) - @PluginProperty(dynamic = true) - protected List beforeCommands; + protected Property> beforeCommands; @Schema( title = "The commands to run" @@ -115,34 +115,35 @@ public class DataformCLI extends Task implements RunnableTask, Nam private TaskRunner taskRunner = Docker.instance(); @Schema(title = "The task runner container image, only used if the task runner is container-based.") - @PluginProperty(dynamic = true) @Builder.Default - private String containerImage = DEFAULT_IMAGE; + private Property containerImage = Property.of(DEFAULT_IMAGE); private NamespaceFiles namespaceFiles; private Object inputFiles; - private List outputFiles; + private Property> outputFiles; @Override public ScriptOutput run(RunContext runContext) throws Exception { + var renderedOutputFiles = runContext.render(this.outputFiles).asList(String.class); + var renderedBeforeCommands = runContext.render(this.beforeCommands).asList(String.class); return new CommandsWrapper(runContext) .withWarningOnStdErr(true) .withDockerOptions(injectDefaults(getDocker())) .withTaskRunner(this.taskRunner) - .withContainerImage(this.containerImage) + .withContainerImage(runContext.render(this.containerImage).as(String.class).orElse(null)) .withEnv(Optional.ofNullable(env).orElse(new HashMap<>())) .withNamespaceFiles(namespaceFiles) .withInputFiles(inputFiles) - .withOutputFiles(outputFiles) + .withOutputFiles(renderedOutputFiles.isEmpty() ? null : renderedOutputFiles) .withCommands( ScriptService.scriptCommands( List.of("/bin/sh", "-c"), - Optional.ofNullable(this.beforeCommands).map(throwFunction(runContext::render)).orElse(null), + renderedBeforeCommands.isEmpty() ? null : renderedBeforeCommands, runContext.render(this.commands) - ) - ) + ) + ) .run(); } @@ -150,7 +151,7 @@ private DockerOptions injectDefaults(DockerOptions original) { if (original == null) { return null; } - + var builder = original.toBuilder(); if (original.getImage() == null) { builder.image(DEFAULT_IMAGE); diff --git a/src/test/java/io/kestra/plugin/dataform/cli/DataformCLITest.java b/src/test/java/io/kestra/plugin/dataform/cli/DataformCLITest.java index 0b217ea..6a7ac4c 100644 --- a/src/test/java/io/kestra/plugin/dataform/cli/DataformCLITest.java +++ b/src/test/java/io/kestra/plugin/dataform/cli/DataformCLITest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.dataform.cli; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -44,10 +45,10 @@ void run() throws Exception { runner = dataformBuilder .env(Map.of("{{ inputs.environmentKey }}", "{{ inputs.environmentValue }}")) - .beforeCommands(List.of( + .beforeCommands(Property.of(List.of( "dataform init postgres new_project", "cd new_project" - )) + ))) .commands(List.of( "echo \"::{\\\"outputs\\\":{" + "\\\"customEnv\\\":\\\"$" + environmentKey + "\\\"" +