Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions src/main/java/io/kestra/plugin/dataform/cli/DataformCLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.*;
import io.kestra.core.models.tasks.runners.ScriptService;
import io.kestra.core.models.tasks.runners.TaskRunner;
Expand Down Expand Up @@ -49,7 +50,7 @@
tasks:
- id: clone_repo
type: io.kestra.plugin.git.Clone
url: https://github.com/dataform-co/dataform-example-project-bigquery
url: https://github.com/dataform-co/dataform-example-project-bigquery

- id: transform
type: io.kestra.plugin.dataform.cli.DataformCLI
Expand Down Expand Up @@ -78,8 +79,7 @@ public class DataformCLI extends Task implements RunnableTask<ScriptOutput>, Nam
@Schema(
title = "The commands to run before main list of commands"
)
@PluginProperty(dynamic = true)
protected List<String> beforeCommands;
protected Property<List<String>> beforeCommands;

@Schema(
title = "The commands to run"
Expand Down Expand Up @@ -115,42 +115,43 @@ public class DataformCLI extends Task implements RunnableTask<ScriptOutput>, Nam
private TaskRunner taskRunner = Docker.instance();

@Schema(title = "The task runner container image, only used if the task runner is container-based.")
@PluginProperty(dynamic = true)
@Builder.Default
private String containerImage = DEFAULT_IMAGE;
private Property<String> containerImage = Property.of(DEFAULT_IMAGE);

private NamespaceFiles namespaceFiles;

private Object inputFiles;

private List<String> outputFiles;
private Property<List<String>> outputFiles;

@Override
public ScriptOutput run(RunContext runContext) throws Exception {
var renderedOutputFiles = runContext.render(this.outputFiles).asList(String.class);
var renderedBeforeCommands = runContext.render(this.beforeCommands).asList(String.class);
return new CommandsWrapper(runContext)
.withWarningOnStdErr(true)
.withDockerOptions(injectDefaults(getDocker()))
.withTaskRunner(this.taskRunner)
.withContainerImage(this.containerImage)
.withContainerImage(runContext.render(this.containerImage).as(String.class).orElse(null))
.withEnv(Optional.ofNullable(env).orElse(new HashMap<>()))
.withNamespaceFiles(namespaceFiles)
.withInputFiles(inputFiles)
.withOutputFiles(outputFiles)
.withOutputFiles(renderedOutputFiles.isEmpty() ? null : renderedOutputFiles)
.withCommands(
ScriptService.scriptCommands(
List.of("/bin/sh", "-c"),
Optional.ofNullable(this.beforeCommands).map(throwFunction(runContext::render)).orElse(null),
renderedBeforeCommands.isEmpty() ? null : renderedBeforeCommands,
runContext.render(this.commands)
)
)
)
)
.run();
}

private DockerOptions injectDefaults(DockerOptions original) {
if (original == null) {
return null;
}

var builder = original.toBuilder();
if (original.getImage() == null) {
builder.image(DEFAULT_IMAGE);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.plugin.dataform.cli;

import io.kestra.core.models.property.Property;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.utils.IdUtils;
Expand Down Expand Up @@ -44,10 +45,10 @@ void run() throws Exception {

runner = dataformBuilder
.env(Map.of("{{ inputs.environmentKey }}", "{{ inputs.environmentValue }}"))
.beforeCommands(List.of(
.beforeCommands(Property.of(List.of(
"dataform init postgres new_project",
"cd new_project"
))
)))
.commands(List.of(
"echo \"::{\\\"outputs\\\":{" +
"\\\"customEnv\\\":\\\"$" + environmentKey + "\\\"" +
Expand Down
Loading