diff --git a/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java b/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java index e8b11db9..a417db1a 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.NoArgsConstructor; @@ -13,6 +14,5 @@ public abstract class AbstractCloningTask extends AbstractGitTask { @Schema( title = "Whether to clone submodules." ) - @PluginProperty - protected Boolean cloneSubmodules; + protected Property cloneSubmodules; } diff --git a/src/main/java/io/kestra/plugin/git/AbstractGitTask.java b/src/main/java/io/kestra/plugin/git/AbstractGitTask.java index 4f7916ac..821e8a24 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractGitTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractGitTask.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.Task; import io.kestra.core.runners.RunContext; import io.kestra.plugin.git.services.SshTransportConfigCallback; @@ -23,63 +24,52 @@ public abstract class AbstractGitTask extends Task { @Schema( title = "The URI to clone from." ) - @PluginProperty(dynamic = true) - protected String url; + protected Property url; @Schema( title = "The username or organization." ) - @PluginProperty(dynamic = true) - protected String username; + protected Property username; @Schema( title = "The password or Personal Access Token (PAT). When you authenticate the task with a PAT, any flows or files pushed to Git from Kestra will be pushed from the user associated with that PAT. This way, you don't need to configure the commit author (the `authorName` and `authorEmail` properties)." ) - @PluginProperty(dynamic = true) - protected String password; + protected Property password; @Schema( title = "PEM-format private key content that is paired with a public key registered on Git.", description = "To generate an ECDSA PEM format key from OpenSSH, use the following command: `ssh-keygen -t ecdsa -b 256 -m PEM`. " + "You can then set this property with your private key content and put your public key on Git." ) - @PluginProperty(dynamic = true) - protected String privateKey; + protected Property privateKey; @Schema( title = "The passphrase for the `privateKey`." ) - @PluginProperty(dynamic = true) - protected String passphrase; + protected Property passphrase; @Schema( title = "The initial Git branch." ) - @PluginProperty(dynamic = true) - public abstract String getBranch(); + public abstract Property getBranch(); public > T authentified(T command, RunContext runContext) throws Exception { if (this.username != null && this.password != null) { command.setCredentialsProvider(new UsernamePasswordCredentialsProvider( - runContext.render(this.username), - runContext.render(this.password) + runContext.render(this.username).as(String.class).orElseThrow(), + runContext.render(this.password).as(String.class).orElseThrow() )); } if (this.privateKey != null) { command.setTransportConfigCallback(new SshTransportConfigCallback( - runContext.render(this.privateKey).getBytes(StandardCharsets.UTF_8), - runContext.render(this.passphrase) + runContext.render(this.privateKey).as(String.class).orElseThrow().getBytes(StandardCharsets.UTF_8), + runContext.render(this.passphrase).as(String.class).orElse(null) )); } return command; } - protected void detectPasswordLeaks() { - if (this.password != null && !PEBBLE_TEMPLATE_PATTERN.matcher(this.password).find()) { - throw new IllegalArgumentException("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables)."); - } - } } diff --git a/src/main/java/io/kestra/plugin/git/AbstractPushTask.java b/src/main/java/io/kestra/plugin/git/AbstractPushTask.java index 170d5e66..3bff3a14 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractPushTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractPushTask.java @@ -1,7 +1,7 @@ package io.kestra.plugin.git; import io.kestra.core.exceptions.IllegalVariableEvaluationException; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -46,41 +46,37 @@ @NoArgsConstructor @Getter public abstract class AbstractPushTask extends AbstractCloningTask implements RunnableTask { - @PluginProperty(dynamic = true) - protected String commitMessage; + protected Property commitMessage; @Schema( title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately." ) - @PluginProperty @Builder.Default - private boolean dryRun = false; + private Property dryRun = Property.of(false); @Schema( title = "The commit author email.", description = "If null, no author will be set on this commit." ) - @PluginProperty(dynamic = true) - private String authorEmail; + private Property authorEmail; @Schema( title = "The commit author name.", description = "If null, the username will be used instead.", defaultValue = "`username`" ) - @PluginProperty(dynamic = true) - private String authorName; + private Property authorName; - public abstract String getCommitMessage(); + public abstract Property getCommitMessage(); - public abstract String getGitDirectory(); + public abstract Property getGitDirectory(); public abstract Object globs(); - public abstract String fetchedNamespace(); + public abstract Property fetchedNamespace(); private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException { - Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()))); + Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElse(null))); flowDirectory.toFile().mkdirs(); return flowDirectory; } @@ -130,14 +126,16 @@ protected void writeResourceFile(Path path, InputStream inputStream) throws IOEx Files.copy(inputStream, path, REPLACE_EXISTING); } - private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException { + private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException, IllegalVariableEvaluationException { File diffFile = runContext.workingDir().createTempFile(".ion").toFile(); + boolean dryRunValue = runContext.render(this.dryRun).as(Boolean.class).orElseThrow(); + try (DiffFormatter diffFormatter = new DiffFormatter(null); BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) { diffFormatter.setRepository(git.getRepository()); DiffCommand diff = git.diff(); - if (this.dryRun) { + if (dryRunValue) { diff = diff.setCached(true); } else { diff = diff.setOldTree(treeIterator(git, "HEAD~1")) @@ -204,21 +202,21 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw String commitId = null; ObjectId commit; try { - String httpUrl = gitService.getHttpUrl(runContext.render(this.url)); - if (this.isDryRun()) { + String httpUrl = gitService.getHttpUrl(runContext.render(this.url).as(String.class).orElse(null)); + if (runContext.render(this.getDryRun()).as(Boolean.class).orElseThrow()) { logger.info( "Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false", httpUrl ); } else { - String renderedBranch = runContext.render(this.getBranch()); + String renderedBranch = runContext.render(this.getBranch()).as(String.class).orElse(null); logger.info( "Pushing to {} on branch {}", httpUrl, renderedBranch ); - String message = runContext.render(this.getCommitMessage()); + String message = runContext.render(this.getCommitMessage()).as(String.class).orElse(null); ObjectId head = git.getRepository().resolve(Constants.HEAD); commit = git.commit() .setAllowEmpty(false) @@ -247,13 +245,13 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw } private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException { - String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username)); - String authorEmail = this.authorEmail; + String name = runContext.render(this.authorName).as(String.class).orElse(runContext.render(this.username).as(String.class).orElse(null)); + String authorEmail = runContext.render(this.authorEmail).as(String.class).orElse(null); if (authorEmail == null || name == null) { return null; } - return new PersonIdent(runContext.render(name), runContext.render(authorEmail)); + return new PersonIdent(name, authorEmail); } private String buildCommitUrl(String httpUrl, String branch, String commitId) { @@ -274,9 +272,8 @@ public O run(RunContext runContext) throws Exception { GitService gitService = new GitService(this); gitService.namespaceAccessGuard(runContext, this.fetchedNamespace()); - this.detectPasswordLeaks(); - Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules); + Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()).as(String.class).orElse(null), this.cloneSubmodules); Path localGitDirectory = this.createGitDirectory(runContext); @@ -291,7 +288,7 @@ public O run(RunContext runContext) throws Exception { this.writeResourceFiles(contentByPath); AddCommand add = git.add(); - add.addFilepattern(runContext.render(this.getGitDirectory())); + add.addFilepattern(runContext.render(this.getGitDirectory()).as(String.class).orElse(null)); add.call(); Output pushOutput = this.push(git, runContext, gitService); diff --git a/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java b/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java index d280e274..f1aea45f 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java @@ -1,7 +1,7 @@ package io.kestra.plugin.git; import io.kestra.core.exceptions.IllegalVariableEvaluationException; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -43,24 +43,23 @@ public abstract class AbstractSyncTask ext @Schema( title = "If `true`, the task will only output modifications without performing any modification to Kestra. If `false` (default), all listed modifications will be applied." ) - @PluginProperty @Builder.Default - private boolean dryRun = false; + private Property dryRun = Property.of(false); - public abstract boolean isDelete(); + public abstract Property getDelete(); - public abstract String getGitDirectory(); + public abstract Property getGitDirectory(); - public abstract String fetchedNamespace(); + public abstract Property fetchedNamespace(); private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException { - Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()))); + Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElse(null))); syncDirectory.toFile().mkdirs(); return syncDirectory; } - protected Map> gitResourcesContentByUri(Path baseDirectory) throws IOException { - try (Stream paths = Files.walk(baseDirectory, this.traverseDirectories() ? MAX_VALUE : 1)) { + protected Map> gitResourcesContentByUri(Path baseDirectory, RunContext runContext) throws IOException, IllegalVariableEvaluationException { + try (Stream paths = Files.walk(baseDirectory, runContext.render(this.traverseDirectories()).as(Boolean.class).orElseThrow() ? MAX_VALUE : 1)) { Stream filtered = paths.skip(1); KestraIgnore kestraIgnore = new KestraIgnore(baseDirectory); filtered = filtered.filter(path -> !kestraIgnore.isIgnoredFile(path.toString(), true)); @@ -78,10 +77,10 @@ protected Map> gitResourcesContentByUri(Path baseDire } } - protected boolean traverseDirectories() { - return true; + protected Property traverseDirectories() { + return Property.of(true); } - + protected boolean mustKeep(RunContext runContext, T instanceResource) { return false; } @@ -100,7 +99,7 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map< try (BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) { List syncResults = new ArrayList<>(); - String renderedGitDirectory = runContext.render(this.getGitDirectory()); + String renderedGitDirectory = runContext.render(this.getGitDirectory()).as(String.class).orElse(null); if (deletedResources != null) { deletedResources.stream() .map(throwFunction(deletedResource -> wrapper( @@ -146,17 +145,16 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map< } public O run(RunContext runContext) throws Exception { - this.detectPasswordLeaks(); GitService gitService = new GitService(this); gitService.namespaceAccessGuard(runContext, this.fetchedNamespace()); - Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules); + Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()).as(String.class).orElse(null), this.cloneSubmodules); Path localGitDirectory = this.createGitDirectory(runContext); - Map> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory); + Map> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory, runContext); - String renderedNamespace = runContext.render(this.fetchedNamespace()); + String renderedNamespace = runContext.render(this.fetchedNamespace()).as(String.class).orElse(null); Map beforeUpdateResourcesByUri = this.fetchResources(runContext, renderedNamespace) .stream() @@ -170,7 +168,7 @@ public O run(RunContext runContext) throws Exception { .map(throwFunction(e -> { InputStream inputStream = e.getValue().get(); T resource; - if (this.dryRun) { + if (runContext.render(this.dryRun).as(Boolean.class).orElseThrow()) { resource = this.simulateResourceWrite(runContext, renderedNamespace, e.getKey(), inputStream); } else { resource = this.writeResource(runContext, renderedNamespace, e.getKey(), inputStream); @@ -191,14 +189,14 @@ public O run(RunContext runContext) throws Exception { ); List deleted; - if (this.isDelete()) { + if (runContext.render(this.getDelete()).as(Boolean.class).orElseThrow()) { deleted = new ArrayList<>(); beforeUpdateResourcesByUri.entrySet().stream().filter(e -> !updatedResourcesByUri.containsKey(e.getKey())).forEach(throwConsumer(e -> { if (this.mustKeep(runContext, e.getValue())) { return; } - if (!this.dryRun) { + if (!runContext.render(this.dryRun).as(Boolean.class).orElseThrow()) { this.deleteResource(runContext, renderedNamespace, e.getValue()); } @@ -215,7 +213,7 @@ public O run(RunContext runContext) throws Exception { return output(diffFileStorageUri); } - protected abstract List fetchResources(RunContext runContext, String renderedNamespace) throws IOException; + protected abstract List fetchResources(RunContext runContext, String renderedNamespace) throws IOException, IllegalVariableEvaluationException; protected abstract URI toUri(String renderedNamespace, T resource); diff --git a/src/main/java/io/kestra/plugin/git/Clone.java b/src/main/java/io/kestra/plugin/git/Clone.java index 5b2233ee..a73554ee 100644 --- a/src/main/java/io/kestra/plugin/git/Clone.java +++ b/src/main/java/io/kestra/plugin/git/Clone.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -78,7 +79,7 @@ code = """ id: git_python namespace: company.team - + tasks: - id: file_system type: io.kestra.plugin.core.flow.WorkingDirectory @@ -102,10 +103,9 @@ public class Clone extends AbstractCloningTask implements RunnableTask directory; - private String branch; + private Property branch; @Schema( title = "Creates a shallow clone with a history truncated to the specified number of commits." @@ -118,11 +118,11 @@ public class Clone extends AbstractCloningTask implements RunnableTask getUrl() { return super.getUrl(); } diff --git a/src/main/java/io/kestra/plugin/git/Push.java b/src/main/java/io/kestra/plugin/git/Push.java index b7ca2137..f86a1c80 100644 --- a/src/main/java/io/kestra/plugin/git/Push.java +++ b/src/main/java/io/kestra/plugin/git/Push.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.InputFilesInterface; import io.kestra.core.models.tasks.NamespaceFiles; import io.kestra.core.models.tasks.NamespaceFilesInterface; @@ -125,22 +126,20 @@ public class Push extends AbstractCloningTask implements RunnableTask directory; @Schema( title = "The branch to which files should be committed and pushed.", description = "If the branch doesn't exist yet, it will be created." ) @NotNull - private String branch; + private Property branch; @Schema( title = "Commit message." ) - @PluginProperty(dynamic = true) @NotNull - private String commitMessage; + private Property commitMessage; private NamespaceFiles namespaceFiles; @@ -157,9 +156,8 @@ public class Push extends AbstractCloningTask implements RunnableTask addFilesPattern = List.of("."); + private Property> addFilesPattern = Property.of(List.of(".")); @Schema(title = "Commit author.") @PluginProperty @@ -167,13 +165,13 @@ public class Push extends AbstractCloningTask implements RunnableTask ref.getName().equals(R_HEADS + branch)); } } - return authentified(Git.lsRemoteRepository().setRemote(runContext.render(url)), runContext) + return authentified(Git.lsRemoteRepository().setRemote(runContext.render(url).as(String.class).orElse(null)), runContext) .callAsMap() .containsKey(R_HEADS + branch); } @@ -184,10 +182,10 @@ public Output run(RunContext runContext) throws Exception { Path basePath = runContext.workingDir().path(); if (this.directory != null) { - basePath = runContext.workingDir().resolve(Path.of(runContext.render(this.directory))); + basePath = runContext.workingDir().resolve(Path.of(runContext.render(this.directory).as(String.class).orElseThrow())); } - String branch = runContext.render(this.branch); + String branch = runContext.render(this.branch).as(String.class).orElse(null); if (this.url != null) { boolean branchExists = branchExists(runContext, branch); @@ -204,7 +202,7 @@ public Output run(RunContext runContext) throws Exception { if (branchExists) { cloneHead.toBuilder() - .branch(branch) + .branch(Property.of(branch)) .build() .run(runContext); } else { @@ -255,7 +253,7 @@ public Output run(RunContext runContext) throws Exception { })); } - if (Boolean.TRUE.equals(this.flows.enabled)) { + if (Boolean.TRUE.equals(runContext.render(this.flows.enabled).as(Boolean.class).orElse(true))) { Map flowProps = Optional.ofNullable((Map) runContext.getVariables().get("flow")).orElse(Collections.emptyMap()); String tenantId = flowProps.get("tenantId"); @@ -264,7 +262,7 @@ public Output run(RunContext runContext) throws Exception { FlowRepositoryInterface flowRepository = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowRepositoryInterface.class); List flows; - if (Boolean.TRUE.equals(this.flows.childNamespaces)) { + if (Boolean.TRUE.equals(runContext.render(this.flows.childNamespaces).as(Boolean.class).orElse(true))) { flows = flowRepository.findWithSource(null, tenantId, null, namespace, null); } else { flows = flowRepository.findByNamespaceWithSource(tenantId, namespace); @@ -272,7 +270,7 @@ public Output run(RunContext runContext) throws Exception { Path flowsDirectory = this.flows.gitDirectory == null ? basePath - : basePath.resolve(runContext.render(this.flows.gitDirectory)); + : basePath.resolve(runContext.render(this.flows.gitDirectory).as(String.class).orElse(null)); // Create flow directory if it doesn't exist flowsDirectory.toFile().mkdirs(); @@ -291,14 +289,14 @@ public Output run(RunContext runContext) throws Exception { ); AddCommand add = git.add(); - runContext.render(this.addFilesPattern).forEach(add::addFilepattern); + runContext.render(this.addFilesPattern).asList(String.class).forEach(add::addFilepattern); add.call(); ObjectId commitId = null; try { commitId = git.commit() .setAllowEmpty(false) - .setMessage(runContext.render(this.commitMessage)) + .setMessage(runContext.render(this.commitMessage).as(String.class).orElse(null)) .setAuthor(author(runContext)) .call() .getId(); @@ -322,10 +320,16 @@ private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluati return null; } if (this.author.email != null && this.author.name != null) { - return new PersonIdent(runContext.render(this.author.name), runContext.render(this.author.email)); + return new PersonIdent( + runContext.render(this.author.name).as(String.class).orElseThrow(), + runContext.render(this.author.email).as(String.class).orElseThrow() + ); } if (this.author.email != null && this.username != null) { - return new PersonIdent(runContext.render(this.username), runContext.render(this.author.email)); + return new PersonIdent( + runContext.render(this.username).as(String.class).orElseThrow(), + runContext.render(this.author.email).as(String.class).orElseThrow() + ); } return null; @@ -350,36 +354,31 @@ public static class FlowFiles { @Schema( title = "Whether to push flows as YAML files to Git." ) - @PluginProperty @Builder.Default @JsonInclude(JsonInclude.Include.NON_NULL) - private Boolean enabled = true; + private Property enabled = Property.of(true); @Schema( title = "Whether flows from child namespaces should be included." ) - @PluginProperty @Builder.Default @JsonInclude(JsonInclude.Include.NON_NULL) - private Boolean childNamespaces = true; + private Property childNamespaces = Property.of(true); @Schema( title = "To which directory flows should be pushed (relative to `directory`)." ) - @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_flows"; + private Property gitDirectory = Property.of("_flows"); } @Builder @Getter public static class Author { @Schema(title = "The commit author name, if null the username will be used instead") - @PluginProperty(dynamic = true) - private String name; + private Property name; @Schema(title = "The commit author email, if null no author will be set on this commit") - @PluginProperty(dynamic = true) - private String email; + private Property email; } } diff --git a/src/main/java/io/kestra/plugin/git/PushFlows.java b/src/main/java/io/kestra/plugin/git/PushFlows.java index cd35e702..165963d9 100644 --- a/src/main/java/io/kestra/plugin/git/PushFlows.java +++ b/src/main/java/io/kestra/plugin/git/PushFlows.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; @@ -56,7 +57,7 @@ Using this task, you can push one or more flows from a given namespace (and opti type: io.kestra.plugin.git.PushFlows sourceNamespace: "{{ taskrun.value }}" gitDirectory: "{{'flows/' ~ taskrun.value}}" - includeChildNamespaces: false + includeChildNamespaces: false - id: scripts type: io.kestra.plugin.git.PushNamespaceFiles @@ -71,20 +72,20 @@ Using this task, you can push one or more flows from a given namespace (and opti password: "{{ secret('GITHUB_ACCESS_TOKEN') }}" branch: main dryRun: false - + triggers: - id: schedule_push_to_git type: io.kestra.plugin.core.trigger.Schedule cron: "0 11 * * 4" """ - ), + ), @Example( title = "Automatically push all saved flows from the dev namespace and all child namespaces to a Git repository every day at 5 p.m. Before pushing to Git, the task will adjust the flow's source code to match the targetNamespace to prepare the Git branch for merging to the production namespace. Note that the automatic conversion of `sourceNamespace` to `targetNamespace` is optional and should only be considered as a helper for facilitating the Git workflow for simple use cases — only the `namespace` property within the flow will be adjusted and if you specify namespace names within e.g. Flow triggers, those may need to be manually adjusted. **We recommend using separate Kestra instances for development and production with the same namespace names across instances.**", full = true, code = """ id: push_to_git namespace: system - + tasks: - id: commit_and_push type: io.kestra.plugin.git.PushFlows @@ -99,7 +100,7 @@ Using this task, you can push one or more flows from a given namespace (and opti branch: main commitMessage: "add flows {{ now() }}" # optional string dryRun: true # if true, you'll see what files will be added, modified or deleted based on the state in Git without overwriting the files yet - + triggers: - id: schedule_push type: io.kestra.plugin.core.trigger.Schedule @@ -112,12 +113,12 @@ Using this task, you can push one or more flows from a given namespace (and opti code = """ id: myflow namespace: prod - + inputs: - id: push type: BOOLEAN defaults: false - + tasks: - id: if type: io.kestra.plugin.core.flow.If @@ -144,7 +145,7 @@ public class PushFlows extends AbstractPushTask { ) @PluginProperty(dynamic = true) @Builder.Default - private String branch = "main"; + private Property branch = Property.of("main"); @Schema( title = "Directory to which flows should be pushed.", @@ -154,23 +155,20 @@ public class PushFlows extends AbstractPushTask { If the `includeChildNamespaces` property is set to true, this task will also push all flows from child namespaces into their corresponding nested directories, e.g., flows from the child namespace called prod.marketing will be added to the marketing folder within the _flows folder. Note that the targetNamespace (here prod) is specified in the flow code; therefore, kestra will not create the prod directory within _flows. You can use the PushFlows task to push flows from the sourceNamespace, and use SyncFlows to then sync PR-approved flows to the targetNamespace, including all child namespaces.""" ) - @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_flows"; + private Property gitDirectory = Property.of("_flows"); @Schema( title = "The source namespace from which flows should be synced to the `gitDirectory`." ) - @PluginProperty(dynamic = true) @Builder.Default - private String sourceNamespace = "{{ flow.namespace }}"; + private Property sourceNamespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "The target namespace, intended as the production namespace.", description = "If set, the `sourceNamespace` will be overwritten to the `targetNamespace` in the flow source code to prepare your branch for merging into the production namespace." ) - @PluginProperty(dynamic = true) - private String targetNamespace; + private Property targetNamespace; @Schema( title = "List of glob patterns or a single one that declare which flows should be included in the Git commit.", @@ -189,7 +187,7 @@ By default, all flows from the specified sourceNamespace will be pushed (and opt title = "Whether you want to push flows from child namespaces as well.", description = """ By default, it’s `false`, so the task will push only flows from the explicitly declared namespace without pushing flows from child namespaces. If set to `true`, flows from child namespaces will be pushed to child directories in Git. See the example below for a practical explanation: - + | Source namespace in the flow code | Git directory path | Synced to target namespace | | --------------------------------- | ------------------------------ | ----------------------------- | | namespace: dev | _flows/flow1.yml | namespace: prod | @@ -199,17 +197,16 @@ By default, all flows from the specified sourceNamespace will be pushed (and opt | namespace: dev.marketing.crm | _flows/marketing/crm/flow5.yml | namespace: prod.marketing.crm | | namespace: dev.marketing.crm | _flows/marketing/crm/flow6.yml | namespace: prod.marketing.crm |""" ) - @PluginProperty @Builder.Default - private boolean includeChildNamespaces = false; + private Property includeChildNamespaces = Property.of(false); @Schema( title = "Git commit message.", defaultValue = "Add flows from sourceNamespace" ) @Override - public String getCommitMessage() { - return Optional.ofNullable(this.commitMessage).orElse("Add flows from " + this.sourceNamespace + " namespace"); + public Property getCommitMessage() { + return Optional.ofNullable(this.commitMessage).orElse(new Property<>("Add flows from " + this.sourceNamespace.toString() + " namespace")); } @Override @@ -218,7 +215,7 @@ public Object globs() { } @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.sourceNamespace; } @@ -228,8 +225,8 @@ protected Map> instanceResourcesContentByPath(RunCon Map flowProps = Optional.ofNullable((Map) runContext.getVariables().get("flow")).orElse(Collections.emptyMap()); String tenantId = flowProps.get("tenantId"); List flowsToPush; - String renderedSourceNamespace = runContext.render(this.sourceNamespace); - if (Boolean.TRUE.equals(this.includeChildNamespaces)) { + String renderedSourceNamespace = runContext.render(this.sourceNamespace).as(String.class).orElse(null); + if (Boolean.TRUE.equals(runContext.render(this.includeChildNamespaces).as(Boolean.class).orElseThrow())) { flowsToPush = flowRepository.findWithSource(null, tenantId, null, renderedSourceNamespace, null); } else { flowsToPush = flowRepository.findByNamespaceWithSource(tenantId, renderedSourceNamespace); @@ -244,7 +241,7 @@ protected Map> instanceResourcesContentByPath(RunCon }); } - Map> flowSourceByPath = filteredFlowsToPush.collect(Collectors.toMap(flowWithSource -> { + return filteredFlowsToPush.collect(Collectors.toMap(flowWithSource -> { Path path = flowDirectory; if (flowWithSource.getNamespace().length() > renderedSourceNamespace.length()) { path = path.resolve(flowWithSource.getNamespace().substring(renderedSourceNamespace.length() + 1).replace(".", "/")); @@ -252,10 +249,11 @@ protected Map> instanceResourcesContentByPath(RunCon return path.resolve(flowWithSource.getId() + ".yml"); }, throwFunction(flowWithSource -> (throwSupplier(() -> { - String modifiedSource = flowWithSource.getSource().replaceAll("(?m)^(\\s*namespace:\\s*)" + runContext.render(sourceNamespace), "$1" + runContext.render(targetNamespace)); + String modifiedSource = flowWithSource.getSource().replaceAll( + "(?m)^(\\s*namespace:\\s*)" + runContext.render(sourceNamespace).as(String.class).orElse(null), + "$1" + runContext.render(targetNamespace).as(String.class).orElse(null)); return new ByteArrayInputStream(modifiedSource.getBytes()); }))))); - return flowSourceByPath; } @Override diff --git a/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java b/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java index e1eded90..967f96c7 100644 --- a/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java +++ b/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.storages.Namespace; import io.kestra.core.utils.PathMatcherPredicate; @@ -51,7 +52,7 @@ type: io.kestra.plugin.git.PushFlows sourceNamespace: "{{ taskrun.value }}" gitDirectory: "{{'flows/' ~ taskrun.value}}" - includeChildNamespaces: false + includeChildNamespaces: false - id: scripts type: io.kestra.plugin.git.PushNamespaceFiles @@ -66,20 +67,20 @@ password: "{{ secret('GITHUB_ACCESS_TOKEN') }}" branch: main dryRun: false - + triggers: - id: schedule_push_to_git type: io.kestra.plugin.core.trigger.Schedule cron: "0 11 * * 4" """ - ), + ), @Example( title = "Push all saved Namespace Files from the dev namespace to a Git repository every 15 minutes.", full = true, code = """ id: push_to_git namespace: system - + tasks: - id: commit_and_push type: io.kestra.plugin.git.PushNamespaceFiles @@ -92,7 +93,7 @@ branch: dev # optional, uses "kestra" by default commitMessage: "add namespace files" # optional string dryRun: true # if true, you'll see what files will be added, modified or deleted based on the state in Git without overwriting the files yet - + triggers: - id: schedule_push_to_git type: io.kestra.plugin.core.trigger.Schedule @@ -108,20 +109,19 @@ public class PushNamespaceFiles extends AbstractPushTask branch = Property.of("main"); @Schema( title = "The namespace from which files should be pushed to the `gitDirectory`." ) - @PluginProperty(dynamic = true) @Builder.Default - private String namespace = "{{ flow.namespace }}"; + private Property namespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "Directory to which Namespace Files should be pushed.", description = """ If not set, files will be pushed to a Git directory named _files. See the table below for an example mapping of Namespace Files to Git paths: - + | Namespace File Path | Git directory path | | --------------------- | ---------------------------- | | scripts/app.py | _files/scripts/app.py | @@ -130,9 +130,8 @@ public class PushNamespaceFiles extends AbstractPushTask gitDirectory = Property.of("_files"); @Schema( title = "Which Namespace Files should be included in the commit.", @@ -152,8 +151,8 @@ Given that this is a glob pattern string (or a list of glob patterns), you can i defaultValue = "Add files from `namespace` namespace" ) @Override - public String getCommitMessage() { - return Optional.ofNullable(this.commitMessage).orElse("Add files from " + this.namespace + " namespace"); + public Property getCommitMessage() { + return Optional.ofNullable(this.commitMessage).orElse(new Property<>("Add files from " + this.namespace.toString() + " namespace")); } @Override @@ -162,14 +161,14 @@ public Object globs() { } @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.namespace; } @Override protected Map> instanceResourcesContentByPath(RunContext runContext, Path baseDirectory, List globs) throws Exception { - Namespace storage = runContext.storage().namespace(runContext.render(this.namespace)); + Namespace storage = runContext.storage().namespace(runContext.render(this.namespace).as(String.class).orElse(null)); Predicate matcher = (globs != null) ? PathMatcherPredicate.matches(globs) : (path -> true); return storage diff --git a/src/main/java/io/kestra/plugin/git/Sync.java b/src/main/java/io/kestra/plugin/git/Sync.java index c35c10ef..9eb41faa 100644 --- a/src/main/java/io/kestra/plugin/git/Sync.java +++ b/src/main/java/io/kestra/plugin/git/Sync.java @@ -4,6 +4,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.models.tasks.VoidOutput; import io.kestra.core.repositories.FlowRepositoryInterface; @@ -96,7 +97,7 @@ public class Sync extends AbstractCloningTask implements RunnableTask branch; @Schema( title = "If true, the task will only display modifications without syncing any files yet. If false (default), all namespace files and flows will be overwritten based on the state in Git." @@ -284,7 +285,7 @@ private static void logUpdate(Logger logger, String path) { @Override @NotNull - public String getUrl() { + public Property getUrl() { return super.getUrl(); } } diff --git a/src/main/java/io/kestra/plugin/git/SyncFlows.java b/src/main/java/io/kestra/plugin/git/SyncFlows.java index 03de1ea9..4c01e53b 100644 --- a/src/main/java/io/kestra/plugin/git/SyncFlows.java +++ b/src/main/java/io/kestra/plugin/git/SyncFlows.java @@ -1,9 +1,11 @@ package io.kestra.plugin.git; +import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; import io.kestra.core.services.FlowService; @@ -106,9 +108,8 @@ public class SyncFlows extends AbstractSyncTask { @Schema( title = "The branch from which flows will be synced to Kestra." ) - @PluginProperty(dynamic = true) @Builder.Default - private String branch = "main"; + private Property branch = Property.of("main"); @Schema( title = "The target namespace to which flows from the `gitDirectory` should be synced.", @@ -131,9 +132,8 @@ public class SyncFlows extends AbstractSyncTask { | namespace: dev.marketing.crm | _flows/marketing/crm/flow6.yml | namespace: prod.marketing.crm | """ ) - @PluginProperty(dynamic = true) @NotNull - private String targetNamespace; + private Property targetNamespace; @Schema( title = "Directory from which flows should be synced.", @@ -146,25 +146,22 @@ public class SyncFlows extends AbstractSyncTask { - flows from the `_flows/marketing` subdirectory in Git will be synced to the `prod.marketing` namespace, - flows from the `_flows/marketing/crm` subdirectory will be synced to the `prod.marketing.crm` namespace.""" ) - @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_flows"; + private Property gitDirectory = Property.of("_flows"); @Schema( title = "Whether you want to sync flows from child namespaces as well.", description = "It’s `false` by default so that we sync only flows from the explicitly declared `gitDirectory` without traversing child directories. If set to `true`, flows from subdirectories in Git will be synced to child namespace in Kestra using the dot notation `.` for each subdirectory in the folder structure." ) - @PluginProperty(dynamic = true) @Builder.Default - private boolean includeChildNamespaces = false; + private Property includeChildNamespaces = Property.of(false); @Schema( title = "Whether you want to delete flows present in kestra but not present in Git.", description = "It’s `false` by default to avoid destructive behavior. Use this property with caution because when set to `true` and `includeChildNamespaces` is also set to `true`, this task will delete all flows from the `targetNamespace` and all its child namespaces that are not present in Git rather than only overwriting the changes." ) - @PluginProperty @Builder.Default - private boolean delete = false; + private Property delete = Property.of(false); @Getter(AccessLevel.NONE) private FlowService flowService; @@ -179,7 +176,7 @@ private FlowService flowService(RunContext runContext) { @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.targetNamespace; } @@ -206,7 +203,7 @@ protected boolean mustKeep(RunContext runContext, Flow instanceResource) { } @Override - protected boolean traverseDirectories() { + protected Property traverseDirectories() { return this.includeChildNamespaces; } @@ -262,8 +259,8 @@ protected SyncResult wrapper(RunContext runContext, String renderedGitDirectory, } @Override - protected List fetchResources(RunContext runContext, String renderedNamespace) { - if (this.includeChildNamespaces) { + protected List fetchResources(RunContext runContext, String renderedNamespace) throws IllegalVariableEvaluationException { + if (runContext.render(this.includeChildNamespaces).as(Boolean.class).orElseThrow()) { return flowService(runContext).findByNamespacePrefix(runContext.flowInfo().tenantId(), renderedNamespace); } diff --git a/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java b/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java index 7824d69f..d621729c 100644 --- a/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java +++ b/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.storages.Namespace; import io.kestra.core.storages.NamespaceFile; @@ -48,7 +49,7 @@ type: io.kestra.plugin.git.SyncFlows targetNamespace: "{{ taskrun.value }}" gitDirectory: "{{'flows/' ~ taskrun.value}}" - includeChildNamespaces: false + includeChildNamespaces: false - id: scripts type: io.kestra.plugin.git.SyncNamespaceFiles @@ -69,14 +70,14 @@ type: io.kestra.plugin.core.trigger.Schedule cron: "0 * * * *" """ - ), + ), @Example( title = "Sync Namespace Files from a Git repository. This flow can run either on a schedule (using the Schedule trigger) or anytime you push a change to a given Git branch (using the Webhook trigger).", full = true, code = """ id: sync_from_git namespace: system - + tasks: - id: git type: io.kestra.plugin.git.SyncNamespaceFiles @@ -88,7 +89,7 @@ username: git_username password: "{{ secret('GITHUB_ACCESS_TOKEN') }}" dryRun: true # if true, the task will only log which flows from Git will be added/modified or deleted in kestra without making any changes in kestra backend yet - + triggers: - id: every_minute type: io.kestra.plugin.core.trigger.Schedule @@ -101,35 +102,31 @@ public class SyncNamespaceFiles extends AbstractSyncTask branch = Property.of("main"); @Schema( title = "The namespace from which files should be synced from the `gitDirectory` to Kestra." ) - @PluginProperty(dynamic = true) @Builder.Default - private String namespace = "{{ flow.namespace }}"; + private Property namespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "Directory from which Namespace Files should be synced.", description = "If not set, this task assumes your branch includes a directory named `_files`" ) - @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_files"; + private Property gitDirectory = Property.of("_files"); @Schema( title = "Whether you want to delete Namespace Files present in kestra but not present in Git.", description = "It’s `false` by default to avoid destructive behavior. Use with caution because when set to `true`, this task will delete all Namespace Files which are not present in Git." ) - @PluginProperty @Builder.Default - private boolean delete = false; + private Property delete = Property.of(false); @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.namespace; } diff --git a/src/main/java/io/kestra/plugin/git/services/GitService.java b/src/main/java/io/kestra/plugin/git/services/GitService.java index 63e02b5d..d6e268cf 100644 --- a/src/main/java/io/kestra/plugin/git/services/GitService.java +++ b/src/main/java/io/kestra/plugin/git/services/GitService.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git.services; import io.kestra.core.exceptions.IllegalVariableEvaluationException; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; import io.kestra.core.services.FlowService; @@ -21,7 +22,7 @@ public class GitService { private AbstractGitTask gitTask; - public Git cloneBranch(RunContext runContext, String branch, Boolean withSubmodules) throws Exception { + public Git cloneBranch(RunContext runContext, String branch, Property withSubmodules) throws Exception { Clone cloneHead = Clone.builder() .url(gitTask.getUrl()) .username(gitTask.getUsername()) @@ -34,7 +35,7 @@ public Git cloneBranch(RunContext runContext, String branch, Boolean withSubmodu boolean branchExists = this.branchExists(runContext, branch); if (branchExists) { cloneHead.toBuilder() - .branch(branch) + .branch(Property.of(branch)) .build() .run(runContext); } else { @@ -56,7 +57,7 @@ public Git cloneBranch(RunContext runContext, String branch, Boolean withSubmodu } public boolean branchExists(RunContext runContext, String branch) throws Exception { - return gitTask.authentified(Git.lsRemoteRepository().setRemote(runContext.render(gitTask.getUrl())), runContext) + return gitTask.authentified(Git.lsRemoteRepository().setRemote(runContext.render(gitTask.getUrl()).as(String.class).orElse(null)), runContext) .callAsMap() .containsKey(R_HEADS + branch); } @@ -81,12 +82,12 @@ public String getHttpUrl(String gitUrl) { return httpUrl; } - public void namespaceAccessGuard(RunContext runContext, String namespaceToAccess) throws IllegalVariableEvaluationException { + public void namespaceAccessGuard(RunContext runContext, Property namespaceToAccess) throws IllegalVariableEvaluationException { FlowService flowService = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowService.class); RunContext.FlowInfo flowInfo = runContext.flowInfo(); flowService.checkAllowedNamespace( runContext.flowInfo().tenantId(), - runContext.render(namespaceToAccess), + runContext.render(namespaceToAccess).as(String.class).orElse(null), flowInfo.tenantId(), flowInfo.namespace() ); diff --git a/src/test/java/io/kestra/plugin/git/CloneTest.java b/src/test/java/io/kestra/plugin/git/CloneTest.java index cd8e5725..c38c69a4 100644 --- a/src/test/java/io/kestra/plugin/git/CloneTest.java +++ b/src/test/java/io/kestra/plugin/git/CloneTest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.git; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.micronaut.context.annotation.Value; @@ -25,7 +26,7 @@ void publicRepository() throws Exception { RunContext runContext = runContextFactory.of(); Clone task = Clone.builder() - .url("https://github.com/kestra-io/plugin-template") + .url(Property.of("https://github.com/kestra-io/plugin-template")) .build(); Clone.Output runOutput = task.run(runContext); @@ -42,9 +43,9 @@ void privateRepository() throws Exception { RunContext runContext = runContextFactory.of(); Clone task = Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) .build(); Clone.Output runOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/PushFlowsTest.java b/src/test/java/io/kestra/plugin/git/PushFlowsTest.java index b26aa2cf..a0f70e04 100644 --- a/src/test/java/io/kestra/plugin/git/PushFlowsTest.java +++ b/src/test/java/io/kestra/plugin/git/PushFlowsTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import io.kestra.core.models.flows.Flow; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; @@ -50,24 +51,6 @@ public class PushFlowsTest extends AbstractGitTest { @Inject private FlowRepositoryInterface flowRepositoryInterface; - @Test - void hardcodedPassword() { - PushFlows pushFlows = PushFlows.builder() - .id("pushFlows") - .type(PushFlows.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> pushFlows.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_SingleRegex() throws Exception { String tenantId = "my-tenant"; @@ -85,18 +68,18 @@ void defaultCase_SingleRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) .flows("second*") - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -107,10 +90,10 @@ void defaultCase_SingleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -159,19 +142,19 @@ void defaultCase_SingleRegexDryRun() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) .flows("second*") - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") - .dryRun(true) + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .dryRun(Property.of(true)) .build(); PushFlows.Output pushOutput = pushFlows.run(runContext); @@ -208,17 +191,17 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -227,10 +210,10 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -309,17 +292,17 @@ void defaultCase_NoRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -328,10 +311,10 @@ void defaultCase_NoRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -387,18 +370,18 @@ void defaultCase_MultipleRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) .flows(List.of("first*", "second*")) - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -407,10 +390,10 @@ void defaultCase_MultipleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -465,15 +448,15 @@ void defaultCase_NoRegexNoChildNsNoAuthorName() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -482,10 +465,10 @@ void defaultCase_NoRegexNoChildNsNoAuthorName() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -529,15 +512,15 @@ void defaultCase_NoRegexNoAuthor() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .sourceNamespace("{{sourceNamespace}}") - .targetNamespace("{{targetNamespace}}") - .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) + .targetNamespace(new Property<>("{{targetNamespace}}")) + .includeChildNamespaces(Property.of(true)) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -546,10 +529,10 @@ void defaultCase_NoRegexNoAuthor() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -625,9 +608,9 @@ private FlowWithSource createFlow(String tenantId, String namespace) { private FlowWithSource createFlow(String tenantId, String flowId, String namespace) { String flowSource = """ id:\s""" + flowId + """ - + namespace:\s""" + namespace + """ - + tasks: - id: my-task type: io.kestra.core.tasks.log.Log diff --git a/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java b/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java index 5785bab8..39000f08 100644 --- a/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java +++ b/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git; import com.fasterxml.jackson.core.type.TypeReference; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.serializers.JacksonMapper; @@ -45,24 +46,6 @@ public class PushNamespaceFilesTest extends AbstractGitTest { @Inject private StorageInterface storage; - @Test - void hardcodedPassword() { - PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() - .id("pushNamespaceFiles") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> pushNamespaceFiles.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_SingleRegex() throws Exception { String tenantId = "my-tenant"; @@ -81,16 +64,16 @@ void defaultCase_SingleRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files("nested/*") - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -101,10 +84,10 @@ void defaultCase_SingleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -154,17 +137,17 @@ void defaultCase_SingleRegexDryRun() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files("second*") - .gitDirectory("{{gitDirectory}}") - .dryRun(true) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .dryRun(Property.of(true)) .build(); PushNamespaceFiles.Output pushOutput = pushNamespaceFiles.run(runContext); @@ -205,15 +188,15 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -222,10 +205,10 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -306,15 +289,15 @@ void defaultCase_NoRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -323,10 +306,10 @@ void defaultCase_NoRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -385,16 +368,16 @@ void defaultCase_MultipleRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files(List.of("first*", "second*")) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -403,10 +386,10 @@ void defaultCase_MultipleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -458,13 +441,13 @@ void defaultCase_NoRegexNoAuthor() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -473,10 +456,10 @@ void defaultCase_NoRegexNoAuthor() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); diff --git a/src/test/java/io/kestra/plugin/git/PushTest.java b/src/test/java/io/kestra/plugin/git/PushTest.java index 3568091d..8a74cb34 100644 --- a/src/test/java/io/kestra/plugin/git/PushTest.java +++ b/src/test/java/io/kestra/plugin/git/PushTest.java @@ -62,10 +62,10 @@ void cloneThenPush_OnlyNeedsCredentialsForPush() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -84,15 +84,15 @@ void cloneThenPush_OnlyNeedsCredentialsForPush() throws Exception { .id("push") .type(Push.class.getName()) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .commitMessage("Push from CI - Clone then push") + .commitMessage(Property.of("Push from CI - Clone then push")) .inputFiles(Map.of( INPUT_FILE_NAME, expectedInputFileContent )) - .username(pat) - .password(pat) - .branch(BRANCH) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); Push.Output pushOutput = push.run(cloneRunContext); @@ -123,10 +123,10 @@ void cloneThenPush_PushBranchContentToAnother() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -138,12 +138,12 @@ void cloneThenPush_PushBranchContentToAnother() throws Exception { .id("push") .type(Push.class.getName()) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .commitMessage("Push from CI - Clone then push") - .username(pat) - .password(pat) - .branch(otherBranch) + .commitMessage(Property.of("Push from CI - Clone then push")) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(otherBranch)) .build(); Push.Output pushOutput = push.run(cloneRunContext); @@ -187,13 +187,13 @@ void oneTaskPush_ExistingBranch() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") + .url(new Property<>(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .username(pat) - .password(pat) + .username(new Property<>(pat)) + .password(new Property<>(pat)) .inputFiles(Map.of( INPUT_FILE_NAME, expectedInputFileContent, shouldNotBeCommitted, "should not be committed" @@ -202,11 +202,11 @@ void oneTaskPush_ExistingBranch() throws Exception { .enabled(Property.of(true)) .build() ) - .addFilesPattern(List.of( + .addFilesPattern(Property.of(List.of( INPUT_FILE_NAME, namespaceFileName - )) - .branch(BRANCH) + ))) + .branch(new Property<>(BRANCH)) .build(); push.run(runContext); @@ -214,10 +214,10 @@ void oneTaskPush_ExistingBranch() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -237,10 +237,10 @@ void oneTaskPush_NonExistingBranch() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); Assertions.assertThrows(TransportException.class, () -> clone.run(runContextFactory.of())); @@ -249,17 +249,17 @@ void oneTaskPush_NonExistingBranch() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) + .url(new Property<>(repositoryUrl)) .inputFiles(Map.of( toDeleteFileName, "some content" )) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .commitMessage("Branch creation") - .username(pat) - .password(pat) - .branch(branchName) + .commitMessage(Property.of("Branch creation")) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); push.run(runContextFactory.of()); @@ -287,27 +287,27 @@ void oneTaskPush_NoChangeShouldNotCommit() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); String toDeleteFileName = "to_delete.txt"; Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) + .url(new Property<>(repositoryUrl)) .inputFiles(Map.of( toDeleteFileName, "some content" )) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .commitMessage("Branch creation") - .username(pat) - .password(pat) - .branch(branchName) + .commitMessage(Property.of("Branch creation")) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); RunContext runContext = runContextFactory.of(); Push.Output firstPush = push.run(runContext); @@ -345,21 +345,21 @@ void oneTaskPush_WithSpecifiedDirectory() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - One-task push with specified directory") + .url(new Property<>(repositoryUrl)) + .commitMessage(Property.of("Push from CI - One-task push with specified directory")) .flows(Push.FlowFiles.builder() - .enabled(false) + .enabled(Property.of(false)) .build()) - .username(pat) - .password(pat) - .directory(directory) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .directory(Property.of(directory)) .inputFiles(Map.of( "not_included_file.txt", "not included", INPUT_FILE_NAME, "not included neither", directory + "/" + INPUT_FILE_NAME, expectedInputFileContent, directory + "/" + nestedFilePath, expectedNestedInputFileContent )) - .branch(BRANCH) + .branch(new Property<>(BRANCH)) .build(); push.run(runContext); @@ -367,10 +367,10 @@ void oneTaskPush_WithSpecifiedDirectory() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -401,11 +401,11 @@ void oneTaskPush_WithFlows() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); try { @@ -414,10 +414,10 @@ void oneTaskPush_WithFlows() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -455,18 +455,18 @@ void oneTaskPush_WithFlowsNoChildNs() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") + .url(new Property<>(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) .flows(Push.FlowFiles.builder() - .childNamespaces(false) + .childNamespaces(Property.of(false)) .build()) - .username(pat) - .password(pat) + .username(new Property<>(pat)) + .password(new Property<>(pat)) .author(Push.Author.builder() - .name(gitUserName) - .email(gitUserEmail) + .name(new Property<>(gitUserName)) + .email(new Property<>(gitUserEmail)) .build()) - .branch(branchName) + .branch(new Property<>(branchName)) .build(); try { @@ -475,10 +475,10 @@ void oneTaskPush_WithFlowsNoChildNs() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -514,16 +514,16 @@ void oneTaskPush_WithFlowsAndDirectory() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") - .username(pat) - .password(pat) + .url(new Property<>(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>(pat)) + .password(new Property<>(pat)) .author(Push.Author.builder() - .email(gitUserEmail) + .email(new Property<>(gitUserEmail)) .build()) - .branch(branchName) + .branch(new Property<>(branchName)) .flows(Push.FlowFiles.builder() - .gitDirectory("my-flows") + .gitDirectory(Property.of("my-flows")) .build()) .build(); @@ -533,10 +533,10 @@ void oneTaskPush_WithFlowsAndDirectory() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); diff --git a/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java b/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java index a8e9a167..07de0b89 100644 --- a/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java @@ -4,6 +4,7 @@ import io.kestra.core.models.executions.LogEntry; import io.kestra.core.models.flows.Flow; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueInterface; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.RunContext; @@ -63,24 +64,6 @@ void init() { }); } - @Test - void hardcodedPassword() { - SyncFlows syncFlows = SyncFlows.builder() - .id("syncFlows") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> syncFlows.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_WithDelete() throws Exception { RunContext runContext = runContext(); @@ -145,14 +128,14 @@ void defaultCase_WithDelete() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") - .delete(true) - .includeChildNamespaces(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) + .delete(Property.of(true)) + .includeChildNamespaces(Property.of(true)) .build(); SyncFlows.Output syncOutput = task.run(runContext); @@ -161,10 +144,10 @@ void defaultCase_WithDelete() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), true, selfFlowSource); @@ -239,13 +222,13 @@ void defaultCase_WithoutDelete() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") - .includeChildNamespaces(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) + .includeChildNamespaces(Property.of(true)) .build(); SyncFlows.Output syncOutput = task.run(runContext); @@ -254,10 +237,10 @@ void defaultCase_WithoutDelete() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), true, selfFlowSource, nonVersionedFlowSource); @@ -338,14 +321,14 @@ void defaultCase_WithDeleteNoChildNs() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") - .delete(true) - .includeChildNamespaces(false) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) + .delete(Property.of(true)) + .includeChildNamespaces(Property.of(false)) .build(); SyncFlows.Output syncOutput = task.run(runContext); @@ -354,10 +337,10 @@ void defaultCase_WithDeleteNoChildNs() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), false, selfFlowSource, unversionedFlowSourceInChildNamespace); @@ -435,15 +418,15 @@ void dryRun_WithDelete() throws Exception { .toArray(String[]::new); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") - .delete(true) - .includeChildNamespaces(true) - .dryRun(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) + .delete(Property.of(true)) + .includeChildNamespaces(Property.of(true)) + .dryRun(Property.of(true)) .build(); SyncFlows.Output syncOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java b/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java index 965cffe0..90382add 100644 --- a/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import io.kestra.core.junit.annotations.KestraTest; import io.kestra.core.models.executions.LogEntry; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueInterface; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; @@ -48,24 +49,6 @@ void init() throws IOException { storage.deleteByPrefix(TENANT_ID, NAMESPACE, URI.create(StorageContext.namespaceFilePrefix(NAMESPACE))); } - @Test - void hardcodedPassword() { - SyncNamespaceFiles syncNamespaceFiles = SyncNamespaceFiles.builder() - .id("syncNamespaceFiles") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> syncNamespaceFiles.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_WithDelete() throws Exception { RunContext runContext = runContext(); @@ -108,13 +91,13 @@ void defaultCase_WithDelete() throws Exception { ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") - .delete(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) + .delete(Property.of(true)) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); @@ -174,12 +157,12 @@ void defaultCase_WithoutDelete() throws Exception { ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); @@ -239,14 +222,14 @@ void defaultCase_DryRunWithDeleteFlag_ShouldStillNotifyWhatWouldBeDeleted() thro ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") - .dryRun(true) - .delete(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) + .dryRun(Property.of(true)) + .delete(Property.of(true)) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/SyncTest.java b/src/test/java/io/kestra/plugin/git/SyncTest.java index 975cf983..68baaa8a 100644 --- a/src/test/java/io/kestra/plugin/git/SyncTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncTest.java @@ -2,6 +2,7 @@ import io.kestra.core.models.executions.LogEntry; import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueFactoryInterface; import io.kestra.core.queues.QueueInterface; import io.kestra.core.repositories.FlowRepositoryInterface; @@ -172,10 +173,10 @@ void reconcileNsFilesAndFlows() throws Exception { String clonedGitDirectory = "to_clone"; String destinationDirectory = "sync_directory"; Sync task = Sync.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .gitDirectory(clonedGitDirectory) .namespaceFilesDirectory(destinationDirectory) .build(); @@ -193,10 +194,10 @@ void reconcileNsFilesAndFlows() throws Exception { RunContext runContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build() .run(runContext); assertFlows(TENANT_ID, runContext.workingDir().path().resolve(Path.of(clonedGitDirectory, "_flows")).toFile(), selfFlowSource); @@ -277,10 +278,10 @@ void reconcile_MinimumSetup() throws Exception { // region WHEN Sync task = Sync.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build(); task.run(runContextFactory.of(Map.of("flow", Map.of( @@ -297,10 +298,10 @@ void reconcile_MinimumSetup() throws Exception { RunContext runContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .build() .run(runContext); assertFlows(TENANT_ID, runContext.workingDir().path().resolve("_flows").toFile(), selfFlowSource); @@ -374,10 +375,10 @@ void reconcile_DryRun_ShouldDoNothing() throws Exception { Sync task = Sync.builder() .id("reconcile") .type(Sync.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(new Property<>(repositoryUrl)) + .username(new Property<>(pat)) + .password(new Property<>(pat)) + .branch(new Property<>(BRANCH)) .gitDirectory("to_clone") .dryRun(true) .build();