Merge branch 'main' into dependabot/gradle/com.vanniktech.maven.publish-0.36.0

Author: fdelbrayelle

.github/ISSUE_TEMPLATE/bug.yml

@@ -21,5 +21,6 @@ body:
     validations:
       required: false
 labels:
-  - bug
-  - area/plugin
+  - area/plugin
+type: Bug
+projects: ["kestra-io/15"]

.github/ISSUE_TEMPLATE/config.yml

@@ -2,3 +2,4 @@ contact_links:
   - name: Chat
     url: https://kestra.io/slack
     about: Chat with us on Slack.
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -8,5 +8,6 @@ body:
     validations:
       required: true
 labels:
-  - enhancement
-  - area/plugin
+  - area/plugin
+type: Feature
+projects: ["kestra-io/15"]

.github/pull_request_template.md

@@ -40,47 +40,4 @@ Thank you for your contribution. ❤️  -->
 
 ### Contributor Checklist ✅
 
-- [ ] PR Title and commits follows [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- [ ] Add a `closes #ISSUE_ID` or `fixes #ISSUE_ID` in the description if the PR relates to an opened issue.
-- [ ] Documentation updated (plugin docs from `@Schema` for properties and outputs, `@Plugin` with examples, `README.md` file with basic knowledge and specifics).
-- [ ] Setup instructions included if needed (API keys, accounts, etc.).
-- [ ] Prefix all rendered properties by `r` not `rendered` (eg: `rHost`).
-- [ ] Use `runContext.logger()` to log enough important infos where it's needed and with the best level (DEBUG, INFO, WARN or ERROR).
-
-⚙️ **Properties**
-- [ ] Properties are declared with `Property<T>` carrier type, do **not** use `@PluginProperty`.
-- [ ] Mandatory properties must be annotated with `@NotNull` and checked during the rendering.
-- [ ] You can model a JSON thanks to a simple `Property<Map<String, Object>>`.
-
-🌐 **HTTP**
-- [ ] Must use Kestra’s internal HTTP client from `io.kestra.core.http.client`
-
-📦 **JSON**
-- [ ] If you are serializing response from an external API, you may have to add a `@JsonIgnoreProperties(ignoreUnknown = true)` at the mapped class level. So that we will avoid to crash the plugin if the provider add a new field suddenly.
-- [ ] Must use Jackson mappers provided by core (`io.kestra.core.serializers`)
-
-✨ **New plugins / subplugins**
-- [ ] Make sure your new plugin is configured like mentioned [here](https://kestra.io/docs/plugin-developer-guide/gradle#mandatory-configuration).
-- [ ] Add a `package-info.java` under each sub package respecting [this format](https://github.com/kestra-io/plugin-odoo/blob/main/src/main/java/io/kestra/plugin/odoo/package-info.java) and choosing the right category.
-- [ ] Every time you use `runContext.metric(...)` you have to add a `@Metric` ([see this doc](https://kestra.io/docs/plugin-developer-guide/document#document-the-plugin-metrics))
-- [ ] Docs don't support to have both tasks/triggers in the root package (e.g. `io.kestra.plugin.kubernetes`) and in a sub package (e.g. `io.kestra.plugin.kubernetes.kubectl`), whether it's: all tasks/triggers in the root package OR only tasks/triggers in sub packages.
-- [ ] Icons added in `src/main/resources/icons` in SVG format and not in thumbnail (keep it big):
-  - `plugin-icon.svg`
-  - One icon per package, e.g. `io.kestra.plugin.aws.svg`
-  - For subpackages, e.g. `io.kestra.plugin.aws.s3`, add `io.kestra.plugin.aws.s3.svg`
-    See example [here](https://github.com/kestra-io/plugin-elasticsearch/blob/master/src/main/java/io/kestra/plugin/elasticsearch/Search.java#L76).
-- [ ] Use `"{{ secret('YOUR_SECRET') }}"` in the examples for sensible infos such as an API KEY.
-- [ ] If you are fetching data (one, many or too many), you must add a `Property<FetchType> fetchType` to be able to use `FETCH_ONE`, `FETCH` and even `STORE` to store big amount of data in the internal storage.
-- [ ] Align the `"""` to close examples blocks with the flow id.
-- [ ] Update the existing `index.yaml` for the main plugin, and for each new subpackage add a metadata file named exactly after the subpackage (e.g. `s3.yaml` for `io.kestra.plugin.aws.s3`) under `src/main/resources/metadata/`, following the same schema.
-
-🧪 **Tests**
-- [ ] Unit Tests added or updated to cover the change (using the `RunContext` to actually run tasks).
-- [ ] Add sanity checks if possible with a YAML flow inside `src/test/resources/flows`.
-- [ ] Avoid disabling tests for CI. Instead, configure a local environment whenever it's possible with `.github/setup-unit.sh` (to be set executable with `chmod +x setup-unit.sh`) (which can be executed locally and in the CI) all along with a new `docker-compose-ci.yml` file (do **not** edit the existing `docker-compose.yml`). If needed, create an executable (`chmod +x cleanup-unit.sh`) `cleanup-unit.sh` to remove the potential costly resources (tables, datasets, etc).
-- [ ] Provide screenshots from your QA / tests locally in the PR description. The goal here is to use the JAR of the plugin and directly test it locally in Kestra UI to ensure it integrates well.
-
-📤 **Outputs**
-- [ ] Do not send back as outputs the same infos you already have in your properties.
-- [ ] If you do not have any output use `VoidOutput`.
-- [ ] Do not output twice the same infos (eg: a status code, an error code saying the same thing...).
+- [ ] I have read and followed the [plugin contribution guidelines](https://kestra.io/docs/plugin-developer-guide/contribution-guidelines)

build.gradle

@@ -5,7 +5,7 @@ plugins {
     id "idea"
     id 'jacoco'
     id "com.adarshr.test-logger" version "4.0.0"
-    id "com.gradleup.shadow" version "9.3.0"
+    id "com.gradleup.shadow" version "9.3.1"
     id 'signing'
     id "com.github.ben-manes.versions" version "0.53.0"
     id 'net.researchgate.release' version '3.1.0'

src/main/java/io/kestra/plugin/fs/ftp/Trigger.java

@@ -72,7 +72,7 @@
                         host: localhost
                         port: 21
                         username: foo
-                        password: bar
+                        password: "{{ secret('FTP_PASSWORD') }}"
                         uri: "/in/{{ taskrun.value | jq('.name') }}"
 
                 triggers:
@@ -81,7 +81,7 @@
                     host: localhost
                     port: 21
                     username: foo
-                    password: bar
+                    password: "{{ secret('FTP_PASSWORD') }}"
                     from: "/in/"
                     interval: PT10S
                     action: NONE
@@ -109,12 +109,12 @@
                     host: localhost
                     port: "21"
                     username: foo
-                    password: bar
+                    password: "{{ secret('FTP_PASSWORD') }}"
                     from: "mydir/"
                     regExp: ".*.csv"
                     action: MOVE
                     moveDirectory: "archive/"
-                    interval: PTS
+                    interval: PT10S
                 """
         )
     }

src/main/java/io/kestra/plugin/fs/ftps/Trigger.java

@@ -82,7 +82,7 @@
                     regExp: ".*.csv"
                     action: MOVE
                     moveDirectory: "archive/"
-                    interval: PTS
+                    interval: PT10S
                 """
         )
     }

src/main/java/io/kestra/plugin/fs/local/Downloads.java

@@ -111,6 +111,12 @@ public class Downloads extends AbstractLocalTask implements RunnableTask<Downloa
     @Builder.Default
     private Property<Boolean> recursive = Property.ofValue(false);
 
+    @Builder.Default
+    @Schema(
+        title = "The maximum number of files to retrieve at once"
+    )
+    private Property<Integer> maxFiles = Property.ofValue(25);
+
     static void performAction(
         java.util.List<File> files,
         Action action,
@@ -166,10 +172,20 @@ public Output run(RunContext runContext) throws Exception {
             .from(Property.ofValue(renderedFrom))
             .regExp(this.regExp)
             .recursive(this.recursive)
+            .maxFiles(this.maxFiles)
             .build();
 
         io.kestra.plugin.fs.local.List.Output listOutput = listTask.run(runContext);
 
+        int rMaxFiles = runContext.render(this.maxFiles).as(Integer.class).orElse(25);
+        if (listOutput.getFiles().size() > rMaxFiles) {
+            runContext.logger().warn("Too many files to process, skipping");
+            return Output.builder()
+                .files(java.util.List.of())
+                .outputFiles(Map.of())
+                .build();
+        }
+
         List<File> downloadedFiles = listOutput
             .getFiles()
             .stream()

src/main/java/io/kestra/plugin/fs/local/List.java

@@ -7,15 +7,14 @@
 import io.kestra.core.runners.RunContext;
 import io.kestra.plugin.fs.local.models.File;
 import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotNull;
 import lombok.*;
 import lombok.experimental.SuperBuilder;
 
-import jakarta.validation.constraints.NotNull;
-
-import java.nio.file.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.Objects;
 
 import static io.kestra.core.utils.Rethrow.throwFunction;
 
@@ -80,6 +79,12 @@ public class List extends AbstractLocalTask implements RunnableTask<List.Output>
     @Builder.Default
     private Property<Boolean> recursive = Property.ofValue(false);
 
+    @Builder.Default
+    @Schema(
+        title = "The maximum number of files to retrieve at once"
+    )
+    private Property<Integer> maxFiles = Property.ofValue(25);
+
     @Override
     public Output run(RunContext runContext) throws Exception {
         String resolvedDirectory = runContext.render(this.from).as(String.class).orElseThrow();
@@ -93,15 +98,22 @@ public Output run(RunContext runContext) throws Exception {
         String fileRegex = this.regExp != null ? runContext.render(this.regExp).as(String.class).orElseThrow() : ".*";
         int maxDepth = runContext.render(recursive).as(Boolean.class).orElse(false) ? Integer.MAX_VALUE : 1;
 
-        java.util.List<File> files = Files.find(directoryPath, maxDepth, (path, basicFileAttributes) -> {
-                return basicFileAttributes.isRegularFile() && path.toString().matches(fileRegex);
-            })
+        java.util.List<File> files = Files.find(directoryPath, maxDepth, (path, basicFileAttributes) -> basicFileAttributes.isRegularFile() && path.toString().matches(fileRegex))
             .map(throwFunction(path -> {
                 BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
                 return File.from(path, attrs);
             }))
             .filter(Objects::nonNull)
-            .collect(Collectors.toList());
+            .toList();
+
+        int rMaxFiles = runContext.render(this.maxFiles).as(Integer.class).orElse(25);
+        if (files.size() > rMaxFiles) {
+            runContext.logger().warn("Too many files to process, skipping");
+            return Output.builder()
+                .files(java.util.List.of())
+                .count(0)
+                .build();
+        }
 
         return Output.builder()
             .files(files)
@@ -122,4 +134,4 @@ public static class Output implements io.kestra.core.models.tasks.Output {
         )
         private Integer count;
     }
-}
+}

src/main/java/io/kestra/plugin/fs/local/Trigger.java

@@ -112,9 +112,16 @@ public class Trigger extends AbstractTrigger implements PollingTriggerInterface,
 
     private Property<Duration> stateTtl;
 
+    @Builder.Default
+    @Schema(
+        title = "The maximum number of files to retrieve at once"
+    )
+    private Property<Integer> maxFiles = Property.ofValue(25);
+
     @Override
     public Optional<Execution> evaluate(ConditionContext conditionContext, TriggerContext triggerContext) throws Exception {
         RunContext runContext = conditionContext.getRunContext();
+        var logger = runContext.logger();
         var rOn = runContext.render(on).as(On.class).orElse(On.CREATE_OR_UPDATE);
         var rStateKey = runContext.render(stateKey).as(String.class).orElse(StatefulTriggerService.defaultKey(triggerContext.getNamespace(), triggerContext.getFlowId(), id));
         var rStateTtl = runContext.render(stateTtl).as(Duration.class);
@@ -126,6 +133,7 @@ public Optional<Execution> evaluate(ConditionContext conditionContext, TriggerCo
             .from(Property.ofValue(rFrom))
             .regExp(this.regExp)
             .recursive(this.recursive)
+            .maxFiles(this.maxFiles)
             .build();
 
         io.kestra.plugin.fs.local.List.Output listOutput = listTask.run(runContext);
@@ -183,6 +191,11 @@ public Optional<Execution> evaluate(ConditionContext conditionContext, TriggerCo
             return Optional.empty();
         }
 
+        if (toFire.size() > runContext.render(this.maxFiles).as(Integer.class).orElse(25)) {
+            logger.warn("Too many files to process, skipping");
+            return Optional.empty();
+        }
+
         Downloads.Action selectedAction = this.action != null ?
             runContext.render(this.action).as(Downloads.Action.class).orElse(Downloads.Action.NONE) :
             Downloads.Action.NONE;