Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ public interface BatchGenerateInterface extends GenerateInterface{

@Schema(
title = "Store generated data",
description = "If true, the generated data will be persisted to Kestra's internal storage. If false, the data is emitted part of the task output."
description = "Persist output to Kestra internal storage as Ion lines when true; defaults to false to return the value inline."
)
Property<Boolean> getStore();

@Schema(
title = "Batch size",
description = "The number of items to generate when storing data. This is only applicable when 'store' is set to true."
description = "Number of items to generate when storing data; only used if `store` is true. Defaults to 1."
)
Property<Integer> getBatchSize();

Expand Down
15 changes: 8 additions & 7 deletions src/main/java/io/kestra/plugin/datagen/Data.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,26 @@
public class Data implements io.kestra.core.models.tasks.Output {

@Schema(
title = "Size of the generated data",
description = "The total size in bytes of the generated data value or file."
title = "Generated size (bytes)",
description = "Total size in bytes of the returned value or stored Ion file."
)
private Long size;

@Schema(
title = "Total number of the generated items",
description = "The total number of items generated by the task."
title = "Items generated",
description = "Number of records produced by the generator."
)
private Integer count;

@Schema(
title = "Generated data value",
description = "The actual content of the generated data. This can be a string, number, JSON object, or binary payload depending on the generator configuration."
title = "Generated value",
description = "Inline content when `store` is false; null when data is stored. May be string, number, JSON object, or byte array depending on the generator."
)
private final Object value;

@Schema(
title = "The generated file URI."
title = "Stored file URI",
description = "URI in internal storage when `store` is true; null for inline outputs."
)
private final URI uri;
}
4 changes: 2 additions & 2 deletions src/main/java/io/kestra/plugin/datagen/GenerateInterface.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
public interface GenerateInterface {

@Schema(
title = "The Data Generator",
description = "The data generator implementation responsible for producing the execution data."
title = "Choose data generator",
description = "Generator used for each record; required for all generate tasks and triggers."
)
@NotNull
@PluginProperty
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/kestra/plugin/datagen/core/Generate.java
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@
}
)
@Schema(
title = "Generate data",
description = "This task generates data, using a configured data generator. Generators can support [Datafaker](https://www.datafaker.net/documentation/expressions/)"
title = "Generate synthetic data",
description = "Runs the configured generator (e.g., [Datafaker](https://www.datafaker.net/documentation/expressions/)) once (inline) or for a batch when `store` is true. When stored, results are written as Ion lines to internal storage; defaults are `store=false` and `batchSize=1`."
)
@SuperBuilder
@NoArgsConstructor
Expand Down
20 changes: 10 additions & 10 deletions src/main/java/io/kestra/plugin/datagen/core/RealtimeTrigger.java
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@
}
)
@Schema(
title = "Generate data in real time",
description = "This task continuously emits generated data in real time, using a configured data generator. It can be used to simulate event streams or high-throughput environments."
title = "Stream generated data in real time",
description = "Continuously runs the generator with backpressure. Controls include `throughput` per second (minimum 1), `maxRecords` cap (default unlimited), and `reportingInterval` for stats; data is emitted in trigger output, not stored."
)
@NoArgsConstructor
@SuperBuilder
Expand All @@ -82,29 +82,29 @@
public class RealtimeTrigger extends AbstractTrigger implements RealtimeTriggerInterface, TriggerOutput<Data>, GenerateInterface {

@Schema(
title = "Total Number of Records",
description = "The total number of records to generate. No further record will be generate once this number is reached."
title = "Total number of records",
description = "Hard limit on records emitted; defaults to unlimited when not set."
)
@Builder.Default
private Property<Long> maxRecords = Property.ofValue(Long.MAX_VALUE);

@Schema(
title = "Trigger Throughput",
description = "The approximate number of records per second that will be created by this trigger."
title = "Trigger throughput",
description = "Approximate records per second; values below 1 are rounded up to 1."
)
@Builder.Default
private Property<Integer> throughput = Property.ofValue(1);

@Schema(
title = "Reporting Interval",
description = "The time interval at which reporting is performed during generation."
title = "Reporting interval",
description = "Period for throughput reporting; defaults to 15 seconds."
)
@Builder.Default
private Property<Duration> reportingInterval = Property.ofValue(Duration.ofSeconds(15));

@Schema(
title = "Data Generator",
description = "The data generator implementation responsible for producing the data."
title = "Data generator",
description = "Generator used for each emitted record; not stored, only emitted in trigger outputs."
)
@NotNull
@PluginProperty
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/kestra/plugin/datagen/core/Trigger.java
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
}
)
@Schema(
title = "Generate data in real time",
description = "This task continuously emits generated data in real time, using a configured data generator. It can be used to simulate event streams or high-throughput environments."
title = "Poll to generate data batches",
description = "Periodically invokes the generator and emits an execution. Defaults: `store=false`, `batchSize=1`, `interval=PT1S`. Use `store=true` to persist Ion lines to internal storage instead of embedding the value."
)
@NoArgsConstructor
@SuperBuilder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
import java.util.Map;

@Schema(
title = "JSON Object Generator",
description = "Generates structured objects"
title = "Generate JSON objects from templates",
description = "Renders the `value` map, then evaluates Datafaker expressions (`#{...}`) on every record. Supports nested maps/lists; locale list overrides Faker locale, otherwise the library default is used."
)
@Plugin
@NoArgsConstructor
Expand All @@ -32,16 +32,16 @@
public final class JsonObjectGenerator extends DataGenerator<Map<String, Object>> {

@Schema(
title = "Object to generate",
description = "A map of key-value pairs where values can contain [Datafaker expressions](https://www.datafaker.net/documentation/expressions/) (e.g., #{name.firstName}) to be evaluated for each output record."
title = "Object template",
description = "Map of key-value pairs rendered per record; strings starting with `#{` are evaluated by [Datafaker](https://www.datafaker.net/documentation/expressions/), including nested maps and lists."
)
@NotNull
@PluginProperty
private Map<String, Object> value;

@Schema(
title = "Locales",
description = "List of locale values in the format [language, country, variant] (e.g., [\"en\", \"US\"], [\"fr\", \"FR\"]). Controls the language and region of the generated data."
description = "Optional locale list in the format [language, country, variant]; empty list uses Faker's default locale."
)
private Property<List<String>> locale;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
import java.security.SecureRandom;

@Schema(
title = "Random Bytes Generator",
description = "Generates random bytes"
title = "Generate random byte arrays",
description = "Allocates a byte array of the configured size and fills it with `SecureRandom` bytes. Size should be positive; randomness relies on the JVM's default seed."
)
@Plugin
@NoArgsConstructor
Expand All @@ -26,7 +26,7 @@ public class RandomBytesGenerator extends DataGenerator<byte[]> {

@Schema(
title = "Byte array size",
description = "The number of bytes to generate for each output value"
description = "Number of bytes produced per record; should be greater than zero"
)
@NotNull
private int size;
Expand All @@ -44,4 +44,4 @@ public byte[] produce() {
random.nextBytes(bytes);
return bytes;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
import java.util.List;

@Schema(
title = "String Value Generator",
description = "Generates textual data based on fixed values, pebble expressions, or [Datafaker expressions](https://www.datafaker.net/documentation/expressions/)."
title = "Generate strings from templates",
description = "Renders the `value` string (Pebble) and then evaluates [Datafaker expressions](https://www.datafaker.net/documentation/expressions/) like `#{name.firstName}` for each record. Locale list overrides Faker locale; empty list uses the library default."
)
@Plugin
@NoArgsConstructor
Expand All @@ -31,16 +31,16 @@
public final class StringValueGenerator extends DataGenerator<String> {

@Schema(
title = "The string value",
description = "The string value to generate for each output value"
title = "String template",
description = "String rendered per record; supports Pebble variables and Datafaker expressions starting with `#{`"
)
@NotNull
@PluginProperty
private String value;

@Schema(
title = "Locales",
description = "List of locale values in the format [language, country, variant] (e.g., [\"en\", \"US\"], [\"fr\", \"FR\"]). Controls the language and region of the generated data."
description = "Optional locale list in the format [language, country, variant]; empty list uses Faker's default locale."
)
private Property<List<String>> locale;

Expand Down
4 changes: 2 additions & 2 deletions src/main/resources/metadata/core.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
group: io.kestra.plugin.datagen.core
name: "core"
title: "Core"
description: "Tasks that generate synthetic records on demand or emit them continuously via triggers."
body: "Use `Generate` to produce a single value inline or batches saved to storage, and `Trigger`/`RealtimeTrigger` to schedule or stream executions. Each task requires a `generator` definition (e.g., JSON, string, bytes); tune `batchSize`, `store`, `throughput`, and `reportingInterval` to shape output volume and persistence."
description: "Tasks that generate synthetic records on demand or emit them continuously."
body: "Use `Generate` for inline values or batches stored in internal storage, `Trigger` for interval-based batches, and `RealtimeTrigger` for streaming at a controlled throughput. Every task needs a `generator` (JSON, string, bytes); key controls include `store` (defaults false), `batchSize` (defaults 1), `throughput`, `reportingInterval`, and `maxRecords` for caps."
videos: []
createdBy: "Kestra Core Team"
managedBy: "Kestra Core Team"
4 changes: 2 additions & 2 deletions src/main/resources/metadata/generators.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
group: io.kestra.plugin.datagen.generators
name: "generators"
title: "Generators"
description: "Tasks that provide reusable data generators for JSON objects, strings, or binary payloads."
body: "Generators plug into DataGen tasks and triggers to emit structured or raw data. Define the `value` template for `JsonObjectGenerator` or `StringValueGenerator` (supports Datafaker expressions and locale lists) or the `size` of random byte arrays, then reference the generator in a core task."
description: "Reusable generators for JSON objects, strings, or random byte payloads that can be referenced across DataGen tasks."
body: "Generators plug into DataGen tasks and triggers to emit structured or raw data. Use templates for `JsonObjectGenerator` or `StringValueGenerator` (Pebble + Datafaker expressions, optional locale list using Faker defaults when empty) or set the `size` for random byte arrays, then reference the generator in a core task."
videos: []
createdBy: "Kestra Core Team"
managedBy: "Kestra Core Team"
Loading