Skip to content

Commit 51f7d6a

Browse files
AcevedoRAcevedoR
authored
Initial commit
0 parents  commit 51f7d6a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+1707
-0
lines changed

‎.env_example‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
SECRET_GITHUB_ACCESS_TOKEN={{ "ghp_xxxxxxxxxx" | base64encode }}
2+
SECRET_AIRBYTE_API_KEY=xbase64_encoded_airbyte_api_key
3+
SECRET_AIRBYTE_USERNAME=base64_encoded_airbyte_username
4+
SECRET_AIRBYTE_PASSWORD=xxxxxxxxxxxxxxxxxxxxxx
5+
SECRET_FIVETRAN_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
6+
SECRET_FIVETRAN_API_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
7+
SECRET_GCP_CREDS={{ '{"type":"service_account","project_id":"geller","private_key_id":"xxx", ...}' | base64encode }}
8+
SECRET_OPENAI_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
9+
SECRET_GOOGLE_SEARCH_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
10+
SECRET_AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxxxxxxx
11+
SECRET_AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

‎.github/workflows/main.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Container Image Packages
2+
3+
on:
4+
push:
5+
paths:
6+
- dockerfiles/*
7+
branches:
8+
- main
9+
10+
jobs:
11+
changes:
12+
name: Dockerfile Changes
13+
runs-on: ubuntu-latest
14+
outputs:
15+
dockerfiles: ${{ steps.filter.outputs.dockerfiles_files }}
16+
dockerfiles_changed: ${{ steps.filter.outputs.dockerfiles }}
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
21+
- name: Filter Dockerfile changes
22+
uses: dorny/paths-filter@v3
23+
id: filter
24+
with:
25+
list-files: json
26+
filters: |
27+
dockerfiles:
28+
- added|modified: 'dockerfiles/*.Dockerfile'
29+
30+
- name: Generate Markdown Summary
31+
run: |
32+
echo New/modified Dockerfiles: ${{ steps.filter.outputs.dockerfiles_files }} >> $GITHUB_STEP_SUMMARY
33+
34+
ghcr:
35+
needs: [changes]
36+
if: ${{ needs.changes.outputs.dockerfiles_changed == 'true' }}
37+
runs-on: ubuntu-latest
38+
permissions:
39+
contents: read
40+
packages: write
41+
strategy:
42+
matrix:
43+
image: ${{ fromJson(needs.changes.outputs.dockerfiles) }}
44+
steps:
45+
- name: Checkout repository
46+
uses: actions/checkout@v4
47+
48+
- name: Set up QEMU for cross-platform builds
49+
uses: docker/setup-qemu-action@v3
50+
51+
- name: Set up Docker Buildx
52+
uses: docker/setup-buildx-action@v3
53+
54+
- name: Login to GHCR
55+
uses: docker/login-action@v3
56+
with:
57+
registry: ghcr.io
58+
username: ${{ github.actor }}
59+
password: ${{ secrets.GITHUB_TOKEN }}
60+
61+
- id: image-tag # example output: "ghcr.io/kestra-io/pydata:latest"
62+
run: |
63+
IMAGE=$(basename ${{ matrix.image }} .Dockerfile)
64+
echo "image_url=ghcr.io/kestra-io/$IMAGE:latest" >> $GITHUB_OUTPUT
65+
echo "file=${{ matrix.image }}" >> $GITHUB_OUTPUT
66+
67+
- name: Build and push Docker image
68+
uses: docker/build-push-action@v4
69+
with:
70+
context: .
71+
push: true
72+
tags: ${{ steps.image-tag.outputs.image_url }}
73+
file: ${{ steps.image-tag.outputs.file }}
74+
platforms: linux/amd64,linux/arm64

‎.github/workflows/manual.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Manual Container Image Builds
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
list-dockerfiles:
8+
runs-on: ubuntu-latest
9+
outputs:
10+
matrix: ${{ steps.dockerfiles.outputs.matrix }}
11+
steps:
12+
- name: Checkout repository
13+
uses: actions/checkout@v4
14+
15+
- id: dockerfiles
16+
run: echo "matrix=$(ls dockerfiles-manual/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
17+
18+
ghcr:
19+
runs-on: ubuntu-latest
20+
needs: list-dockerfiles
21+
permissions:
22+
contents: read
23+
packages: write
24+
strategy:
25+
matrix:
26+
image: ${{ fromJson(needs.list-dockerfiles.outputs.matrix) }}
27+
steps:
28+
- name: Checkout repository
29+
uses: actions/checkout@v4
30+
31+
- name: Set up QEMU for cross-platform builds
32+
uses: docker/setup-qemu-action@v3
33+
34+
- name: Set up Docker Buildx
35+
uses: docker/setup-buildx-action@v3
36+
37+
- name: Login to GHCR
38+
uses: docker/login-action@v3
39+
with:
40+
registry: ghcr.io
41+
username: ${{ github.actor }}
42+
password: ${{ secrets.GITHUB_TOKEN }}
43+
44+
- id: image-tag # example output: "ghcr.io/kestra-io/pydata:latest"
45+
run: |
46+
IMAGE=$(basename ${{ matrix.image }} .Dockerfile)
47+
echo "image_url=ghcr.io/kestra-io/$IMAGE:latest" >> $GITHUB_OUTPUT
48+
echo "file=dockerfiles-manual/${{ matrix.image }}" >> $GITHUB_OUTPUT
49+
50+
- name: Build and push Docker image
51+
uses: docker/build-push-action@v4
52+
with:
53+
context: .
54+
push: true
55+
tags: ${{ steps.image-tag.outputs.image_url }}
56+
file: ${{ steps.image-tag.outputs.file }}
57+
platforms: linux/amd64,linux/arm64

‎.github/workflows/scheduled.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Scheduled Container Image Packages
2+
3+
on:
4+
schedule:
5+
- cron: '0 9 * * MON'
6+
workflow_dispatch:
7+
8+
jobs:
9+
list-dockerfiles:
10+
runs-on: ubuntu-latest
11+
outputs:
12+
matrix: ${{ steps.dockerfiles.outputs.matrix }}
13+
steps:
14+
- name: Checkout repository
15+
uses: actions/checkout@v4
16+
17+
- id: dockerfiles
18+
run: echo "matrix=$(ls dockerfiles/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
19+
20+
ghcr:
21+
runs-on: ubuntu-latest
22+
needs: list-dockerfiles
23+
permissions:
24+
contents: read
25+
packages: write
26+
strategy:
27+
matrix:
28+
image: ${{ fromJson(needs.list-dockerfiles.outputs.matrix) }}
29+
steps:
30+
- name: Checkout repository
31+
uses: actions/checkout@v4
32+
33+
- name: Set up QEMU for cross-platform builds
34+
uses: docker/setup-qemu-action@v3
35+
36+
- name: Set up Docker Buildx
37+
uses: docker/setup-buildx-action@v3
38+
39+
- name: Login to GHCR
40+
uses: docker/login-action@v3
41+
with:
42+
registry: ghcr.io
43+
username: ${{ github.actor }}
44+
password: ${{ secrets.GITHUB_TOKEN }}
45+
46+
- id: image-tag # example output: "ghcr.io/kestra-io/pydata:latest"
47+
run: |
48+
IMAGE=$(basename ${{ matrix.image }} .Dockerfile)
49+
echo "image_url=ghcr.io/kestra-io/$IMAGE:latest" >> $GITHUB_OUTPUT
50+
echo "file=dockerfiles/${{ matrix.image }}" >> $GITHUB_OUTPUT
51+
52+
- name: Build and push Docker image
53+
uses: docker/build-push-action@v4
54+
with:
55+
context: .
56+
push: true
57+
tags: ${{ steps.image-tag.outputs.image_url }}
58+
file: ${{ steps.image-tag.outputs.file }}
59+
platforms: linux/amd64,linux/arm64

‎.gitignore‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Local .terraform directories
2+
**/.terraform/*
3+
4+
# .tfstate files
5+
*.tfstate
6+
*.tfstate.*
7+
8+
# Crash log files
9+
crash.log
10+
crash.*.log
11+
12+
# Exclude all .tfvars files, which are likely to contain sensitive data, such as
13+
# password, private keys, and other secrets. These should not be part of version
14+
# control as they are data points which are potentially sensitive and subject
15+
# to change depending on the environment.
16+
*.tfvars
17+
*.tfvars.json
18+
19+
# Ignore override files as they are usually used to override resources locally and so
20+
# are not checked in
21+
override.tf
22+
override.tf.json
23+
*_override.tf
24+
*_override.tf.json
25+
26+
# Include override files you do wish to add to version control using negated pattern
27+
# !example_override.tf
28+
29+
# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
30+
# example: *tfplan*
31+
32+
# Ignore CLI configuration files
33+
.terraformrc
34+
terraform.rc
35+
36+
# IDE
37+
.idea/
38+
.vscode/
39+
.archive/
40+
.wip/
41+
.DS_Store
42+
.env
43+
credentials.json
44+
*.hcl
45+
.env_encoded

‎.terraform.lock.hcl‎

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎ForEachItem/0_orders_parallel.yaml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
id: orders_parallel
2+
namespace: company.team
3+
4+
description: |
5+
Child flow:
6+
```yaml
7+
id: orders
8+
description: works also with batch size > 1
9+
namespace: company.team
10+
11+
inputs:
12+
- id: order
13+
type: STRING
14+
description: internal storage URL
15+
16+
- id: order2
17+
type: FILE
18+
description: internal storage file
19+
20+
- id: order3
21+
type: STRING
22+
description: internal storage file's content
23+
24+
tasks:
25+
- id: read_file_content
26+
type: io.kestra.plugin.core.log.Log
27+
message: "{{ read(inputs.order) }}"
28+
29+
- id: read_file_content2
30+
type: io.kestra.plugin.core.log.Log
31+
message: "{{ read(inputs.order2) }}"
32+
33+
- id: read_file_content3
34+
type: io.kestra.plugin.core.log.Log
35+
message: "{{ inputs.order3 }}"
36+
37+
- id: transform
38+
type: io.kestra.plugin.transform.jsonata.TransformItems
39+
from: "{{ inputs.order2 }}"
40+
expression: |
41+
{
42+
"customer_email": customer_email,
43+
"total": total
44+
}
45+
46+
outputs:
47+
- id: myoutput
48+
type: FILE
49+
value: "{{ outputs.transform.uri }}"
50+
```
51+
52+
tasks:
53+
- id: extract
54+
type: io.kestra.plugin.jdbc.duckdb.Query
55+
sql: |
56+
INSTALL httpfs;
57+
LOAD httpfs;
58+
SELECT *
59+
FROM read_csv_auto('https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv', header=True);
60+
store: true
61+
62+
- id: for_each_item
63+
type: io.kestra.plugin.core.flow.ForEachItem
64+
items: "{{ outputs.extract.uri }}"
65+
batch:
66+
rows: 1
67+
namespace: company.team
68+
flowId: orders
69+
wait: true # wait for the subflow execution
70+
transmitFailed: true # fail the task run if the subflow execution fails
71+
inputs:
72+
order: "{{ taskrun.items }}"
73+
order2: "{{ taskrun.items }}"
74+
order3: "{{ read(taskrun.items) }}"
75+
76+
- id: ion_to_json
77+
type: io.kestra.plugin.serdes.json.IonToJson
78+
from: "{{ outputs.for_each_item_merge.subflowOutputs }}"
79+
newLine: false
80+
81+
- id: merge
82+
type: io.kestra.plugin.core.storage.Concat
83+
files: "{{ read(outputs.ion_to_json.uri) | jq('map(.myoutput)') | flatten }}"
84+
separator: "\n"
85+
extension: .ion
86+
87+
- id: for_each
88+
type: io.kestra.plugin.core.flow.ForEach
89+
allowFailure: true
90+
description: this will fail with Unable to read the file 'kestra:///company/team/orders/executions/vBBGnSRLntC6xhA1IjFYc/tasks/transform/541J75lWtI0WQUvxz4v0qH/8580806061061976640.ion' as it didn't belong to the current execution
91+
values: "{{ read(outputs.ion_to_json.uri) }}"
92+
tasks:
93+
- id: log
94+
type: io.kestra.plugin.core.log.Log
95+
allowFailure: true
96+
message: "{{ read(json(taskrun.value).myoutput) }}"

0 commit comments

Comments
 (0)