Skip to content

Commit 0c8e0f8

Browse files
nlamiraultnlamirault
committed
feat(spiceai): new components
Signed-off-by: Nicolas Lamirault <[email protected]>
1 parent be3fc32 commit 0c8e0f8

File tree

7 files changed

+386
-0
lines changed

7 files changed

+386
-0
lines changed

‎gitops/argocd/apps/data/Chart.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
apiVersion: v2
19+
type: application
20+
name: data
21+
version: 1.0.0
22+
appVersion: 1.0.0
23+
dependencies:
24+
- name: argocd-apps
25+
repository: oci://ghcr.io/argoproj/argo-helm
26+
version: 2.0.0

‎gitops/argocd/apps/data/values-talos-homelab.yaml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
argocd-apps:
19+
applicationsets:
20+
data:
21+
additionalLabels:
22+
portefaix.xyz/env: homelab
23+
additionalAnnotations: {}
24+
generators:
25+
- list:
26+
elements:
27+
- appName: spiceai
28+
namespace: data-lakehouse
29+
targetRevision: HEAD
30+
template:
31+
metadata:
32+
annotations:
33+
notifications.argoproj.io/subscribe.on-degraded.slack: portefaix-homelab-gitops
34+
notifications.argoproj.io/subscribe.on-deployed.slack: portefaix-homelab-gitops
35+
notifications.argoproj.io/subscribe.on-sync-failed: portefaix-homelab-gitops
36+
notifications.argoproj.io/subscribe.on-sync-running.slack: portefaix-homelab-gitops
37+
notifications.argoproj.io/subscribe.on-sync-status-unknown: portefaix-homelab-gitops
38+
notifications.argoproj.io/subscribe.on-sync-succeeded.slack: portefaix-homelab-gitops
39+
labels:
40+
portefaix.xyz/env: homelab
41+
spec:
42+
project: portefaix-talos-homelab
43+
source:
44+
helm:
45+
valueFiles:
46+
- values.yaml
47+
- values-talos-homelab.yaml

‎gitops/argocd/apps/data/values.yaml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
argocd-apps:
19+
applicationsets:
20+
data:
21+
additionalAnnotations:
22+
a8r.io/bugs: "https://github.com/portefaix/portefaix/issues"
23+
a8r.io/description: Argo-CD Data application
24+
a8r.io/documentation: "https://github.com/portefaix/portefaix/blob/master/README.md"
25+
a8r.io/owner: portefaix
26+
a8r.io/support: "https://github.com/portefaix/portefaix/issues"
27+
a8s.io/repository: >-
28+
https://github.com/portefaix/portefaix/tree/master/gitops/argocd/apps/data
29+
additionalLabels:
30+
app.kubernetes.io/component: argocd-applicationset
31+
app.kubernetes.io/instance: data-argocd-applicationset
32+
app.kubernetes.io/managed-by: helm
33+
app.kubernetes.io/name: data
34+
app.kubernetes.io/part-of: data-argocd
35+
app.kubernetes.io/version: v1.0.0
36+
portefaix.xyz/stack: data
37+
portefaix.xyz/version: v0.54.0
38+
goTemplate: true
39+
goTemplateOptions: ["missingkey=error"]
40+
generators:
41+
- list:
42+
elements: []
43+
template:
44+
metadata:
45+
name: "{{ .appName }}"
46+
annotations:
47+
# https://argo-cd.readthedocs.io/en/stable/operator-manual/high_availability/#webhook-and-manifest-paths-annotation
48+
argocd.argoproj.io/manifest-generate-paths: "."
49+
labels:
50+
app.kubernetes.io/component: argocd-application
51+
app.kubernetes.io/instance: "{{ .appName }}-argocd-application"
52+
app.kubernetes.io/managed-by: helm
53+
app.kubernetes.io/name: "{{ .appName }}"
54+
app.kubernetes.io/part-of: "{{ .appName }}-argocd"
55+
app.kubernetes.io/version: v1.0.0
56+
portefaix.xyz/stack: data
57+
portefaix.xyz/version: v0.54.0
58+
finalizers:
59+
- resources-finalizer.argocd.argoproj.io
60+
spec:
61+
destination:
62+
namespace: "{{ .namespace }}"
63+
server: https://kubernetes.default.svc
64+
source:
65+
repoURL: https://github.com/portefaix/portefaix-kubernetes.git
66+
path: "gitops/argocd/charts/{{ .namespace }}/{{ .appName }}"
67+
syncPolicy:
68+
# automated:
69+
# prune: true
70+
# selfHeal: true
71+
# allowEmpty: false
72+
syncOptions:
73+
- CreateNamespace=true
74+
- Validate=false
75+
- SkipDryRunOnMissingResource=true
76+
- ApplyOutOfSyncOnly=true
77+
# - ServerSideApply=true
78+
# retry:
79+
# limit: 5
80+
# backoff:
81+
# duration: 5s
82+
# factor: 2
83+
# maxDuration: 3m0s
84+
templatePatch: |
85+
spec:
86+
source:
87+
{{- if .targetRevision }}
88+
targetRevision: '{{ .targetRevision }}'
89+
{{- end }}
90+
{{- if .autoSync }}
91+
syncPolicy:
92+
automated:
93+
{{- if .prune }}
94+
prune: {{ .prune }}
95+
{{- end }}
96+
{{- end }}

‎gitops/argocd/charts/data-lakehouse/spiceai/Chart.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
apiVersion: v2
19+
type: application
20+
name: spiceai
21+
version: 1.0.0
22+
appVersion: 1.0.0
23+
dependencies:
24+
- name: spiceai
25+
repository: https://helm.spiceai.org
26+
version: 0.1.29

‎gitops/argocd/charts/data-lakehouse/spiceai/templates/credentials.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Copyright (C) Nicolas Lamirault <[email protected]>
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# SPDX-License-Identifier: Apache-2.0
16+
17+
---
18+
apiVersion: external-secrets.io/v1beta1
19+
kind: ExternalSecret
20+
metadata:
21+
labels:
22+
app: spiceai
23+
name: spice-credentials
24+
namespace: {{ .Release.Namespace }}
25+
spec:
26+
refreshInterval: 1h
27+
secretStoreRef:
28+
kind: ClusterSecretStore
29+
name: akeyless
30+
target:
31+
name: spice-credentials
32+
creationPolicy: Owner
33+
data:
34+
- secretKey: SPICE_GITHUB_PAT
35+
remoteRef:
36+
key: SPICE_GITHUB_PAT
37+
- secretKey: SPICE_OPENAI_API_KEY
38+
remoteRef:
39+
key: SPICE_OPENAI_API_KEY
40+
- secretKey: SPICE_HF_TOKEN
41+
remoteRef:
42+
key: SPICE_HF_TOKEN

‎gitops/argocd/charts/data-lakehouse/spiceai/values-talos-homelab.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
spiceai:
19+
resources:
20+
limits:
21+
memory: 500Mi
22+
requests:
23+
cpu: 10m
24+
memory: 50Mi
25+
26+
monitoring:
27+
podMonitor:
28+
relabelings:
29+
- action: replace
30+
replacement: portefaix-talos-homelab
31+
targetLabel: cluster

‎gitops/argocd/charts/data-lakehouse/spiceai/values.yaml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
---
2+
# Copyright (C) Nicolas Lamirault <[email protected]>
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
spiceai:
19+
# TODO:
20+
# additionalLabels:
21+
# portefaix.xyz/version: v0.54.0
22+
monitoring:
23+
podMonitor:
24+
enabled: true
25+
additionalLabels:
26+
prometheus.io/operator: portefaix
27+
28+
additionalEnv:
29+
- name: SPICE_GITHUB_PAT
30+
valueFrom:
31+
secretKeyRef:
32+
name: spice-credentials
33+
key: SPICE_GITHUB_PAT
34+
- name: SPICE_OPENAI_API_KEY
35+
valueFrom:
36+
secretKeyRef:
37+
name: spice-credentials
38+
key: SPICE_OPENAI_API_KEY
39+
- name: SPICE_HF_TOKEN
40+
valueFrom:
41+
secretKeyRef:
42+
name: spice-credentials
43+
key: SPICE_HF_TOKEN
44+
45+
spicepod:
46+
name: app
47+
version: v1beta1
48+
kind: Spicepod
49+
50+
datasets:
51+
- from: s3://spiceai-demo-datasets/taxi_trips/2024/
52+
name: taxi_trips
53+
description: Demo taxi trips in s3
54+
params:
55+
file_format: parquet
56+
acceleration:
57+
enabled: true
58+
# Uncomment to refresh the acceleration on a schedule
59+
# refresh_check_interval: 1h
60+
# refresh_mode: full
61+
engine: duckdb
62+
mode: file
63+
params:
64+
duckdb_file: /my/chosen/location/duckdb.db
65+
- from: github:github.com/portefaix/portefaix-kubernetes/issues
66+
name: spiceai.issues
67+
params:
68+
github_token: ${env:SPICE_GITHUB_PAT}
69+
acceleration:
70+
enabled: true
71+
- from: github:github.com/portefaix/portefaix-kubernetes/pulls
72+
name: spiceai.pulls
73+
params:
74+
github_token: ${env:SPICE_GITHUB_PAT}
75+
github_query_mode: search
76+
time_column: created_at
77+
acceleration:
78+
enabled: true
79+
refresh_mode: append
80+
refresh_check_interval: 6h # check for new results every 6 hours
81+
refresh_data_window: 90d # at initial load, load the last 90 days of pulls
82+
- from: github:github.com/portefaix/portefaix-kubernetes/commits
83+
name: spiceai.commits
84+
params:
85+
github_token: ${env:SPICE_GITHUB_PAT}
86+
acceleration:
87+
enabled: true
88+
89+
models:
90+
# - from: huggingface:huggingface.co/meta-llama/Llama-3.2-1B
91+
# name: llama_3.2_1B
92+
# params:
93+
# hf_token: ${env:HF_TOKEN}
94+
- from: openai:gpt-4o
95+
name: local_fs_model
96+
params:
97+
openai_api_key: ${env:SPICE_OPENAI_API_KEY}
98+
- from: huggingface:huggingface.co/spiceai/darts:latest
99+
name: hf_model
100+
files:
101+
- path: model.onnx
102+
params:
103+
hf_token: ${env:SPICE_HF_TOKEN}
104+
datasets:
105+
- taxi_trips
106+
107+
# grafanaDashboard:
108+
# enabled: true
109+
# folder: database
110+
# annotations:
111+
# name: grafana.com/folder
112+
# labels:
113+
# name: grafana.com/dashboard
114+
# grafanaOperator:
115+
# enabled: true
116+
# allowCrossNamespaceImport: true
117+
# matchLabels:
118+
# grafana.com/dashboards: portefaix

0 commit comments

Comments
 (0)