Skip to content

Commit 0f304e4

Browse files
mmurtommurto
committed
chore(dev): WIP on local K8s dev environment
Signed-off-by: Mikko Murto <[email protected]>
1 parent 0144db6 commit 0f304e4

24 files changed

+2000
-0
lines changed

‎README.md

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,43 @@ migrated by following these steps:
177177
docker compose up -d
178178
```
179179

180+
### Local Kubernetes with Tilt
181+
182+
#### Setup
183+
184+
1. Install Tilt
185+
2. Install OpenTofu
186+
3. Install Minikube
187+
4. Install ctlptl
188+
189+
Create the Minikube cluster with ctlptl:
190+
191+
```console
192+
ctlptl create cluster minikube --registry=ctlptl-registry --minikube-start-flags "--memory=6g" --minikube-start-flags "--cpus=4"
193+
```
194+
195+
Start ORT Server:
196+
197+
```console
198+
tilt up
199+
```
200+
201+
Destroy the cluster:
202+
203+
```console
204+
ctlptl delete cluster minikube
205+
```
206+
207+
To get the UI working, create file `/ui/.env.local` with the following content:
208+
209+
```env
210+
VITE_CLIENT_ID=ort-server
211+
VITE_AUTHORITY=http://localhost:8081/realms/ort-server
212+
```
213+
214+
and run it with `pnpm -C ui dev`.
215+
216+
180217
### Accessing the services
181218

182219
| Service | URL | Credentials |

‎Tiltfile

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
load('ext://helm_remote', 'helm_remote')
2+
load('ext://helm_resource', 'helm_resource', 'helm_repo')
3+
load('ext://configmap', 'configmap_create', 'configmap_from_dict')
4+
load('ext://secret', 'secret_create_generic')
5+
6+
update_settings(k8s_upsert_timeout_secs=120)
7+
8+
gradlew = "./gradlew"
9+
if os.name == "nt":
10+
gradlew = "gradlew.bat"
11+
12+
k8s_yaml('./scripts/kubernetes/namespace.yaml')
13+
14+
helm_resource(
15+
'postgresql',
16+
'bitnami/postgresql',
17+
resource_deps=['bitnami'],
18+
namespace='ort-server',
19+
flags=[
20+
'--set=global.postgresql.auth.postgresPassword=postgres',
21+
'--set=global.postgresql.auth.database=ort',
22+
],
23+
labels=['ort-server']
24+
)
25+
26+
helm_repo('bitnami', 'https://charts.bitnami.com/bitnami', labels=['helm_repos'])
27+
28+
secret_create_generic('keycloak-master-realm',
29+
namespace='ort-server',
30+
secret_type='generic',
31+
from_file='./scripts/docker/keycloak/master-realm.json')
32+
33+
configmap_create('keycloak-init',
34+
namespace='ort-server',
35+
from_file=['master-realm.json=./scripts/docker/keycloak/master-realm.json'])
36+
37+
configmap_create('ort-core-secrets',
38+
namespace='ort-server',
39+
from_file=['secrets.properties=./scripts/compose/secrets.properties'])
40+
41+
secret_create_generic('ort-secrets',
42+
namespace='ort-server',
43+
from_file=['secrets.properties=./scripts/compose/secrets.properties'])
44+
45+
secret_create_generic('ort-core-secrets',
46+
namespace='ort-server',
47+
from_file=['secrets.properties=./scripts/compose/secrets.properties']
48+
)
49+
50+
secret_create_generic('ort-config-secret',
51+
namespace='ort-server',
52+
from_file=[
53+
'evaluator.rules.kts=./scripts/compose/config/evaluator.rules.kts',
54+
'ort-server.params.kts=./scripts/compose/config/ort-server.params.kts',
55+
]
56+
)
57+
58+
helm_resource(
59+
'keycloak',
60+
'bitnami/keycloak',
61+
resource_deps=['bitnami'],
62+
namespace='ort-server',
63+
flags=[
64+
'--version=21.4.5',
65+
'--set=auth.adminUser=keycloak-admin',
66+
'--set=auth.adminPassword=keycloak-admin',
67+
'--set=extraStartupArgs=--hostname-strict-backchannel=false',
68+
],
69+
labels=['keycloak']
70+
)
71+
72+
local_resource('keycloak-terraform',
73+
resource_deps=['keycloak'],
74+
cmd='cd ./scripts/kubernetes/keycloak && tofu init && tofu apply -auto-approve',
75+
deps=['./scripts/kubernetes/keycloak/keycloak.tf'],
76+
labels=['keycloak']
77+
)
78+
79+
# Keycloak port forward has to be done manually because the Chart contains multiple containers, and
80+
# the port forward may hit the database if done in helm_resource.
81+
k8s_resource(
82+
workload='keycloak',
83+
port_forwards=["8081:8080"],
84+
extra_pod_selectors={'statefulset.kubernetes.io/pod-name': 'keycloak-0'},
85+
discovery_strategy='selectors-only')
86+
87+
secret_create_generic('rabbitmq-load-definition',
88+
namespace='ort-server',
89+
secret_type='generic',
90+
from_file='load_definition.json=./scripts/docker/rabbitmq/load_definition.json')
91+
92+
helm_resource(
93+
'rabbitmq',
94+
'bitnami/rabbitmq',
95+
resource_deps=['bitnami'],
96+
namespace='ort-server',
97+
flags=[
98+
'--version=14.4.6',
99+
"--set=auth.username=admin",
100+
"--set=auth.password=admin",
101+
],
102+
labels=['rabbitmq']
103+
)
104+
105+
k8s_resource(
106+
workload='rabbitmq',
107+
port_forwards=["15672"],
108+
)
109+
110+
local_resource('rabbitmq-terraform',
111+
resource_deps=['rabbitmq'],
112+
cmd='cd ./scripts/kubernetes/rabbitmq && tofu init && tofu apply -auto-approve',
113+
deps=['./scripts/kubernetes/rabbitmq/rabbitmq.tf'],
114+
labels=['rabbitmq']
115+
)
116+
117+
helm_repo('kiwigrid', 'https://kiwigrid.github.io', labels=['helm_repos'])
118+
119+
helm_resource(
120+
'graphite',
121+
'kiwigrid/graphite',
122+
resource_deps=['kiwigrid'],
123+
namespace='ort-server',
124+
labels=['monitoring'],
125+
)
126+
127+
configmap_create('ort-core-config',
128+
namespace='ort-server',
129+
from_file=['application.conf=./scripts/kubernetes/core.application.conf'])
130+
131+
custom_build(
132+
'core',
133+
'./gradlew :core:jibDockerBuild --image $EXPECTED_REF',
134+
live_update= [
135+
sync('./core/build/classes/kotlin/main', '/app/classes')
136+
],
137+
deps=['./core/build/classes', './core/build.gradle.kts', './scripts/kubernetes/core.application.conf'],
138+
)
139+
140+
k8s_resource(
141+
workload='ort-server-core',
142+
port_forwards=[
143+
port_forward(8080, 8080, "API Endpoint")],
144+
links=[
145+
link('http://localhost:8080/swagger-ui', "Swagger UI"),
146+
],
147+
resource_deps=['keycloak', 'rabbitmq', 'rabbitmq-terraform', 'graphite', 'postgresql', 'keycloak-terraform'],
148+
labels=['ort-server'],
149+
)
150+
151+
configmap_create('ort-orchestrator-config',
152+
namespace='ort-server',
153+
from_file=['application.conf=./scripts/kubernetes/orchestrator.application.conf'])
154+
155+
secret_create_generic('ort-config-worker-config',
156+
secret_type='generic',
157+
namespace='ort-server',
158+
from_file=['application.conf=./scripts/kubernetes/config.application.conf'],
159+
)
160+
161+
k8s_resource(
162+
workload='ort-server-orchestrator',
163+
resource_deps=['keycloak', 'rabbitmq', 'rabbitmq-terraform', 'graphite', 'postgresql', 'ort-server-core', 'worker-base-images'],
164+
labels=['ort-server'],
165+
)
166+
167+
custom_build(
168+
'ort-server-orchestrator',
169+
'./gradlew :orchestrator:jibDockerBuild --image $EXPECTED_REF',
170+
live_update= [
171+
sync('./orchestrator/build/classes/kotlin/main', '/app/classes')
172+
],
173+
deps=['./orchestrator/build/classes', './orchestrator/build.gradle.kts', './scripts/kubernetes/orchestrator.application.conf'],
174+
)
175+
176+
k8s_yaml('./scripts/kubernetes/core.yaml')
177+
k8s_yaml('./scripts/kubernetes/orchestrator.yaml')
178+
179+
# Worker images
180+
181+
local_resource(
182+
'worker-base-images',
183+
cmd='./gradlew buildAllWorkerImages',
184+
deps=[
185+
'./workers/analyzer/docker/Analyzer.Dockerfile',
186+
'./workers/config/docker/Config.Dockerfile',
187+
'./workers/evaluator/docker/Evaluator.Dockerfile',
188+
'./workers/notifier/docker/Notifier.Dockerfile',
189+
'./workers/reporter/docker/Reporter.Dockerfile',
190+
'./workers/scanner/docker/Scanner.Dockerfile'
191+
],
192+
labels=["ort-server"]
193+
)
194+
195+
custom_build(
196+
'advisor-worker-image',
197+
'./gradlew :workers:advisor:jibDockerBuild --image $EXPECTED_REF',
198+
live_update= [
199+
sync('./workers/advisor/build/classes/kotlin/main', '/app/classes')
200+
],
201+
deps=['./workers/advisor/build/classes', './workers/advisor/build.gradle.kts'],
202+
match_in_env_vars=True,
203+
)
204+
205+
custom_build(
206+
'analyzer-worker-image',
207+
'./gradlew :workers:analyzer:jibDockerBuild --image $EXPECTED_REF',
208+
live_update= [
209+
sync('./workers/analyzer/build/classes/kotlin/main', '/app/classes')
210+
],
211+
deps=['./workers/analyzer/build/classes', './workers/analyzer/build.gradle.kts'],
212+
match_in_env_vars=True,
213+
)
214+
215+
custom_build(
216+
'config-worker-image',
217+
'./gradlew :workers:config:jibDockerBuild --image $EXPECTED_REF',
218+
live_update= [
219+
sync('./workers/config/build/classes/kotlin/main', '/app/classes')
220+
],
221+
deps=['./workers/config/build/classes', './workers/config/build.gradle.kts'],
222+
match_in_env_vars=True,
223+
)
224+
225+
custom_build(
226+
'evaluator-worker-image',
227+
'./gradlew :workers:evaluator:jibDockerBuild --image $EXPECTED_REF',
228+
live_update= [
229+
sync('./workers/evaluator/build/classes/kotlin/main', '/app/classes')
230+
],
231+
deps=['./workers/evaluator/build/classes', './workers/evaluator/build.gradle.kts'],
232+
match_in_env_vars=True,
233+
)
234+
235+
custom_build(
236+
'notifier-worker-image',
237+
'./gradlew :workers:notifier:jibDockerBuild --image $EXPECTED_REF',
238+
live_update= [
239+
sync('./workers/notifier/build/classes/kotlin/main', '/app/classes')
240+
],
241+
deps=['./workers/notifier/build/classes', './workers/notifier/build.gradle.kts'],
242+
match_in_env_vars=True,
243+
)
244+
245+
custom_build(
246+
'reporter-worker-image',
247+
'./gradlew :workers:reporter:jibDockerBuild --image $EXPECTED_REF',
248+
live_update= [
249+
sync('./workers/reporter/build/classes/kotlin/main', '/app/classes')
250+
],
251+
deps=['./workers/reporter/build/classes', './workers/reporter/build.gradle.kts'],
252+
match_in_env_vars=True,
253+
)
254+
255+
custom_build(
256+
'scanner-worker-image',
257+
'./gradlew :workers:scanner:jibDockerBuild --image $EXPECTED_REF',
258+
live_update= [
259+
sync('./workers/scanner/build/classes/kotlin/main', '/app/classes')
260+
],
261+
deps=['./workers/scanner/build/classes', './workers/scanner/build.gradle.kts'],
262+
match_in_env_vars=True,
263+
)
264+
265+
helm_repo('grafana-repo', 'https://grafana.github.io/helm-charts', labels=['helm_repos'])
266+
267+
helm_resource(
268+
'loki-stack',
269+
'grafana/loki-stack',
270+
resource_deps=['grafana-repo'],
271+
namespace='ort-server',
272+
flags=[
273+
'--values=./scripts/kubernetes/loki-values.yaml',
274+
],
275+
deps=['./scripts/kubernetes/loki-values.yaml'],
276+
labels=['monitoring'],
277+
)
278+
279+
k8s_yaml('./scripts/kubernetes/alloy-config.yaml')
280+
281+
helm_resource(
282+
'alloy',
283+
'grafana/alloy',
284+
resource_deps=['grafana-repo'],
285+
namespace='ort-server',
286+
flags=[
287+
'--values=./scripts/kubernetes/grafana-alloy-values.yaml',
288+
],
289+
deps=['./scripts/kubernetes/grafana-alloy-values.yaml', './scripts/kubernetes/alloy-config.yaml'],
290+
labels=['monitoring'],
291+
)
292+
293+
k8s_resource(
294+
workload='loki-stack',
295+
port_forwards=["19000:3000"],
296+
extra_pod_selectors={'app.kubernetes.io/name': 'grafana'},
297+
discovery_strategy='selectors-only')
298+
299+
custom_build(
300+
'kubernetes-jobmonitor-image',
301+
'./gradlew :transport:kubernetes-jobmonitor:jibDockerBuild --image $EXPECTED_REF',
302+
live_update= [
303+
sync('./transport/kubernetes-jobmonitor/build/classes/kotlin/main', '/app/classes')
304+
],
305+
deps=[
306+
'./transport/kubernetes-jobmonitor/build/classes',
307+
'./transport/kubernetes-jobmonitor/build.gradle.kts',
308+
'./scripts/kubernetes/jobmonitor.application.conf'],
309+
match_in_env_vars=True,
310+
)
311+
312+
configmap_create('ort-jobmonitor-config',
313+
namespace='ort-server',
314+
from_file=['application.conf=./scripts/kubernetes/jobmonitor.application.conf'])
315+
316+
k8s_yaml('./scripts/kubernetes/kubernetes-jobmonitor.yaml')
317+
318+
k8s_resource(
319+
workload='kubernetes-jobmonitor',
320+
resource_deps=['rabbitmq', 'rabbitmq-terraform', 'postgresql'],
321+
labels=['ort-server'],
322+
)

‎scripts/kubernetes/README.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Setup
2+
3+
## Keycloak
4+
5+
Expoted the following clients from Docker Compose setup through the UI and imported them to the
6+
Kubernetes setup through the UI:
7+
8+
1. ort-server
9+
2. ort-server-ui
10+
3. ort-server-ui-dev
11+
12+
Changed ort-server client authentication on and added "Service accounts roles" as authentication
13+
flow. Created a client secret and added it the the manifest as "KEYCLOAK_API_SECRET". Added admin to
14+
service accounts roles.
15+
16+
Create client scope "ort-server-client" with "Display on consent screen" off.
17+
18+
Configure mapper:
19+
20+
- Mapper type: Audience.
21+
- Name: ORT-server-audience-mapping
22+
- Included Client Audience: ort-server
23+
24+
Add the client scope to ort-server-ui-dev.

0 commit comments

Comments
 (0)