Skip to content

Commit 149065f

Browse files
adolfo-abadolfo-ab
authored
feat: workflow to run tests in kind
feat: workflow to run tests in kind
2 parents 5edcb6e + e456929 commit 149065f

18 files changed

+16572
-5
lines changed

‎.github/workflows/test-kind.yaml‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
name: Run Kind Tests
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Build and Push Distro Image"]
6+
types:
7+
- completed
8+
pull_request:
9+
branches:
10+
- main
11+
12+
env:
13+
REGISTRY: quay.io
14+
ORG: trustyai_testing
15+
IMAGE_NAME: llama-stack-trustyai-fms
16+
17+
jobs:
18+
deploy:
19+
runs-on: ubuntu-latest
20+
if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' }}
21+
env:
22+
PR_NUMBER: ${{ github.event.pull_request.number || 'default-pr-number' }}
23+
steps:
24+
- name: Check labels
25+
uses: mheap/github-action-required-labels@v5
26+
with:
27+
mode: minimum
28+
count: 1
29+
labels: "ok-to-test, lgtm, approved"
30+
31+
- name: Checkout
32+
uses: actions/checkout@v2
33+
34+
- name: Create kind config
35+
run: |
36+
cat > kind-config.yaml << EOF
37+
kind: Cluster
38+
apiVersion: kind.x-k8s.io/v1alpha4
39+
containerdConfigPatches:
40+
- |-
41+
[plugins."io.containerd.grpc.v1.cri".registry]
42+
config_path = "/etc/containerd/certs.d"
43+
nodes:
44+
- role: control-plane
45+
kubeadmConfigPatches:
46+
- |
47+
kind: InitConfiguration
48+
nodeRegistration:
49+
kubeletExtraArgs:
50+
system-reserved: memory=1Gi
51+
eviction-hard: memory.available<500Mi
52+
- |
53+
kind: ClusterConfiguration
54+
apiServer:
55+
extraArgs:
56+
disable-admission-plugins: "PodSecurity"
57+
extraPortMappings:
58+
- containerPort: 80
59+
hostPort: 80
60+
protocol: TCP
61+
- containerPort: 443
62+
hostPort: 443
63+
protocol: TCP
64+
EOF
65+
66+
- name: Create k8s Kind Cluster
67+
id: kind
68+
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
69+
with:
70+
registry: true
71+
registry_name: kind-registry
72+
registry_port: 5000
73+
registry_enable_delete: true
74+
config: kind-config.yaml
75+
wait: 120s
76+
77+
- name: Clone llama-stack-k8s-operator
78+
run: |
79+
git clone https://github.com/opendatahub-io/llama-stack-k8s-operator.git
80+
cd llama-stack-k8s-operator
81+
git checkout odh
82+
83+
- name: Build llama-stack-k8s-operator and push to Kind registry
84+
run: |
85+
cd llama-stack-k8s-operator
86+
docker build -t kind-registry:5000/llama-stack-k8s-operator:latest -f Dockerfile .
87+
docker push kind-registry:5000/llama-stack-k8s-operator:latest
88+
89+
- name: Deploy llama-stack-k8s-operator
90+
run: |
91+
92+
cd llama-stack-k8s-operator
93+
make deploy IMG=kind-registry:5000/llama-stack-k8s-operator:latest
94+
95+
# Wait for operator deployment to be ready
96+
if ! kubectl wait --for=condition=available --timeout=300s deployment/llama-stack-k8s-operator-controller-manager -n llama-stack-k8s-operator-system; then
97+
echo "Deployment failed to become ready. Debugging information:"
98+
kubectl describe deployment llama-stack-k8s-operator-controller-manager -n llama-stack-k8s-operator-system
99+
kubectl logs -l control-plane=controller-manager -n llama-stack-k8s-operator-system --tail=100
100+
kubectl get events -n system --sort-by='.lastTimestamp'
101+
exit 1
102+
fi
103+
104+
- name: Pull FMS provider image and push to kind registry
105+
run: |
106+
docker pull ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE_NAME }}:pr-${{ env.PR_NUMBER }}
107+
docker tag ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE_NAME }}:pr-${{ env.PR_NUMBER }} kind-registry:5000/llama-stack-trustyai-fms:latest
108+
docker push kind-registry:5000/llama-stack-trustyai-fms:latest
109+
110+
- name: Load TrustyAI operator image into Kind
111+
run: |
112+
docker pull quay.io/trustyai/trustyai-service-operator:latest
113+
docker tag quay.io/trustyai/trustyai-service-operator:latest kind-registry:5000/trustyai-service-operator:latest
114+
docker push kind-registry:5000/trustyai-service-operator:latest
115+
116+
- name: Load VLLM emulator image into Kind
117+
run: |
118+
docker pull quay.io/trustyai_testing/vllm_emulator:latest
119+
docker tag quay.io/trustyai_testing/vllm_emulator:latest kind-registry:5000/vllm_emulator:latest
120+
docker push kind-registry:5000/vllm_emulator:latest
121+
122+
- name: Install kustomize
123+
run: |
124+
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
125+
sudo mv kustomize /usr/local/bin/
126+
127+
- name: Clone TrustyAI operator repository
128+
run: |
129+
git clone https://github.com/trustyai-explainability/trustyai-service-operator.git
130+
cd trustyai-service-operator
131+
git checkout main
132+
133+
- name: Apply CRDs
134+
run: |
135+
kubectl apply -f tests/kind/manifests/route_crd.yaml
136+
kubectl apply -f tests/kind/manifests/monitoring.coreos.com_servicemonitors.yaml
137+
kubectl apply -f tests/kind/manifests/serving.kserve.io_inferenceservices.yaml
138+
139+
kubectl apply -k https://github.com/llamastack/llama-stack-k8s-operator/config/crd
140+
141+
cd trustyai-service-operator
142+
kustomize build config/crd | kubectl apply -f -
143+
144+
- name: Deploy TrustyAI operator
145+
run: |
146+
kubectl create namespace system
147+
148+
cd trustyai-service-operator
149+
kustomize build config/base | kubectl apply -n system -f -
150+
151+
- name: Run Kind Tests
152+
run: ./tests/kind/test_kind.sh
153+
154+
- name: Get logs
155+
if: ${{ always() }}
156+
run: |
157+
# Test namespace resources and logs
158+
kubectl -n test get all -o yaml > test-ns-logs.log
159+
kubectl -n test describe all > test-ns-describe.log
160+
kubectl -n test describe events > test-ns-events.log
161+
162+
# LlamaStack-related logs
163+
kubectl -n llama-stack-k8s-operator-system logs deployment.apps/llama-stack-k8s-operator-controller-manager > lls-controller-manager.log
164+
kubectl -n test get llamastackdistributions -o yaml > lls-dist.log
165+
kubectl -n test logs -l app.kubernetes.io/instance=llamastack-custom-distribution --all-containers=true > llama-stack-distribution-pods.log || echo "No llama stack distribution pods found" > lls-dist-pod-logs.log
166+
167+
# TrustyAI operator logs and resources
168+
kubectl -n system logs -l control-plane=controller-manager > trustyai-operator.log || echo "No TrustyAI operator logs available" > trustyai-operator.log
169+
170+
# GuardrailsOrchestrator resources
171+
kubectl -n test describe GuardrailsOrchestrator > guardrails-orchestrator-describe.log || echo "No GuardrailsOrchestrator in test namespace" > guardrails-orchestrator-describe.log
172+
173+
# Cluster-wide information
174+
kubectl get pods --all-namespaces > all-pods.log
175+
kubectl get events --all-namespaces --sort-by='.lastTimestamp' > all-events.log
176+
177+
- name: Upload all logs to artifacts
178+
if: ${{ always() }}
179+
uses: actions/upload-artifact@v4
180+
with:
181+
name: logs-${{ github.run_id }}-${{ github.run_attempt }}
182+
path: |
183+
*.log
184+
retention-days: 1

.github/workflows/test.yml renamed to .github/workflows/test-unit.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Tests
1+
name: Run Unit Tests
22

33
on:
44
pull_request:
@@ -25,4 +25,4 @@ jobs:
2525
2626
- name: Run tests
2727
run: |
28-
python -m pytest tests/ -v
28+
python -m pytest tests/unit/ -v

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ readme = "README.md"
1616
requires-python = ">=3.12"
1717
license = "Apache-2.0"
1818
dependencies = [
19-
"llama-stack>=0.2.14",
2019
"fastapi",
2120
"opentelemetry-api",
2221
"opentelemetry-exporter-otlp",
@@ -34,6 +33,7 @@ test = [
3433
"pytest-asyncio>=0.21.0",
3534
"pytest-httpx>=0.21.0",
3635
"pytest-mock>=3.10.0",
36+
"llama-stack==0.2.22",
3737
]
3838

3939
[tool.ruff]
@@ -74,4 +74,4 @@ include = ["llama_stack_provider_trustyai_fms*"]
7474
trustyai_fms = "llama_stack_provider_trustyai_fms.provider:get_provider_spec"
7575

7676
[project.entry-points."llama_stack.providers.remote.shields"]
77-
trustyai_fms = "llama_stack_provider_trustyai_fms.provider:get_shields_provider_spec"
77+
trustyai_fms = "llama_stack_provider_trustyai_fms.provider:get_shields_provider_spec"

‎run.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
version: "2"
2-
image_name: trustyai-fms
2+
image_name: rh
33
apis:
44
- safety
55
- shields

‎tests/kind/manifests/gorch.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: trustyai.opendatahub.io/v1alpha1
2+
kind: GuardrailsOrchestrator
3+
metadata:
4+
name: guardrails-orchestrator
5+
spec:
6+
orchestratorConfig: "fms-orchestr8-config-nlp"
7+
enableBuiltInDetectors: true
8+
enableGuardrailsGateway: false
9+
logLevel: DEBUG
10+
replicas: 1

‎tests/kind/manifests/gorch_cm.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
kind: ConfigMap
2+
apiVersion: v1
3+
metadata:
4+
name: fms-orchestr8-config-nlp
5+
data:
6+
config.yaml: |
7+
chat_generation:
8+
service:
9+
hostname: vllm-emulator-service.test.svc.cluster.local
10+
port: 8032
11+
detectors:
12+
regex:
13+
type: text_contents
14+
service:
15+
hostname: "127.0.0.1"
16+
port: 8080
17+
chunker_id: whole_doc_chunker
18+
default_threshold: 0.5

‎tests/kind/manifests/gorch_tls.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: guardrails-orchestrator-tls
5+
annotations:
6+
service.beta.openshift.io/serving-cert-secret-name: guardrails-orchestrator-tls
7+
type: kubernetes.io/tls
8+
data:
9+
# gg-ignore
10+
# dummy tls certs used for testing
11+
tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNLekNDQWRHZ0F3SUJBZ0lVYkttWnBQdmxYSU1oNWtjTHUrL25MaHl5UG1vd0NnWUlLb1pJemowRUF3SXcKYXpFTE1Ba0dBMVVFQmhNQ1ZWTXhEVEFMQmdOVkJBZ01CRlJsYzNReERUQUxCZ05WQkFjTUJGUmxjM1F4RFRBTApCZ05WQkFvTUJGUmxjM1F4RFRBTEJnTlZCQXNNQkZSbGMzUXhJREFlQmdOVkJBTU1GMmQxWVhKa2NtRnBiSE10CmIzSmphR1Z6ZEhKaGRHOXlNQjRYRFRJMU1Ea3lNekV6TkRrMU5Wb1hEVEkyTURreU16RXpORGsxTlZvd2F6RUwKTUFrR0ExVUVCaE1DVlZNeERUQUxCZ05WQkFnTUJGUmxjM1F4RFRBTEJnTlZCQWNNQkZSbGMzUXhEVEFMQmdOVgpCQW9NQkZSbGMzUXhEVEFMQmdOVkJBc01CRlJsYzNReElEQWVCZ05WQkFNTUYyZDFZWEprY21GcGJITXRiM0pqCmFHVnpkSEpoZEc5eU1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRVpoNlBRbW5yRmFadGpFUzYKaC9UejI4TktDaWF3TlNFdWd3dkxvQmlUZm9CRmZURjVwU1FOWXptbmNjM05DUGJTQlhIRHVsdmFzcDV0VHJ0QgpGNHVXVXFOVE1GRXdIUVlEVlIwT0JCWUVGUFduZnRjZnhlckZCY0pBczNuMlIvKzQzTmVFTUI4R0ExVWRJd1FZCk1CYUFGUFduZnRjZnhlckZCY0pBczNuMlIvKzQzTmVFTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3Q2dZSUtvWkkKemowRUF3SURTQUF3UlFJZ0drM2ZzcnlibGZ0N29GbWJsNmZEMC94L3BLMUUrMGo3RXZ1TGNJOVhuOFFDSVFEMQp0YnU4aVBzQjRYUVM5ZkVXNCtMYkdEd2pVYjRrTU5YQURGQlI4ZEF0RUE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== # notsecret
12+
tls.key: LS0tLS1CRUdJTiBFQyBQQVJBTUVURVJTLS0tLS0KQmdncWhrak9QUU1CQnc9PQotLS0tLUVORCBFQyBQQVJBTUVURVJTLS0tLS0KLS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUlGdlFGeUJHSlJXZlUwTFVjYUQyZnNqNDQ0TjVxNkswNHIyMmt3ZTY2RGdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWmg2UFFtbnJGYVp0akVTNmgvVHoyOE5LQ2lhd05TRXVnd3ZMb0JpVGZvQkZmVEY1cFNRTgpZem1uY2MzTkNQYlNCWEhEdWx2YXNwNXRUcnRCRjR1V1VnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= # notsecret

‎tests/kind/manifests/llama_stack_distribution.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
apiVersion: llamastack.io/v1alpha1
2+
kind: LlamaStackDistribution
3+
metadata:
4+
name: llamastack-custom-distribution
5+
spec:
6+
replicas: 1
7+
server:
8+
containerSpec:
9+
env:
10+
- name: MODULE_VERSION
11+
value: '0.2.2'
12+
- name: LLAMA_STACK_CONFIG_DIR
13+
value: /opt/app-root/src/.llama/distributions/rh/
14+
- name: VLLM_URL
15+
value: 'http://vllm-emulator-service.test.svc.cluster.local:8032'
16+
- name: INFERENCE_MODEL
17+
value: vllm-emulator
18+
- name: MILVUS_DB_PATH
19+
value: ~/.llama/milvus.db
20+
- name: VLLM_TLS_VERIFY
21+
value: 'false'
22+
- name: FMS_ORCHESTRATOR_URL
23+
value: 'https://guardrails-orchestrator.test.svc.cluster.local:8032'
24+
name: llama-stack
25+
port: 8321
26+
distribution:
27+
image: "kind-registry:5000/llama-stack-trustyai-fms:latest"

0 commit comments

Comments
 (0)