Skip to content

Commit 9373eb0

Browse files
vitkyrkaclaude
vitkyrka
and
claude
authored
Run Weaviate E2E with the Kubernetes Agent backend (DataDog#24830)
* Run Weaviate E2E tests with the Kubernetes Agent backend Switch from host-side kube_port_forward to the new Kubernetes Agent E2E backend: the API endpoint uses Service DNS, the metrics endpoint falls back to the pod IP since no Service targets that port, and the readiness check plus data-seeding POST now run from disposable in-cluster pods since the host can no longer reach the cluster directly. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * Remove redundant readiness wait before seeding Weaviate data `/v1/.well-known/live` (DEFAULT_LIVENESS_ENDPOINT) is an unconditional 200 in Weaviate v1.20.0 with no actual readiness check, while the Pod `Ready` condition that `kubectl wait --for=condition=Ready` already blocks on is driven by the StatefulSet's readinessProbe (`/v1/.well-known/ready`, the real `DB.StartupComplete() && ClusterHealthScore() == 0` check). The extra busybox-based WaitFor poll was checking a strictly weaker signal than what setup_weaviate() already waits for, so it can't add any real confidence and was only burning disposable pods. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * Fail loudly if cluster setup doesn't become ready in setup_weaviate `run_command` defaults to `check=False`, so a timed-out or transient failure in the `kubectl create ns`/`apply`/`rollout status`/`wait` calls previously fell through silently into the pod-IP lookup and one-shot data-seeding curl, turning a clear "cluster never became Ready" failure into a confusing generic curl error. Add `check=True` to all four calls, matching the pattern already used for kuma's equivalent `setup_kuma()` (DataDog#24642). Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
1 parent bead066 commit 9373eb0

1 file changed

Lines changed: 74 additions & 51 deletions

File tree

‎weaviate/tests/conftest.py‎

Lines changed: 74 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,84 +3,107 @@
33
# Licensed under a 3-clause BSD style license (see LICENSE)
44
import json
55
import os
6-
import time
7-
from contextlib import ExitStack
86

97
import pytest
10-
import requests
118

129
from datadog_checks.dev import get_here
10+
from datadog_checks.dev._env import get_state, save_state
1311
from datadog_checks.dev.kind import kind_run
14-
from datadog_checks.dev.kube_port_forward import port_forward
1512
from datadog_checks.dev.subprocess import run_command
16-
from datadog_checks.weaviate.check import DEFAULT_LIVENESS_ENDPOINT
1713

1814
from .common import BATCH_OBJECTS, USE_AUTH
1915

2016
HERE = get_here()
2117
opj = os.path.join
2218

19+
NAMESPACE = 'weaviate'
20+
# No Service targets the StatefulSet's metrics port, only the API port (see weaviate_install.yaml /
21+
# weaviate_auth.yaml), so the API endpoint uses Service DNS while metrics fall back to the pod IP.
22+
WEAVIATE_API_ENDPOINT = f'http://weaviate.{NAMESPACE}.svc.cluster.local:80'
23+
POD_IP_STATE = 'weaviate_pod_ip'
24+
2325

2426
def setup_weaviate():
25-
run_command(['kubectl', 'create', 'ns', 'weaviate'])
27+
run_command(['kubectl', 'create', 'ns', 'weaviate'], check=True)
2628

2729
if USE_AUTH:
28-
run_command(['kubectl', 'apply', '-f', opj(HERE, 'kind', 'weaviate_auth.yaml'), '-n', 'weaviate'])
30+
run_command(['kubectl', 'apply', '-f', opj(HERE, 'kind', 'weaviate_auth.yaml'), '-n', 'weaviate'], check=True)
2931
else:
30-
run_command(['kubectl', 'apply', '-f', opj(HERE, 'kind', 'weaviate_install.yaml'), '-n', 'weaviate'])
32+
run_command(
33+
['kubectl', 'apply', '-f', opj(HERE, 'kind', 'weaviate_install.yaml'), '-n', 'weaviate'], check=True
34+
)
3135

3236
# Tries to ensure that the Kubernetes resources are deployed and ready before we do anything else
33-
run_command(['kubectl', 'rollout', 'status', 'statefulset/weaviate', '-n', 'weaviate'])
34-
run_command(['kubectl', 'wait', 'pods', '--all', '-n', 'weaviate', '--for=condition=Ready', '--timeout=600s'])
37+
run_command(['kubectl', 'rollout', 'status', 'statefulset/weaviate', '-n', 'weaviate'], check=True)
38+
run_command(
39+
['kubectl', 'wait', 'pods', '--all', '-n', 'weaviate', '--for=condition=Ready', '--timeout=600s'],
40+
check=True,
41+
)
42+
43+
# `setup_weaviate` only runs once, on the initial `ddev env start`. Later invocations of the
44+
# `dd_environment` fixture (e.g. during `ddev env stop`) run in a fresh process after the cluster
45+
# is torn down, so the pod IP is cached here via `save_state`/`get_state` rather than looked up live.
46+
save_state(POD_IP_STATE, get_weaviate_pod_ip())
47+
48+
make_weaviate_request()
49+
50+
51+
def get_weaviate_pod_ip() -> str:
52+
result = run_command(
53+
['kubectl', 'get', 'pods', '--namespace', NAMESPACE, '--selector', 'app=weaviate', '--output', 'json'],
54+
capture='out',
55+
check=True,
56+
)
57+
pods = json.loads(result.stdout)['items']
58+
if len(pods) != 1 or not pods[0].get('status', {}).get('podIP'):
59+
raise RuntimeError(f'Expected one ready Weaviate pod, found {len(pods)}')
60+
return pods[0]['status']['podIP']
61+
62+
63+
def make_weaviate_request():
64+
# This helps seed some dummy data in to Weaviate to make some metrics available. Run from a
65+
# temporary pod since the host cannot reach the cluster directly.
66+
weaviate_batch_endpoint = f'{WEAVIATE_API_ENDPOINT}/v1/batch/objects'
67+
68+
command = [
69+
'kubectl',
70+
'run',
71+
'weaviate-seed-data',
72+
'--namespace',
73+
NAMESPACE,
74+
'--image=curlimages/curl',
75+
'--restart=Never',
76+
'--attach',
77+
'--rm',
78+
'--quiet',
79+
'--',
80+
'curl',
81+
'-sf',
82+
'-X',
83+
'POST',
84+
weaviate_batch_endpoint,
85+
'-H',
86+
'Content-Type: application/json',
87+
]
88+
if USE_AUTH:
89+
command.extend(['-H', 'Authorization: Bearer test123'])
90+
command.extend(['-d', json.dumps(BATCH_OBJECTS)])
91+
92+
run_command(command, capture='both', check=True)
3593

3694

3795
@pytest.fixture(scope='session')
3896
def dd_environment():
39-
with kind_run(conditions=[setup_weaviate]) as kubeconfig, ExitStack() as stack:
40-
weaviate_host, weaviate_port = stack.enter_context(
41-
port_forward(kubeconfig, 'weaviate', 2112, 'statefulset', 'weaviate')
42-
)
43-
weaviate_host, weaviate_api_port = stack.enter_context(
44-
port_forward(kubeconfig, 'weaviate', 8080, 'statefulset', 'weaviate')
45-
)
97+
with kind_run(conditions=[setup_weaviate]) as kubeconfig:
98+
weaviate_metrics_port = 2112
4699

47100
instance = {
48-
'openmetrics_endpoint': f'http://{weaviate_host}:{weaviate_port}/metrics',
49-
'weaviate_api_endpoint': f'http://{weaviate_host}:{weaviate_api_port}',
101+
'openmetrics_endpoint': f'http://{get_state(POD_IP_STATE)}:{weaviate_metrics_port}/metrics',
102+
'weaviate_api_endpoint': WEAVIATE_API_ENDPOINT,
50103
}
51104
if USE_AUTH:
52105
instance['headers'] = {'Authorization': 'Bearer test123'}
53106

54-
make_weaviate_request(instance)
55-
yield instance
56-
57-
58-
def make_weaviate_request(instance):
59-
# This helps seed some dummy data in to Weaviate to make some metrics available
60-
weaviate_api_endpoint = instance.get('weaviate_api_endpoint')
61-
weaviate_batch_endpoint = f'{weaviate_api_endpoint}/v1/batch/objects'
62-
headers = {'content-type': 'application/json'}
63-
64-
if instance.get('headers'):
65-
headers.update(instance['headers'])
66-
67-
if ready_check(weaviate_api_endpoint, 300):
68-
requests.post(weaviate_batch_endpoint, headers=headers, data=json.dumps(BATCH_OBJECTS))
69-
70-
71-
def ready_check(endpoint, timeout=300):
72-
# Sometimes the API endpoint isn't ready when the cluster is ready. This will try to ensure the
73-
# API is ready for requests before we seed some dummy data.
74-
stop_time = time.time() + timeout
75-
endpoint = f'{endpoint}{DEFAULT_LIVENESS_ENDPOINT}'
76-
while time.time() < stop_time:
77-
try:
78-
response = requests.get(endpoint, timeout=5)
79-
if response.ok:
80-
return True
81-
except requests.RequestException as e:
82-
print(f'Request failed: {e}')
83-
84-
time.sleep(1)
107+
metadata = {'agent_type': 'kubernetes', 'kubernetes': {'kubeconfig': kubeconfig}}
85108

86-
return False
109+
yield instance, metadata

0 commit comments

Comments
 (0)