Skip to content

Commit 83f58f3

Browse files
authored
Add buildkite-webhook-handler lambda to ingest webhook events from Buildkite (#6998)
This lambda receives webhook events from vLLM so that we can build HUD-like dashboard there. https://app.hex.tech/533fe68e-dcd8-4a52-a101-aefba762f581/app/030kdEgDv6lSlh1UPYOkWP is an early example from Simon. ### Testing I have manually created `buildkite-webhook-handler-debug` lambda and writing into `vllm-buildkite-*` dynamo table since few weeks back. Also create a test release for the lambda at https://github.com/pytorch/test-infra/actions/runs/16928003507/job/47967490767 --------- Signed-off-by: Huy Do <huydhn@gmail.com>
1 parent d698472 commit 83f58f3

6 files changed

Lines changed: 522 additions & 4 deletions

File tree

.github/workflows/_lambda-do-release-runners.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,13 @@ jobs:
8585
fail-fast: false
8686
matrix:
8787
include: [
88-
{ dir-name: 'ci-queue-pct', zip-name: 'ci-queue-pct' },
89-
{ dir-name: 'oss_ci_job_queue_time', zip-name: 'oss-ci-job-queue-time' },
90-
{ dir-name: 'oss_ci_cur', zip-name: 'oss-ci-cur' },
88+
{ dir-name: 'ci-queue-pct', zip-name: 'ci-queue-pct' },
89+
{ dir-name: 'oss_ci_job_queue_time', zip-name: 'oss-ci-job-queue-time' },
90+
{ dir-name: 'oss_ci_cur', zip-name: 'oss-ci-cur' },
9191
{ dir-name: 'benchmark-results-uploader', zip-name: 'benchmark-results-uploader' },
92-
{ dir-name: 'pytorch-auto-revert', zip-name: 'pytorch-auto-revert' },
92+
{ dir-name: 'pytorch-auto-revert', zip-name: 'pytorch-auto-revert' },
9393
{ dir-name: 'keep-going-call-log-classifier', zip-name: 'keep-going-call-log-classifier' },
94+
{ dir-name: 'buildkite-webhook-handler', zip-name: 'buildkite-webhook-handler' },
9495
]
9596
name: Upload Release for ${{ matrix.dir-name }} lambda
9697
runs-on: ubuntu-latest
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
all: run-local
2+
3+
clean:
4+
rm -rf deployment
5+
rm -rf venv
6+
rm -rf deployment.zip
7+
8+
venv/bin/python:
9+
virtualenv venv
10+
venv/bin/pip install -r requirements.txt
11+
12+
deployment.zip:
13+
mkdir -p deployment
14+
cp lambda_function.py ./deployment/.
15+
pip3.10 install -r requirements.txt -t ./deployment/. --platform manylinux2014_x86_64 --only-binary=:all: --implementation cp --python-version 3.10 --upgrade
16+
cd ./deployment && zip -q -r ../deployment.zip .
17+
18+
.PHONY: create-deployment-package
19+
create-deployment-package: deployment.zip
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Buildkite Webhook Handler Lambda
2+
3+
This Lambda function receives and processes Buildkite webhook events for
4+
all available Buildkite webhook events, saving them to DynamoDB tables.
5+
6+
* In the near-term, this allows vLLM maintainers to explore their CI data
7+
like time to signals or queueing time.
8+
* In the longer-term, this will provide the foundation for future UX projects
9+
on vLLM like vLLM HUD, CI failures notifications.
10+
11+
## Overview
12+
13+
The lambda handles two types of Buildkite webhook events:
14+
- **Agent events** (`agent.*`) - Saved to `vllm-buildkite-agent-events` table
15+
- **Build events** (`build.*`) - Saved to `vllm-buildkite-build-events` table
16+
- **Job events** (`job.*`) - Saved to `vllm-buildkite-job-events` table
17+
18+
## DynamoDB Schema
19+
20+
### Agent Events Table: `vllm-buildkite-agent-events`
21+
- **Partition Key**: `dynamoKey` (format: `AGENT_ID`)
22+
- https://buildkite.com/docs/apis/webhooks/pipelines/agent-events
23+
24+
### Build Events Table: `vllm-buildkite-build-events`
25+
- **Partition Key**: `dynamoKey` (format: `REPO_NAME/PIPELINE_NAME/BUILD_NUMBER`)
26+
- https://buildkite.com/docs/apis/webhooks/pipelines/build-events
27+
28+
### Job Events Table: `vllm-buildkite-job-events`
29+
- **Partition Key**: `dynamoKey` (format: `REPO_NAME/JOB_ID`)
30+
- https://buildkite.com/docs/apis/webhooks/pipelines/job-events
31+
32+
## Deployment
33+
34+
```bash
35+
make create-deployment-package
36+
```
37+
38+
This creates a `deployment.zip` file ready for AWS Lambda deployment.
39+
40+
## Event Processing
41+
42+
The lambda automatically:
43+
1. Identifies event type from webhook payload
44+
2. Extracts repository name and relevant IDs
45+
3. Saves to appropriate DynamoDB table with structured key
46+
4. Returns success/error response
47+
48+
## Error Handling
49+
50+
- Invalid JSON payloads return 400 status
51+
- Missing required fields return 400 status
52+
- DynamoDB errors return 500 status
53+
- Unsupported event types return 400 status
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
import json
2+
from typing import Any, Dict
3+
4+
import boto3
5+
from botocore.exceptions import ClientError
6+
7+
8+
dynamodb = boto3.resource("dynamodb")
9+
agent_events_table = dynamodb.Table("vllm-buildkite-agent-events")
10+
build_events_table = dynamodb.Table("vllm-buildkite-build-events")
11+
job_events_table = dynamodb.Table("vllm-buildkite-job-events")
12+
13+
14+
def save_agent_event(event_data: Dict[str, Any]) -> Dict[str, Any]:
15+
"""
16+
Save agent events to DynamoDB table.
17+
18+
Args:
19+
event_data: The agent event payload from Buildkite
20+
21+
Returns:
22+
Dict[str, Any]: Response containing status and result information
23+
"""
24+
try:
25+
agent = event_data.get("agent", {})
26+
agent_id = agent.get("id", "")
27+
28+
if not agent_id:
29+
return {
30+
"statusCode": 400,
31+
"body": json.dumps({"message": "Missing agent ID"}),
32+
}
33+
34+
dynamo_key = agent_id
35+
item = {"dynamoKey": dynamo_key, **event_data}
36+
37+
agent_events_table.put_item(Item=item)
38+
39+
return {
40+
"statusCode": 200,
41+
"body": json.dumps(
42+
{"message": f"Agent event saved successfully with key: {dynamo_key}"}
43+
),
44+
}
45+
46+
except ClientError as e:
47+
return {
48+
"statusCode": 500,
49+
"body": json.dumps({"message": f"DynamoDB error: {str(e)}"}),
50+
}
51+
except Exception as e:
52+
return {
53+
"statusCode": 500,
54+
"body": json.dumps({"message": f"Error saving agent event: {str(e)}"}),
55+
}
56+
57+
58+
def save_build_event(event_data: Dict[str, Any]) -> Dict[str, Any]:
59+
"""
60+
Save build event to DynamoDB table.
61+
62+
Args:
63+
event_data: The build event payload from Buildkite
64+
65+
Returns:
66+
Dict[str, Any]: Response containing status and result information
67+
"""
68+
try:
69+
build = event_data.get("build", {})
70+
repo_name = event_data.get("pipeline", {}).get("repository", "").split("/")[-1]
71+
pipeline_name = event_data.get("pipeline", {}).get("name", "")
72+
build_number = build.get("number", "")
73+
74+
if not repo_name or not build_number:
75+
return {
76+
"statusCode": 400,
77+
"body": json.dumps(
78+
{"message": "Missing repository name or build number"}
79+
),
80+
}
81+
82+
# Buildkite build_number is only unique in a pipeline
83+
dynamo_key = f"{repo_name}/{pipeline_name}/{build_number}"
84+
85+
item = {"dynamoKey": dynamo_key, **event_data}
86+
build_events_table.put_item(Item=item)
87+
88+
return {
89+
"statusCode": 200,
90+
"body": json.dumps(
91+
{"message": f"Build event saved successfully with key: {dynamo_key}"}
92+
),
93+
}
94+
95+
except ClientError as e:
96+
return {
97+
"statusCode": 500,
98+
"body": json.dumps({"message": f"DynamoDB error: {str(e)}"}),
99+
}
100+
except Exception as e:
101+
return {
102+
"statusCode": 500,
103+
"body": json.dumps({"message": f"Error saving build event: {str(e)}"}),
104+
}
105+
106+
107+
def save_job_event(event_data: Dict[str, Any]) -> Dict[str, Any]:
108+
"""
109+
Save job event to DynamoDB table.
110+
111+
Args:
112+
event_data: The job event payload from Buildkite
113+
114+
Returns:
115+
Dict[str, Any]: Response containing status and result information
116+
"""
117+
try:
118+
job = event_data.get("job", {})
119+
repo_name = event_data.get("pipeline", {}).get("repository", "").split("/")[-1]
120+
job_id = job.get("id", "")
121+
122+
if not repo_name or not job_id:
123+
return {
124+
"statusCode": 400,
125+
"body": json.dumps({"message": "Missing repository name or job ID"}),
126+
}
127+
128+
dynamo_key = f"{repo_name}/{job_id}"
129+
130+
item = {"dynamoKey": dynamo_key, **event_data}
131+
132+
job_events_table.put_item(Item=item)
133+
134+
return {
135+
"statusCode": 200,
136+
"body": json.dumps(
137+
{"message": f"Job event saved successfully with key: {dynamo_key}"}
138+
),
139+
}
140+
141+
except ClientError as e:
142+
return {
143+
"statusCode": 500,
144+
"body": json.dumps({"message": f"DynamoDB error: {str(e)}"}),
145+
}
146+
except Exception as e:
147+
return {
148+
"statusCode": 500,
149+
"body": json.dumps({"message": f"Error saving job event: {str(e)}"}),
150+
}
151+
152+
153+
def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
154+
"""
155+
Main Lambda handler function for Buildkite webhook events.
156+
157+
Args:
158+
event: Contains the webhook payload from Buildkite
159+
context: Provides runtime information about the Lambda function
160+
161+
Returns:
162+
Dict[str, Any]: Response containing status and result information
163+
"""
164+
try:
165+
if event.get("body"):
166+
body = json.loads(event["body"])
167+
else:
168+
body = event
169+
170+
event_type = body.get("event")
171+
172+
if not event_type:
173+
return {
174+
"statusCode": 400,
175+
"body": json.dumps(
176+
{"message": "Missing event type in webhook payload"}
177+
),
178+
}
179+
180+
if event_type.startswith("agent."):
181+
return save_agent_event(body)
182+
elif event_type.startswith("build."):
183+
return save_build_event(body)
184+
elif event_type.startswith("job."):
185+
return save_job_event(body)
186+
else:
187+
return {
188+
"statusCode": 400,
189+
"body": json.dumps(
190+
{"message": f"Unsupported event type: {event_type}"}
191+
),
192+
}
193+
194+
except json.JSONDecodeError as e:
195+
return {
196+
"statusCode": 400,
197+
"body": json.dumps({"message": f"Invalid JSON payload: {str(e)}"}),
198+
}
199+
except Exception as e:
200+
return {
201+
"statusCode": 500,
202+
"body": json.dumps({"message": f"Unexpected error: {str(e)}"}),
203+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
boto3==1.36.21

0 commit comments

Comments
 (0)