Skip to content

Commit 1b7faf9

Browse files
authored
[CRCR] Add event_type column to ClickHouse schema for nightly/periodic (#8353)
## Summary Prepares the data layer for nightly and periodic CI support ([RFC 98](pytorch/rfcs#98)). The `crcr_workflow_job` ClickHouse table currently has no way to distinguish between PR-triggered, nightly, and periodic CI runs. This PR: 1. **Schema**: Adds `event_type String DEFAULT 'pull_request'` column to `crcr_workflow_job` — existing rows automatically get `'pull_request'` via the DEFAULT 2. **Write path**: Extracts `event_type` from the callback payload in `crcrUtils.ts` and persists it through DynamoDB to ClickHouse 3. **Queries**: Adds `pr_number > 0` filter to all PR-related queries (`crcr_summary`, `crcr_success_rate`, `crcr_backend_dashboard`, `crcr_backend_summary`, `crcr_pr_results`) to prevent nightly/periodic rows from polluting PR dashboards 4. **Tests**: Adds tests for `event_type` extraction (PR, nightly, periodic, empty) ### Deployment note Run the following ALTER TABLE on ClickHouse before or after merge: ```sql ALTER TABLE default.crcr_workflow_job ADD COLUMN IF NOT EXISTS event_type String DEFAULT 'pull_request' COMMENT 'Dispatch event type: pull_request, nightly, periodic' AFTER downstream_repo_level ``` ### Data flow ``` Downstream CI → Relay Lambda → POST /api/crcr/results → DynamoDB → Replicator → ClickHouse ↑ event_type extracted here ``` ### Nightly HUD design The nightly results will be displayed on the CRCR summary page using a **Page-Level Tabs** design (PR | Nightly tabs at the top). **Mockup**: https://subinz1.github.io/CRCR/mockups/crcr-summary-nightly-design.html Fixes pytorch/crcr-test#17 ## Related PRs - RFC 98: pytorch/rfcs#98 - Nightly handler: #8302 - Callback action: #8303 - SHA validator: #8304 cc @atalman @jewelkm89
1 parent a4ba360 commit 1b7faf9

8 files changed

Lines changed: 39 additions & 0 deletions

File tree

clickhouse_db_schema/default.crcr_workflow_job/schema.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ CREATE TABLE default.crcr_workflow_job
2626
`artifact_url` String DEFAULT '' COMMENT 'URL to downstream-hosted artifacts (logs, reports)',
2727
`environment` String DEFAULT '' COMMENT 'JSON: {"sdk": "<version>", "device": "<hardware>", ...}',
2828
`downstream_repo_level` String DEFAULT '' COMMENT 'Relay level at dispatch time: L2, L3, L4',
29+
`event_type` String DEFAULT 'pull_request' COMMENT 'Dispatch event type: pull_request, nightly, periodic',
2930
`_inserted_at` DateTime MATERIALIZED now(),
3031
`repository_full_name` String ALIAS downstream_repo COMMENT 'Alias for consistency with workflow_job queries',
3132
`duration_seconds` Float64 ALIAS if(completed_at = toDateTime64(0, 9), 0, dateDiff(second, started_at, completed_at)),

torchci/clickhouse_queries/crcr_backend_dashboard/query.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,14 @@ FROM
2525
WHERE
2626
downstream_repo = {repo: String}
2727
AND started_at > now() - INTERVAL {days: UInt64} DAY
28+
AND pr_number > 0
2829
AND pr_number IN (
2930
SELECT pr_number
3031
FROM default.crcr_workflow_job FINAL
3132
WHERE
3233
downstream_repo = {repo: String}
3334
AND started_at > now() - INTERVAL {days: UInt64} DAY
35+
AND pr_number > 0
3436
GROUP BY pr_number
3537
ORDER BY max(started_at) DESC
3638
LIMIT

torchci/clickhouse_queries/crcr_backend_summary/query.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ SELECT
1818
downstream_repo = {repo: String}
1919
AND started_at > now() - INTERVAL {days: UInt64} DAY
2020
AND status = 'completed'
21+
AND pr_number > 0
2122
GROUP BY pr_number, job_name
2223
HAVING
2324
countIf(conclusion = 'success') > 0
@@ -30,3 +31,4 @@ WHERE
3031
downstream_repo = {repo: String}
3132
AND started_at > now() - INTERVAL {days: UInt64} DAY
3233
AND status = 'completed'
34+
AND pr_number > 0

torchci/clickhouse_queries/crcr_pr_results/query.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ FROM
1717
default.crcr_workflow_job FINAL
1818
WHERE
1919
pr_number = {pr: UInt64}
20+
AND pr_number > 0
2021
ORDER BY
2122
downstream_repo, started_at DESC
2223
LIMIT 100

torchci/clickhouse_queries/crcr_success_rate/query.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ FROM
1111
WHERE
1212
started_at > now() - INTERVAL {days: UInt64} DAY
1313
AND status = 'completed'
14+
AND pr_number > 0
1415
GROUP BY
1516
day, repo
1617
ORDER BY

torchci/clickhouse_queries/crcr_summary/query.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ FROM
1313
WHERE
1414
started_at > now() - INTERVAL {days: UInt64} DAY
1515
AND status = 'completed'
16+
AND pr_number > 0
1617
GROUP BY
1718
repo
1819
ORDER BY

torchci/lib/crcr/crcrUtils.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ export interface CrcrWorkflowJobRecord {
7979
failed_tests?: number;
8080
skipped_tests?: number;
8181
downstream_repo_level?: string;
82+
event_type?: string;
8283
artifact_url?: string;
8384
environment?: string;
8485
}
@@ -133,6 +134,10 @@ export function extractDynamoRecord(
133134
record.downstream_repo_level = trusted.downstream_repo_level;
134135
}
135136

137+
if (cb.event_type) {
138+
record.event_type = cb.event_type;
139+
}
140+
136141
// Only set timing metrics when the relay provides a non-null value.
137142
// in_progress sets queue_time; completed sets execution_time.
138143
// Using UpdateItem ensures the completed callback doesn't clobber

torchci/test/crcrUtils.test.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,32 @@ describe("extractDynamoRecord", () => {
8282
expect(record.downstream_repo_level).toBeUndefined();
8383
});
8484

85+
test("sets event_type from callback payload", () => {
86+
const record = extractDynamoRecord(makePayload());
87+
expect(record.event_type).toBe("workflow_job");
88+
});
89+
90+
test("sets event_type for nightly callback", () => {
91+
const record = extractDynamoRecord(
92+
makePayload({ callback: { event_type: "nightly" } })
93+
);
94+
expect(record.event_type).toBe("nightly");
95+
});
96+
97+
test("sets event_type for periodic callback", () => {
98+
const record = extractDynamoRecord(
99+
makePayload({ callback: { event_type: "periodic" } })
100+
);
101+
expect(record.event_type).toBe("periodic");
102+
});
103+
104+
test("omits event_type when empty", () => {
105+
const record = extractDynamoRecord(
106+
makePayload({ callback: { event_type: "" } })
107+
);
108+
expect(record.event_type).toBeUndefined();
109+
});
110+
85111
test("sets queue_time from ci_metrics when non-null", () => {
86112
const record = extractDynamoRecord(makePayload());
87113
expect(record.queue_time).toBe(12.5);

0 commit comments

Comments
 (0)