foo

Author: SonSang

.github/workflows/alarm.yml

@@ -74,3 +74,37 @@ jobs:
           ARTIFACTS_DIR: ${{ steps.dl.outputs.download-path }}
         run: |
           python test_wandb.py
+
+          # if pr_comment.md exists, inject it to the next step as an ENV
+          if [ -f "pr_comment.md" ]; then
+            {
+              echo 'SCRIPT_OUTPUT<<__EOF__'
+              cat pr_comment.md
+              echo '__EOF__'
+            } >> "$GITHUB_ENV"
+          else
+            echo "SCRIPT_OUTPUT=" >> "$GITHUB_ENV"
+          fi
+
+      - name: Add PR comment
+        if: ${{ env.SCRIPT_OUTPUT != '' }}
+        uses: actions/github-script@v7
+        env:
+          COMMENT_BODY: ${{ env.SCRIPT_OUTPUT }}
+        with:
+          script: |
+            // workflow_run event does not have context.issue -> extract PR from payload
+            const prs = (context.payload.workflow_run.pull_requests || []);
+            if (!prs.length) {
+              core.info('No associated PR; skipping comment.');
+              return;
+            }
+            const prNumber = prs[0].number;
+
+            await github.rest.issues.createComment({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: process.env.COMMENT_BODY
+            });
+

test_wandb.py

@@ -7,20 +7,20 @@
 
 
 def _norm_rev(text):
-    """revision 문자열에서 SHA 추출 + '@...' 제거"""
+    # Normalize revision string to SHA
     if not text:
         return None
     text = text.split("@", 1)[0]
     m = SHA_RE.search(text)
     return m.group(0) if m else text
 
 
-# 공용: dict(k->v)을 "k=v-k=v-..." 형태의 정규화된 ID로 변환
+# Common: convert dict(k->v) to normalized ID in "k=v-k=v-..." format
 def _normalize_kv_id(kv: dict, drop_keys=None) -> str:
     drop = set(drop_keys or [])
     pairs = []
     for k, v in kv.items():
-        if k in drop:  # 메트릭/불필요 키 제거
+        if k in drop:
             continue
         if v is None:
             continue
@@ -29,33 +29,33 @@ def _normalize_kv_id(kv: dict, drop_keys=None) -> str:
         if not k or not v:
             continue
         pairs.append((k, v))
-    pairs.sort(key=lambda x: x[0])  # 키 기준 정렬
+    pairs.sort(key=lambda x: x[0])
     return "-".join(f"{k}={v}" for k, v in pairs)
 
 
-# 1) W&B benchmark_id (예: "rigid_body-batch_size=...-env=...-...") -> 정규화 ID
-def normalize_benchmark_id(bid: str) -> str:
+# 1) W&B benchmark_id (e.g. "rigid_body-batch_size=...-env=...-...") -> normalized ID
+def wandb_normalize_benchmark_id(bid: str) -> str:
     """
     'rigid_body-batch_size=30000-constraint_solver=Newton-env=batched_franka-...'
-    형태에서 key=value 토큰만 뽑아 사전식 정렬 후 '-'로 이어 붙인 ID를 반환.
+    from key=value tokens, sort alphabetically, and join with '-'
     """
     kv = {}
     for token in (p.strip() for p in bid.split("-") if p.strip()):
         if "=" not in token:
-            # 접두 토큰(예: 'rigid_body')은 버림
+            # prefix token (e.g. 'rigid_body') is dropped
             continue
         k, v = token.split("=", 1)
         kv[k.strip()] = v.strip()
-    # W&B쪽은 metrics가 포함되지 않지만, 규칙 통일을 위해 drop은 비워 둠
+    # W&B side does not include metrics, but for consistency, drop is left empty
     return _normalize_kv_id(kv)
 
 
-# 2) speed_test*.txt 라인들을 파싱해 {id: {runtime_fps, compile_time}} 맵으로 변환
-def parse_speed_txt_lines(lines):
+# 2) speed_test*.txt lines to {id: {runtime_fps, compile_time}} map
+def artifacts_parse_speed_txt_lines(lines):
     """
-    speed_test.txt 내용을 받아
-    { normalized_id: {"runtime_fps": float|None, "compile_time": float|None} } 반환.
-    normalized_id 규칙은 normalize_benchmark_id와 동일.
+    speed_test.txt content to
+    { normalized_id: {"runtime_fps": float|None, "compile_time": float|None} } return.
+    normalized_id rule is the same as wandb_normalize_benchmark_id.
     """
     METRIC_KEYS = {"compile_time", "runtime_fps", "realtime_factor"}
 
@@ -64,17 +64,17 @@ def parse_speed_txt_lines(lines):
         if not line.strip():
             continue
 
-        # "env=... | batch_size=... | ..." 형태 파싱
+        # "env=... | batch_size=... | ..." format parsing
         parts = [p.strip() for p in line.strip().split("|") if "=" in p]
         kv = {}
         for p in parts:
             k, v = p.split("=", 1)
             kv[k.strip()] = v.strip()
 
-        # 정규화된 테스트 ID: metrics 키는 제외하고 W&B와 동일 규칙 적용
+        # Normalized test ID: metrics key is excluded, and the same rule as W&B
         test_id = _normalize_kv_id(kv, drop_keys=METRIC_KEYS)
 
-        # 수치 파싱
+        # Numerical parsing
         rt = kv.get("runtime_fps")
         ct = kv.get("compile_time")
         try:
@@ -114,7 +114,7 @@ def parse_speed_txt_lines(lines):
 
 with open(current_txt_path, "r", encoding="utf-8") as f:
     current_txt_lines = f.readlines()
-    current_benchmark = parse_speed_txt_lines(current_txt_lines)
+    current_benchmark = artifacts_parse_speed_txt_lines(current_txt_lines)
 
 # ---------- Read event / find PR ----------
 API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
@@ -136,7 +136,7 @@ def parse_speed_txt_lines(lines):
 prs = wr.get("pull_requests") or []
 pr = prs[0] if prs else None
 if not pr:
-    # 커밋에서 PR 역조회
+    # Find PR from commit
     url = f"{API}/repos/{owner}/{name}/commits/{head_sha}/pulls"
     r = s.get(url, headers={"Accept": "application/vnd.github.groot-preview+json"})
     if r.ok and r.json():
@@ -167,7 +167,7 @@ def parse_speed_txt_lines(lines):
 runs_iter = api.runs(f"{ENTITY}/{PROJECT}", order="-created_at")
 
 by_rev = {}  # revision -> {benchmark_id: {runtime_fps, compile_time}}
-rev_order = []  # 최신 -> 과거
+rev_order = []  # latest -> oldest
 selected_revs = None
 no_change_streak = 0
 
@@ -220,7 +220,7 @@ def parse_speed_txt_lines(lines):
             if rowcnt >= 10:
                 break
 
-        nbid = normalize_benchmark_id(bid)
+        nbid = wandb_normalize_benchmark_id(bid)
 
         by_rev[rev][nbid] = {
             "runtime_fps": runtime_fps,
@@ -294,6 +294,8 @@ def trunc(s, n=120):
 lines = []
 lines.append(":warning: **Benchmark regression detected (vs W&B history)**")
 lines.append(f"- Revisions considered: **{len(rev_order)}**")
+for i in range(len(rev_order)):
+    lines.append(f"\t- Revision {i+1}: {rev_order[i]}")
 lines.append(f"- Runtime tolerance: **-{tol_rt:.1f}%**; Compile tolerance: **+{tol_ct:.1f}%**")
 lines.append("")
 
@@ -335,5 +337,15 @@ def trunc(s, n=120):
 
 print(body)
 
-resp = s.post(f"{target_repo_api}/issues/{pr_num}/comments", json={"body": body})
-print("Comment status:", resp.status_code, resp.text[:200])
+# ---------- Save to a local file; DO NOT POST here ----------
+comment_path = os.environ.get("PR_COMMENT_PATH", "pr_comment.md")
+
+# if runtime_regs or compile_regs:
+with open(comment_path, "w", encoding="utf-8") as f:
+    f.write(body + "\n")
+print(f"[INFO] Wrote PR comment to: {os.path.abspath(comment_path)}")
+# else:
+#     print("[INFO] No regressions; not writing pr_comment.md")
+
+# resp = s.post(f"{target_repo_api}/issues/{pr_num}/comments", json={"body": body})
+# print("Comment status:", resp.status_code, resp.text[:200])