Fix binary recording script (#2208)

Author: jefchien

.github/scripts/parse-s3-binaries.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# Parse `aws s3 ls --recursive` output (piped through awk '{print $3, $4}')
+# and produce a DynamoDB item JSON with binary sizes.
+#
+# Usage: aws s3 ls ... | awk '{print $3, $4}' | python3 parse-s3-binaries.py
+#
+# Required env: S3_PREFIX, COMMIT_HASH, COMMIT_DATE
+# Optional env: TAG
+
+import json
+import os
+import sys
+
+prefix = os.environ["S3_PREFIX"]
+commit_hash = os.environ["COMMIT_HASH"]
+commit_date = os.environ["COMMIT_DATE"]
+tag = os.environ.get("TAG", "")
+
+SKIP_EXT = (".sig", ".jar", ".rpm", ".deb", ".pkg", ".msi", ".tar.gz", ".gz", ".zip")
+SKIP_NAMES = ("CWAGENT_VERSION", "buildMSI.zip")
+
+binaries = {}
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+    parts = line.split(" ", 1)
+    if len(parts) != 2:
+        continue
+    try:
+        size, key = int(parts[0]), parts[1]
+    except ValueError:
+        continue
+    rel_path = key[len(prefix):] if key.startswith(prefix) else key
+    segments = rel_path.split("/")
+    if len(segments) != 2:
+        continue
+    filename = segments[1]
+    if any(filename.endswith(ext) for ext in SKIP_EXT):
+        continue
+    if filename in SKIP_NAMES:
+        continue
+    binaries[rel_path] = {"N": str(size)}
+
+if not binaries:
+    sys.exit(1)
+
+record_type = "release" if tag else "commit"
+item = {
+    "CommitHash": {"S": commit_hash},
+    "CommitDate": {"N": commit_date},
+    "Branch": {"S": "main"},
+    "RecordType": {"S": record_type},
+    "Binaries": {"M": binaries},
+}
+if tag:
+    item["Tag"] = {"S": tag}
+
+print(json.dumps(item))

.github/scripts/record-binary-sizes.sh

@@ -8,65 +8,25 @@ set -euo pipefail
 
 TABLE_NAME="CWABinarySizes"
 S3_BUCKET="${S3_BUCKET:?S3_BUCKET is required}"
-COMMIT_HASH="${COMMIT_HASH:?COMMIT_HASH is required}"
-COMMIT_DATE="${COMMIT_DATE:?COMMIT_DATE is required}"
-TAG="${TAG:-}"
+export COMMIT_HASH="${COMMIT_HASH:?COMMIT_HASH is required}"
+export COMMIT_DATE="${COMMIT_DATE:?COMMIT_DATE is required}"
+export TAG="${TAG:-}"
 REGION="us-west-2"
 
-S3_PREFIX="integration-test/binary/${COMMIT_HASH}/"
+export S3_PREFIX="integration-test/binary/${COMMIT_HASH}/"
 
-echo "Collecting binary sizes from S3 (${COMMIT_HASH:0:12})..."
-
-ITEM_JSON=$(
-     aws s3 ls "s3://${S3_BUCKET}/${S3_PREFIX}" --recursive --region "$REGION" |
-          awk '{print $3, $4}' |
-          python3 - "$S3_PREFIX" "$COMMIT_HASH" "$COMMIT_DATE" "$TAG" <<'PYEOF'
-import sys, json
-
-prefix = sys.argv[1]
-commit_hash = sys.argv[2]
-commit_date = sys.argv[3]
-tag = sys.argv[4]
-
-skip_ext = ('.sig', '.jar', '.rpm', '.deb', '.pkg', '.msi', '.tar.gz', '.gz', '.zip')
-skip_names = ('CWAGENT_VERSION', 'buildMSI.zip')
-
-binaries = {}
-for line in sys.stdin:
-    line = line.strip()
-    if not line:
-        continue
-    parts = line.split(' ', 1)
-    if len(parts) != 2:
-        continue
-    size, key = int(parts[0]), parts[1]
-    rel_path = key[len(prefix):] if key.startswith(prefix) else key
-    segments = rel_path.split('/')
-    if len(segments) != 2:
-        continue
-    filename = segments[1]
-    if any(filename.endswith(ext) for ext in skip_ext):
-        continue
-    if filename in skip_names:
-        continue
-    binaries[rel_path] = {"N": str(size)}
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
-if not binaries:
-    sys.exit(1)
+echo "Collecting binary sizes from S3 (${COMMIT_HASH:0:12})..."
 
-record_type = "release" if tag else "commit"
-item = {
-    "CommitHash": {"S": commit_hash},
-    "CommitDate": {"N": commit_date},
-    "Branch": {"S": "main"},
-    "RecordType": {"S": record_type},
-    "Binaries": {"M": binaries},
-}
-if tag:
-    item["Tag"] = {"S": tag}
+S3_LISTING=$(aws s3 ls "s3://${S3_BUCKET}/${S3_PREFIX}" --recursive --region "$REGION" || true)
+if [[ -z "$S3_LISTING" ]]; then
+     echo "No binaries found in S3, skipping."
+     exit 0
+fi
 
-print(json.dumps(item))
-PYEOF
+ITEM_JSON=$(
+     echo "$S3_LISTING" | awk '{print $3, $4}' | python3 "$SCRIPT_DIR/parse-s3-binaries.py"
 ) || {
      echo "No binaries found in S3, skipping."
      exit 0

.github/workflows/build-test-artifacts.yml

@@ -49,6 +49,7 @@ jobs:
 
   RecordBinarySizes:
     needs: [BuildAndUpload]
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
     uses: ./.github/workflows/record-binary-sizes.yml
     secrets: inherit
     permissions:

.github/workflows/record-binary-sizes.yml

@@ -9,7 +9,7 @@ on:
   workflow_call:
 
 jobs:
-  record:
+  Record:
     runs-on: ubuntu-latest
     permissions:
       id-token: write