[Benchmark] Add benchmark output parser and update metrics upload.

Author: zihugithub

.github/workflows/functional_tests_benchmark.yml

@@ -235,12 +235,34 @@ jobs:
           exit $exit_code
         timeout-minutes: 60
 
+      - name: Parse benchmark output to JSON
+        if: always() && steps.benchmark_test.outcome == 'success'
+        run: |
+          set -euo pipefail
+          cd $PROJECT_ROOT
+
+          TASK='${{ matrix.test_config.task }}'
+          MODEL='${{ matrix.test_config.model }}'
+          CASE='${{ matrix.test_config.case }}'
+
+          LOG_FILE="tests/functional_tests/${TASK}/${MODEL}/test_results/${CASE}/logs/host_0_localhost.output"
+          GOLD_FILE="tests/functional_tests/${TASK}/${MODEL}/gold_values/${CASE}.json"
+          OUTPUT_FILE="tests/functional_tests/${TASK}/${MODEL}/test_results/${CASE}/logs/benchmark_metrics.json"
+
+          echo "Parsing benchmark output to JSON"
+          echo "Log file: $LOG_FILE"
+          echo "Gold values: $GOLD_FILE"
+          echo "Output: $OUTPUT_FILE"
+
+          python tests/test_utils/runners/parse_benchmark_output.py \
+            "$LOG_FILE" "$GOLD_FILE" "$OUTPUT_FILE"
+
       - name: Upload benchmark data to backend
         if: always() && steps.benchmark_test.outcome == 'success'
         uses: flagos-ai/FlagOps/actions/post-benchmark-report@main
         with:
-          backend_url: 'http://10.1.4.167:30181/flagcicd-files/'
-          results_dir: ${{ env.PROJECT_ROOT }}/tests/functional_tests/${{ matrix.test_config.task }}/${{ matrix.test_config.model }}/test_results/${{ matrix.test_config.case }}
+          backend_url: 'http://10.1.4.167:30180/flagcicd-backend/metrics/'
+          report_path: ${{ env.PROJECT_ROOT }}/tests/functional_tests/${{ matrix.test_config.task }}/${{ matrix.test_config.model }}/test_results/${{ matrix.test_config.case }}/logs/benchmark_metrics.json
           job_name: 'benchmark_${{ matrix.test_config.model }}_${{ matrix.test_config.case }}'
           fail_on_error: 'false'
 

tests/test_utils/runners/parse_benchmark_output.py

@@ -0,0 +1,99 @@
+"""Parse host_0_localhost.output log file and produce benchmark.json.
+
+Usage:
+    python parse_benchmark_output.py <log_file> <gold_values_file> <output_json>
+
+The script extracts benchmark metrics from the training log using metric keys
+defined in the gold values file, then writes a structured JSON report.
+"""
+
+import json
+import re
+import sys
+
+
+# Default benchmark metric keys if no gold values file is provided
+DEFAULT_METRIC_KEYS = [
+    "elapsed time per iteration (ms):",
+    "throughput per GPU (TFLOP/s/GPU):",
+]
+
+
+def extract_metrics_from_log(lines, metric_keys):
+    """Extract metrics from training log lines.
+
+    Log format (pipe-separated):
+        " [2026-01-15 09:13:30] iteration 4/10 | ... | lm loss: 1.161108E+01 | ... |"
+    """
+    results = {key: [] for key in metric_keys}
+
+    for line in lines:
+        if "iteration" not in line:
+            continue
+
+        parts = line.split("|")
+        for part in parts:
+            part = part.strip()
+            for key in metric_keys:
+                if part.startswith(key.rstrip(":")):
+                    match = re.search(
+                        r":\s*([+-]?\d+\.?\d*(?:[eE][+-]?\d+)?)", part
+                    )
+                    if match:
+                        try:
+                            results[key].append(float(match.group(1)))
+                        except ValueError:
+                            continue
+
+    return results
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(
+            f"Usage: {sys.argv[0]} <log_file> <gold_values_file> [output_json]",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    log_file = sys.argv[1]
+    gold_values_file = sys.argv[2]
+    output_json = sys.argv[3] if len(sys.argv) > 3 else "benchmark.json"
+
+    # Read log file
+    with open(log_file, "r") as f:
+        lines = f.readlines()
+
+    # Determine metric keys from gold values file
+    try:
+        with open(gold_values_file, "r") as f:
+            gold_data = json.load(f)
+        metric_keys = list(gold_data.keys())
+    except (FileNotFoundError, json.JSONDecodeError):
+        print(
+            f"Warning: Could not load gold values from {gold_values_file}, "
+            f"using default metric keys",
+            file=sys.stderr,
+        )
+        metric_keys = DEFAULT_METRIC_KEYS
+
+    # Extract metrics
+    metrics = extract_metrics_from_log(lines, metric_keys)
+
+    # Build benchmark.json with values structure matching gold values format
+    benchmark = {}
+    for key in metric_keys:
+        benchmark[key] = {"values": metrics.get(key, [])}
+
+    # Write output
+    with open(output_json, "w") as f:
+        json.dump(benchmark, f, indent=4)
+
+    print(f"Benchmark results written to {output_json}")
+    print(f"Metrics extracted: {list(benchmark.keys())}")
+    for key, data in benchmark.items():
+        print(f"  {key}: {len(data['values'])} values")
+
+
+if __name__ == "__main__":
+    main()