Integrate trace reports into build and test actions

Author: librarian

.github/actions/build/action.yaml

@@ -59,6 +59,15 @@ outputs:
   build_duration_seconds:
     description: "Elapsed time of the ya make build step in seconds"
     value: ${{ steps.build.outputs.build_duration_seconds }}
+  trace_report_url:
+    description: "URL to the static ya build trace"
+    value: ${{ steps.build-trace.outputs.html_url }}
+  trace_otlp_url:
+    description: "URL to the raw ya build OTLP bundle"
+    value: ${{ steps.build-trace.outputs.otlp_url }}
+  trace_inputs_url:
+    description: "URL to the archived raw ya build trace inputs"
+    value: ${{ steps.build-trace.outputs.inputs_url }}
 
 runs:
   using: composite
@@ -264,6 +273,7 @@ runs:
         while [ "$attempt" -lt "$RETRIES" ]; do
             attempt=$((attempt + 1))
             echo "ya build attempt ${attempt}/${RETRIES}"
+            TRACE_START_TIME_NS=$(date +%s%N)
             date
             EXIT_CODE_FILE=$(mktemp)
             set +e
@@ -273,6 +283,7 @@ runs:
             } |& tee "$TMP_DIR/ya_make_output.txt"
             set -e
             YA_MAKE_EXIT_CODE=$(< "$EXIT_CODE_FILE")
+            TRACE_END_TIME_NS=$(date +%s%N)
             rm -f "$EXIT_CODE_FILE"
             if [ "$YA_MAKE_EXIT_CODE" = "0" ]; then
                 break
@@ -288,8 +299,60 @@ runs:
         BUILD_DURATION_SECONDS=$((BUILD_END_TIME - BUILD_START_TIME))
         echo "BUILD_DURATION_SECONDS=$BUILD_DURATION_SECONDS" | tee -a "$GITHUB_ENV"
         echo "build_duration_seconds=$BUILD_DURATION_SECONDS" | tee -a "$GITHUB_OUTPUT"
-        echo "YA_MAKE_EXIT_CODE=$YA_MAKE_EXIT_CODE" >> "$GITHUB_ENV"
+        {
+            echo "YA_MAKE_EXIT_CODE=$YA_MAKE_EXIT_CODE"
+            echo "YA_BUILD_ATTEMPT=$attempt"
+            echo "TRACE_START_TIME_NS=$TRACE_START_TIME_NS"
+            echo "TRACE_END_TIME_NS=$TRACE_END_TIME_NS"
+        } >> "$GITHUB_ENV"
         date
+    - name: Install trace dependencies
+      if: always() && env.YA_MAKE_EXIT_CODE != ''
+      continue-on-error: true
+      shell: bash
+      run: |
+        pip install -r .github/scripts/requirements.txt
+    - name: Render build trace
+      id: build-trace
+      if: always() && env.YA_MAKE_EXIT_CODE != ''
+      continue-on-error: true
+      shell: bash
+      env:
+        BUILD_PRESET: ${{ inputs.build_preset }}
+        BUILD_TARGET: ${{ inputs.build_target }}
+      run: |
+        set -euo pipefail
+        export PYTHONPATH="${PYTHONPATH:-}:$GITHUB_WORKSPACE/.github"
+        BUILD_TARGET=${BUILD_TARGET//\"/}
+        TRACE_REPORT_URL="${S3_WEBSITE_PREFIX}/build_logs/${COMPONENT_DIR}trace.html"
+        TRACE_OTLP_URL="${S3_WEBSITE_PREFIX}/build_logs/${COMPONENT_DIR}trace.otlp.jsonl.gz"
+        TRACE_INPUTS_URL="${S3_WEBSITE_PREFIX}/build_logs/${COMPONENT_DIR}trace-inputs.tar.gz"
+        if [ -n "${S3_REPORTS_WEBSITE_PREFIX:-}" ]; then
+            TRACE_REPORT_URL="${S3_REPORTS_WEBSITE_PREFIX}/${COMPONENT_DIR}build/trace.html"
+            TRACE_OTLP_URL="${S3_REPORTS_WEBSITE_PREFIX}/${COMPONENT_DIR}build/trace.otlp.jsonl.gz"
+            TRACE_INPUTS_URL="${S3_REPORTS_WEBSITE_PREFIX}/${COMPONENT_DIR}build/trace-inputs.tar.gz"
+        fi
+        bash "$GITHUB_WORKSPACE/.github/scripts/tracing/render_ya_trace_bundle.sh" \
+            --warning-title "Build trace" \
+            --report-dir "$TMP_DIR" \
+            --ya-out "$TMP_DIR" \
+            --evlog "$TMP_DIR/ya_evlog.jsonl" \
+            --html-url "$TRACE_REPORT_URL" \
+            --otlp-url "$TRACE_OTLP_URL" \
+            --inputs-url "$TRACE_INPUTS_URL" \
+            --summary "$GITHUB_STEP_SUMMARY" \
+            --summary-heading "Build trace" \
+            --github-output "$GITHUB_OUTPUT" \
+            -- \
+            --attempt-start-ns "$TRACE_START_TIME_NS" \
+            --attempt-end-ns "$TRACE_END_TIME_NS" \
+            --exit-code "$YA_MAKE_EXIT_CODE" \
+            --result-code "$YA_MAKE_EXIT_CODE" \
+            --component "${COMPONENT_DIR%/}" \
+            --build-preset "$BUILD_PRESET" \
+            --build-target "$BUILD_TARGET" \
+            --retry "$YA_BUILD_ATTEMPT" \
+            --operation build
     - name: Summarize build errors
       id: build-errors
       if: env.YA_MAKE_EXIT_CODE != '' && env.YA_MAKE_EXIT_CODE != '0'
@@ -321,7 +384,21 @@ runs:
       shell: bash
       run: |
         echo "::group::s3-sync"
-        aws s3 sync --acl public-read --no-progress "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/${COMPONENT_DIR}"
+        aws s3 sync --acl public-read --no-progress \
+            --exclude ".trace-inputs.*.tar.gz.tmp" \
+            --exclude "trace-inputs.files" \
+            "$TMP_DIR/" \
+            "$S3_BUCKET_PATH/build_logs/${COMPONENT_DIR}"
+        if [ -n "${S3_REPORTS_BUCKET_PATH:-}" ]; then
+            aws s3 sync --acl public-read --follow-symlinks --no-progress \
+                --exclude "*" \
+                --include "trace.html" \
+                --include "trace.manifest.json" \
+                --include "trace.otlp.jsonl.gz" \
+                --include "trace-inputs.tar.gz" \
+                "$TMP_DIR/" \
+                "$S3_REPORTS_BUCKET_PATH/${COMPONENT_DIR}build/"
+        fi
         echo "::endgroup::"
     - name: Create directory listing on s3
       if: always()

.github/actions/test/action.yaml

@@ -159,6 +159,11 @@ runs:
         echo -n "$GITHUB_TOKEN" > "$GITHUB_TOKEN_FILE"
         echo "GITHUB_TOKEN_FILE=$GITHUB_TOKEN_FILE" >> "$GITHUB_ENV"
         chmod 600 "$GITHUB_TOKEN_FILE"
+    - name: Install trace dependencies
+      continue-on-error: true
+      shell: bash
+      run: |
+        pip install -r .github/scripts/requirements.txt
     - name: ya test
       id: check_test_results
       shell: bash --noprofile --norc -eo pipefail -x {0}
@@ -299,13 +304,14 @@ runs:
 
         echo "RETRIES=${RETRIES}" | tee -a "$GITHUB_ENV"
         echo "report_generated=no" >> "$GITHUB_OUTPUT"
-        RETRY_FLAG=""
+        retry_params=()
         i=0
         echo "LAST_STEP=0" | tee -a "$GITHUB_ENV"
         # setting variable in case that test will be interrupter by OOM killer that
         # will kill VM with runner
         echo "failed_tests=yes" >> $GITHUB_OUTPUT
         while [ "$RETRIES" -gt 0 ]; do
+            TRACE_START_TIME_NS=$(date +%s%N)
             TEST_START_TIME=$(date +%s)
             echo TEST_START_TIME=$TEST_START_TIME | tee -a "$GITHUB_ENV"
             RETRIES=$((RETRIES - 1))
@@ -338,7 +344,7 @@ runs:
                     --junit "${JUNIT_REPORT_XML}.${i}" \
                     --log-file "$RETRY_LOG_DIR/ya_log.txt" \
                     --evlog-file "$RETRY_LOG_DIR/ya_evlog.jsonl" \
-                    "$RETRY_FLAG"
+                    "${retry_params[@]}"
                 echo $? > "$EXIT_CODE_FILE"
             } |& tee "$RETRY_LOG_DIR/ya_make_output.txt"
             set -e
@@ -376,7 +382,7 @@ runs:
                         echo "ya test returned $RC after timeout killed it with SIGKILL"
                         ATTEMPT_TIMED_OUT=1
                         DO_RETRY=1
-                        RETRY_FLAG="-X"
+                        retry_params=(-X)
                     else
                         echo "ya test returned $RC it was killed by OOM killer"
                         if [ $OOM_CAN_BE_RETRIED -eq 0 ]; then
@@ -402,11 +408,12 @@ runs:
                     # disabling retries here for now until we will be able to
                     # process junit with retried runs
                     DO_RETRY=1
-                    RETRY_FLAG="-X"
+                    retry_params=(-X)
                     ;;
             esac
 
             touch "$LOG_DIR/${i}_build_finished"
+            TRACE_END_TIME_NS=$(date +%s%N)
             TEST_END_TIME=$(date +%s)
             echo TEST_END_TIME=$TEST_END_TIME | tee -a "$GITHUB_ENV"
 
@@ -573,6 +580,15 @@ runs:
                 fi
             fi
 
+            REPORT_DIR="$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}"
+            TRACE_REPORT_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace.html"
+            TRACE_OTLP_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace.otlp.jsonl.gz"
+            TRACE_INPUTS_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace-inputs.tar.gz"
+            if [ -n "${S3_REPORTS_WEBSITE_PREFIX:-}" ]; then
+                TRACE_REPORT_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace.html"
+                TRACE_OTLP_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace.otlp.jsonl.gz"
+                TRACE_INPUTS_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace-inputs.tar.gz"
+            fi
             export SUMMARY_OUT_ENV_PATH=$(mktemp -p /home/github)
             set +x
             GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
@@ -588,6 +604,7 @@ runs:
                 --test-target "${TEST_TARGET//\"/}" \
                 --test-time $((TEST_END_TIME - TEST_START_TIME)) \
                 --build-error-log-url "$BUILD_ERROR_LOG_URL_FOR_ITERATION" \
+                --trace-report-url "$TRACE_REPORT_URL" \
                 "Tests" ya-test.html "${JUNIT_REPORT_XML}.${i}"
             unset GITHUB_TOKEN
             echo "report_generated=yes" >> "$GITHUB_OUTPUT"
@@ -598,6 +615,29 @@ runs:
                     echo "[Extracted build errors]($BUILD_ERROR_LOG_URL_FOR_ITERATION)"
                 } >> "$SUMMARY_OUT_ENV_PATH"
             fi
+            PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
+                bash "$GITHUB_WORKSPACE/.github/scripts/tracing/render_ya_trace_bundle.sh" \
+                --warning-title "Trace report" \
+                --report-dir "$REPORT_DIR" \
+                --ya-out "$OUT_DIR" \
+                --evlog "$LOG_DIR/${i}/ya_evlog.jsonl" \
+                --html-url "$TRACE_REPORT_URL" \
+                --otlp-url "$TRACE_OTLP_URL" \
+                --inputs-url "$TRACE_INPUTS_URL" \
+                --summary "$SUMMARY_OUT_ENV_PATH" \
+                -- \
+                --attempt-start-ns "$TRACE_START_TIME_NS" \
+                --attempt-end-ns "$TRACE_END_TIME_NS" \
+                --exit-code "$RC" \
+                --result-code "$FAIL_CHECKER_RC" \
+                --component "${COMPONENT_DIR%/}" \
+                --build-preset "$BUILD_PRESET" \
+                --test-target "$TEST_TARGET" \
+                --test-type "$TEST_TYPE" \
+                --test-size "$TEST_SIZE" \
+                --retry "$i" \
+                --test-log-url-prefix "$S3_WEBSITE_PREFIX/logs/${COMPONENT_DIR}${i}/" \
+                --test-data-url-prefix "$S3_WEBSITE_PREFIX/test_data/${COMPONENT_DIR}${i}$GITHUB_WORKSPACE/"
             cat $SUMMARY_OUT_ENV_PATH | tee -a $GITHUB_STEP_SUMMARY
             echo "::endgroup::"
 
@@ -613,11 +653,21 @@ runs:
             fi
 
             echo "::group::s3-sync"
-            aws s3 sync --acl public-read --follow-symlinks --no-progress "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
-            if [ -n "${S3_REPORTS_BUCKET_PATH:-}" ] && [ -f "$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}/summary.json" ]; then
-                aws s3 cp --acl public-read --follow-symlinks --no-progress \
-                    "$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}/summary.json" \
-                    "$S3_REPORTS_BUCKET_PATH/${COMPONENT_DIR}${i}/summary.json"
+            aws s3 sync --acl public-read --follow-symlinks --no-progress \
+                --exclude "*.trace-inputs.*.tar.gz.tmp" \
+                --exclude "*trace-inputs.files" \
+                "$ARTIFACTS_DIR/" \
+                "$S3_BUCKET_PATH/"
+            if [ -n "${S3_REPORTS_BUCKET_PATH:-}" ]; then
+                aws s3 sync --acl public-read --follow-symlinks --no-progress \
+                    --exclude "*" \
+                    --include "summary.json" \
+                    --include "trace.html" \
+                    --include "trace.manifest.json" \
+                    --include "trace.otlp.jsonl.gz" \
+                    --include "trace-inputs.tar.gz" \
+                    "$REPORT_DIR/" \
+                    "$S3_REPORTS_BUCKET_PATH/${COMPONENT_DIR}${i}/"
             fi
             touch "$LOG_DIR/${i}_artifacts_synced"
             echo "::endgroup::"

.github/packer/scripts/requirements.txt

@@ -26,3 +26,4 @@ PyGithub==2.9.1
 nebius==0.3.59
 tabulate
 numpy
+opentelemetry-proto-json==0.65b0

.github/scripts/tests/generate_summary.py

@@ -454,6 +454,7 @@ def render_testlist_html(
     fn: str,
     summary_url: str,
     build_error_log_url: str = "",
+    trace_report_url: str = "",
 ) -> None:
     templates_path = Path(__file__).with_name("templates")
 
@@ -482,6 +483,7 @@ def render_testlist_html(
         summary_url=summary_url,
         build_error_log_url=build_error_log_url,
         build_error_target_url=build_error_target_url,
+        trace_report_url=trace_report_url,
     )
 
     with open(fn, "w") as fp:
@@ -557,6 +559,7 @@ def gen_summary(
     summary_out_folder: str,
     paths: list[TitlePathTriplet],
     build_error_log_url: str = "",
+    trace_report_url: str = "",
 ) -> TestSummary:
     summary = TestSummary()
 
@@ -577,6 +580,7 @@ def gen_summary(
             os.path.join(summary_out_folder, html_fn),
             summary_url=summary_url_prefix,
             build_error_log_url=build_error_log_url,
+            trace_report_url=trace_report_url,
         )
         summary_line.add_report(html_fn, report_url)
         summary.add_line(summary_line)
@@ -1377,6 +1381,7 @@ def main() -> None:
     )
     parser.add_argument("--test-time", default="0", required=False)
     parser.add_argument("--build-error-log-url", default="", required=False)
+    parser.add_argument("--trace-report-url", default="", required=False)
     parser.add_argument(
         "--workload-status",
         choices=("in_progress", "completed"),
@@ -1405,6 +1410,7 @@ def main() -> None:
             args.summary_out_path,
             title_path,
             build_error_log_url=args.build_error_log_url,
+            trace_report_url=args.trace_report_url,
         )
         write_summary(
             summary,

.github/scripts/tests/generate_summary_test.py

@@ -45,6 +45,7 @@ def test_gen_summary_creates_html_and_aggregates_counters(
         "https://summary/",
         str(tmp_path),
         [("Tests", "ya-test.html", str(xml_path))],
+        trace_report_url="https://reports.example/trace.html",
     )
 
     assert summary.is_empty is False
@@ -60,6 +61,8 @@ def test_gen_summary_creates_html_and_aggregates_counters(
     assert 'id="FAIL"' in html
     assert "a/fail" in html
     assert "Summary dir file listing" in html
+    assert 'href="https://reports.example/trace.html"' in html
+    assert "best-effort basis and may be missing" in html
 
 
 def test_gen_summary_links_build_errors_in_fail_build_html(

.github/scripts/tests/templates/summary.html

@@ -40,7 +40,13 @@
 </head>
 
 <body>
-    <p><a href="{{ summary_url }}index.html">Summary dir file listing</a></p>
+    <p>
+        <a href="{{ summary_url }}index.html">Summary dir file listing</a>
+        {% if trace_report_url %}
+        | <a href="{{ trace_report_url }}"
+            title="The trace report is generated on a best-effort basis and may be missing">Ya make trace</a>
+        {% endif %}
+    </p>
     {% for status in status_order %}
     <h1 id="{{ status.name }}">{{ status.name }} ({{ tests[status] | length }})</h1>
     {% if status.name == "FAIL_BUILD" and build_error_log_url %}

.github/workflows/README.md

@@ -4,6 +4,48 @@ For validating syntax use action-validator
 find .github/workflows .github/actions -type f \( -iname \*.yaml -o -iname \*.yml \) -print | while read path; do echo Checking $path; action-validator --verbose $path; done
 ```
 
+## Static execution traces
+
+Each `ya make` test attempt writes these files beside `summary.json`:
+
+- `trace.otlp.jsonl.gz`: canonical OTLP/JSON Lines spans;
+- `trace.manifest.json`: bundle metadata and span counts;
+- `trace.html`: a self-contained, searchable waterfall.
+
+The HTML test summary links to `trace.html`; the link warns that trace
+generation is best-effort and its target may be missing.
+
+The standalone `build` action writes the same three files beside its build
+logs and copies them to the workflow trace reports prefix. Its bundle has a
+`ya make build` root with graph, cache, build-operation, and critical-path
+spans, but no test chunks.
+
+Observed `subtest-started`/`subtest-finished` events become test spans. Older
+finish-only events use the reported test duration and are marked with
+`test.timing.inferred=true`. Ya recipe-stage durations do not have absolute
+timestamps, so their `ya.test.stage` spans are positioned from the chunk start
+and marked as inferred while retaining the reported duration.
+
+The per-attempt ya event log supplies absolute timing for graph generation,
+execution, and report-finalization phase spans. Completed worker nodes become
+searchable build spans for compilation, linking, archive creation, cache
+restores, and result materialization. Test worker nodes are matched to their
+logical chunks and provide worker and worker-phase spans around the test tree.
+The `build operations` span reports both its wall-clock execution envelope and
+the cumulative worker-node time; the latter can be larger because nodes run in
+parallel. It also carries ya's authoritative considered-task cache statistics,
+observed test-excluded worker-node cache ratios (including per-tool `CC`, `AR`,
+`LD`, and similar breakdowns), total task reuse/avoidance, execution-stage wall
+times, distributed-cache I/O, and the build-only portion of ya's reported
+critical path. Critical-path build nodes are marked on their individual spans.
+
+To render a saved OTLP bundle locally:
+
+```bash
+PYTHONPATH=.github python3 -m scripts.tracing.trace_report \
+  trace.otlp.jsonl.gz -o trace.html
+```
+
 You can use [act](https://github.com/nektos/act) as a debugging tool for pipelines it acts as a GitHub runner of some sort, using docker.
 
 It is not 100% replacement for GitHub actions altogether (i.e. you can't run self-hosted GitHub runners), but you can use it to debug some of your changes before committing