action.yaml

name: Run tests (ya make)
description: Run tests using ya make
inputs:
  component:
    required: false
    description: "component name"
  test_target:
    required: false
    description: "test target"
  build_preset:
    required: true
    default: "relwithdebinfo"
    description: "relwithdebinfo, release-asan, release-tsan, release-msan, release-ubsan"
  test_size:
    required: false
    default: "small,medium,large"
    description: "small or small-medium or all"
  test_type:
    required: false
    default: "unittest,py3test,py2test,pytest"
    description: "run test type"
  test_threads:
    required: false
    default: "28"
    description: "Test threads count"
  link_threads:
    required: false
    default: "8"
    description: "link threads count"
  bazel_remote_uri:
    required: false
    description: "bazel-remote endpoint"
  bazel_remote_username:
    required: false
    description: "bazel-remote username"
  bazel_remote_password:
    required: false
    description: "bazel-remote password"
  cache_update:
    required: false
    description: "Use cache for tests"
  sync_to_s3:
    required: false
    default: "false"
    description: "Sync failed tests folders to s3"
  upload_ya_dir:
    required: false
    default: "no"
    description: "Upload ya dir to s3"
  clean_ya_dir:
    required: false
    default: "no"
    description: "Clean ya dir"
  use_network_cache:
    required: false
    default: "yes"
    description: "Use network cache"
  truncate_enabled:
    required: false
    default: "yes"
    description: "Truncate err files"
  number_of_retries:
    required: false
    default: "1"
    description: "Number of retries for ya make"
  test_timeout_minutes:
    required: false
    default: "390"
    description: "Per ya make test attempt timeout in minutes"
  test_budget_minutes:
    required: false
    default: "480"
    description: "Total job budget in minutes"
  test_budget_reserve_minutes:
    required: false
    default: "30"
    description: "Minutes reserved for uploading and processing after tests"
  build_duration_seconds:
    required: false
    default: "0"
    description: "Elapsed build time in seconds to subtract from test budget"
  tmpfs_mount_point:
    required: false
    default: "/mnt/ya_make_tmpfs"
    description: "Mount point path for the tmpfs filesystem"
outputs:
  failed_tests:
    description: "List of failed tests"
    value: ${{ steps.check_test_results.outputs.failed_tests }}
  test_failed:
    description: "Whether tests failed"
    value: ${{ steps.check_test_results.outputs.failed_tests }}
  report_generated:
    description: "Whether the test summary was generated"
    value: ${{ steps.check_test_results.outputs.report_generated }}

runs:
  using: composite
  steps:
    - name: Init
      shell: bash --noprofile --norc -eo pipefail -x {0}
      id: init
      run: |
        export TMP_DIR=/home/github/tmp_test
        COMPONENT_DIR=""
        if [ -n "$COMPONENT" ]; then
            COMPONENT_DIR="${COMPONENT}/"
        fi
        if [ "$COMPONENT" == "all" ]; then
            COMPONENT_DIR=""
        fi
        {
            echo "TMP_DIR=$TMP_DIR"
            echo "LOG_DIR=$TMP_DIR/logs"
            echo "OUT_DIR=$TMP_DIR/out"
            echo "ARTIFACTS_DIR=$TMP_DIR/artifacts"
            echo "TESTS_DATA_DIR=$TMP_DIR/test_data"
            echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports"
            echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml"
            echo "SUMMARY_LINKS=$(mktemp -p /home/github summary_links.XXXX)"
            echo "COMPONENT_DIR=$COMPONENT_DIR"
        } | tee -a $GITHUB_ENV
      env:
        COMPONENT: ${{ inputs.component }}

    - name: prepare
      shell: bash --noprofile --norc -eo pipefail -x {0}
      env:
        CLEAN_YA_DIR: ${{ inputs.clean_ya_dir }}
      run: |
        rm -rf $TMP_DIR $JUNIT_REPORT_XML $REPORTS_ARTIFACTS_DIR $TESTS_DATA_DIR
        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $REPORTS_ARTIFACTS_DIR $TESTS_DATA_DIR
        # shellcheck disable=SC2193
        if [ "$CLEAN_YA_DIR" == "yes" ] && [ -d /home/github/.ya/ ]; then
            echo "Cleaning ya dir"
            rm -rf /home/github/.ya/
        fi
        # checking /etc/hosts to see if vm was created correctly
        cat /etc/hosts
    - name: Setup cache password file
      shell: bash --noprofile --norc -eo pipefail {0}
      env:
        BAZEL_REMOTE_PASSWORD: ${{ inputs.bazel_remote_password }}
      run: |
        export BAZEL_REMOTE_PASSWORD_FILE=$(mktemp -p /home/github bazel.XXXX)
        echo -n "$BAZEL_REMOTE_PASSWORD" > "$BAZEL_REMOTE_PASSWORD_FILE"
        echo "BAZEL_REMOTE_PASSWORD_FILE=$BAZEL_REMOTE_PASSWORD_FILE" >> "$GITHUB_ENV"
        chmod 600 "$BAZEL_REMOTE_PASSWORD_FILE"
    - name: Setup Github token file
      shell: bash --noprofile --norc -eo pipefail {0}
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        export GITHUB_TOKEN_FILE=$(mktemp -p /home/github github.XXXX)
        echo -n "$GITHUB_TOKEN" > "$GITHUB_TOKEN_FILE"
        echo "GITHUB_TOKEN_FILE=$GITHUB_TOKEN_FILE" >> "$GITHUB_ENV"
        chmod 600 "$GITHUB_TOKEN_FILE"
    - name: Install trace dependencies
      continue-on-error: true
      shell: bash
      run: |
        pip install -r .github/scripts/requirements.txt
    - name: ya test
      id: check_test_results
      shell: bash --noprofile --norc -eo pipefail -x {0}
      env:
        NUMBER_OF_RETRIES: ${{ inputs.number_of_retries }}
        TEST_TIMEOUT_MINUTES: ${{ inputs.test_timeout_minutes }}
        TEST_BUDGET_MINUTES: ${{ inputs.test_budget_minutes }}
        TEST_BUDGET_RESERVE_MINUTES: ${{ inputs.test_budget_reserve_minutes }}
        BUILD_DURATION_SECONDS: ${{ inputs.build_duration_seconds }}
        BUILD_PRESET: ${{ inputs.build_preset }}
        TEST_TARGET: ${{ inputs.test_target }}
        TEST_SIZE: ${{ inputs.test_size }}
        TEST_TYPE: ${{ inputs.test_type }}
        TEST_THREADS: ${{ inputs.test_threads }}
        LINK_THREADS: ${{ inputs.link_threads }}
        BAZEL_REMOTE_USERNAME: ${{ inputs.bazel_remote_username }}
        BAZEL_REMOTE_URI: ${{ inputs.bazel_remote_uri }}
        USE_NETWORK_CACHE: ${{ inputs.use_network_cache }}
        CACHE_UPDATE: ${{ inputs.cache_update }}
        TRUNCATE_ENABLED: ${{ inputs.truncate_enabled }}
        SYNC_TO_S3: ${{ inputs.sync_to_s3 || 'false' }}
        TMPFS_MOUNT_POINT: ${{ inputs.tmpfs_mount_point }}
      run: |
        extra_params=(
            --run-all-tests
            --keep-going
            --retest
            --test-threads "$TEST_THREADS"
            --link-threads "$LINK_THREADS"
            --cache-size 512G
            --do-not-output-stderrs
            -T
            --nice 0
            --stat
            --output "$OUT_DIR"
            -DGITHUB_CI=yes
        )

        case "$BUILD_PRESET" in
            debug)
                extra_params+=(--build "debug")
                ;;
            relwithdebinfo)
                extra_params+=(--build "relwithdebinfo")
                ;;
            release-asan)
                extra_params+=(--build "release")
                extra_params+=(--sanitize="address")
                extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
                ;;
            release-tsan)
                extra_params+=(--build "release")
                extra_params+=(--sanitize="thread")
                extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
                ;;
            release-msan)
                extra_params+=(--build "release")
                extra_params+=(--sanitize="memory")
                extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
                ;;
            release-ubsan)
                extra_params+=(--build "release")
                extra_params+=(--sanitize="undefined")
                extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
                ;;
            *)
                echo "Invalid preset: $BUILD_PRESET"
                exit 1
                ;;
        esac

        # strip quotes from the string
        # TODO: remove it when pr.yaml gets updated
        TEST_TARGET=${TEST_TARGET//\"/}

        if [ -n "$TEST_TARGET" ]; then
            readarray -d ',' -t targets < <(printf "%s" "$TEST_TARGET")
            for target in "${targets[@]}"; do
                extra_params+=(--target="${target}")
            done
        fi

        if [ "$USE_NETWORK_CACHE" == "yes" ]; then
            if [ -n "$BAZEL_REMOTE_URI" ]; then
                extra_params+=(--bazel-remote-store)
                extra_params+=(--bazel-remote-base-uri "$BAZEL_REMOTE_URI")
            fi

            if [ -n "$BAZEL_REMOTE_USERNAME" ]; then
                extra_params+=(--bazel-remote-username "$BAZEL_REMOTE_USERNAME")
                extra_params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE")
            fi

            if [ "$CACHE_UPDATE" = "true" ]; then
                extra_params+=(--bazel-remote-put)
            fi
        fi

        if [[ -n "$TMPFS_MOUNT_POINT" ]]; then
            if mountpoint -q "$TMPFS_MOUNT_POINT"; then
                export YA_RAM_DRIVE_PATH="$TMPFS_MOUNT_POINT"
                echo "Tmpfs enabled ($YA_RAM_DRIVE_PATH)"
            else
                echo "Error: $TMPFS_MOUNT_POINT is not mounted"
            fi
        fi

        # shellcheck disable=SC2153
        readarray -d ',' -t test_size < <(printf "%s" "$TEST_SIZE")
        # shellcheck disable=SC2153
        readarray -d ',' -t test_type < <(printf "%s" "$TEST_TYPE")

        date
        ./ya --version

        function remaining_test_budget_seconds() {
            local now elapsed_test_seconds
            now=$(date +%s)
            elapsed_test_seconds=$((now - TEST_BUDGET_STARTED_AT))
            echo $((TEST_BUDGET_MINUTES * 60 - TEST_BUDGET_RESERVE_MINUTES * 60 - BUILD_DURATION_SECONDS - elapsed_test_seconds))
        }

        if [ "$TEST_BUDGET_RESERVE_MINUTES" -ge "$TEST_BUDGET_MINUTES" ]; then
            echo "Invalid test budget reserve (${TEST_BUDGET_RESERVE_MINUTES}m) for total budget (${TEST_BUDGET_MINUTES}m), using 30m reserve"
            TEST_BUDGET_RESERVE_MINUTES=30
        fi
        TEST_BUDGET_STARTED_AT=$(date +%s)
        echo "TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES}" | tee -a "$GITHUB_ENV"
        echo "TEST_BUDGET_MINUTES=${TEST_BUDGET_MINUTES}" | tee -a "$GITHUB_ENV"
        echo "TEST_BUDGET_RESERVE_MINUTES=${TEST_BUDGET_RESERVE_MINUTES}" | tee -a "$GITHUB_ENV"
        echo "BUILD_DURATION_SECONDS=${BUILD_DURATION_SECONDS}" | tee -a "$GITHUB_ENV"

        RETRIES="${NUMBER_OF_RETRIES}"
        OOM_CAN_BE_RETRIED=1
        BUILD_FAILED=0
        FAIL_CHECKER_RC=1

        echo "RETRIES=${RETRIES}" | tee -a "$GITHUB_ENV"
        echo "report_generated=no" >> "$GITHUB_OUTPUT"
        retry_params=()
        i=0
        echo "LAST_STEP=0" | tee -a "$GITHUB_ENV"
        # setting variable in case that test will be interrupter by OOM killer that
        # will kill VM with runner
        echo "failed_tests=yes" >> $GITHUB_OUTPUT
        while [ "$RETRIES" -gt 0 ]; do
            TRACE_START_TIME_NS=$(date +%s%N)
            TEST_START_TIME=$(date +%s)
            echo TEST_START_TIME=$TEST_START_TIME | tee -a "$GITHUB_ENV"
            RETRIES=$((RETRIES - 1))
            i=$((i + 1))
            echo "report_generated=no" >> "$GITHUB_OUTPUT"
            RETRY_LOG_DIR="$LOG_DIR/$i"
            mkdir -p "$RETRY_LOG_DIR"
            echo "LAST_STEP=${i}" | tee -a "$GITHUB_ENV"
            echo "::group::ya-make-test-${i}"
            REMAINING_TEST_BUDGET_SECONDS=$(remaining_test_budget_seconds)
            ATTEMPT_TIMEOUT_SECONDS=$((TEST_TIMEOUT_MINUTES * 60))
            if [ "$REMAINING_TEST_BUDGET_SECONDS" -lt "$ATTEMPT_TIMEOUT_SECONDS" ]; then
                ATTEMPT_TIMEOUT_SECONDS="$REMAINING_TEST_BUDGET_SECONDS"
            fi
            if [ "$ATTEMPT_TIMEOUT_SECONDS" -lt 60 ]; then
                echo "No test budget left: remaining=${REMAINING_TEST_BUDGET_SECONDS}s build=${BUILD_DURATION_SECONDS}s reserve=${TEST_BUDGET_RESERVE_MINUTES}m total=${TEST_BUDGET_MINUTES}m"
                FAIL_CHECKER_RC=124
                echo "::endgroup::"
                break
            fi
            echo "ya make timeout for attempt ${i}: ${ATTEMPT_TIMEOUT_SECONDS}s (cap=${TEST_TIMEOUT_MINUTES}m remaining_budget=${REMAINING_TEST_BUDGET_SECONDS}s build=${BUILD_DURATION_SECONDS}s reserve=${TEST_BUDGET_RESERVE_MINUTES}m total=${TEST_BUDGET_MINUTES}m)"
            EXIT_CODE_FILE=$(mktemp)
            YA_MAKE_ATTEMPT_STARTED_AT=$(date +%s)
            set +e
            {
                timeout --signal=TERM --kill-after=60s "${ATTEMPT_TIMEOUT_SECONDS}s" \
                    ./ya make "${test_size[@]/#/--test-size=}" \
                    "${test_type[@]/#/--test-type=}" \
                    "${extra_params[@]}" \
                    --junit "${JUNIT_REPORT_XML}.${i}" \
                    --log-file "$RETRY_LOG_DIR/ya_log.txt" \
                    --evlog-file "$RETRY_LOG_DIR/ya_evlog.jsonl" \
                    "${retry_params[@]}"
                echo $? > "$EXIT_CODE_FILE"
            } |& tee "$RETRY_LOG_DIR/ya_make_output.txt"
            set -e
            YA_MAKE_ATTEMPT_FINISHED_AT=$(date +%s)
            YA_MAKE_ATTEMPT_ELAPSED_SECONDS=$((YA_MAKE_ATTEMPT_FINISHED_AT - YA_MAKE_ATTEMPT_STARTED_AT))
            RC=$(< "$EXIT_CODE_FILE")
            rm -f "$EXIT_CODE_FILE"
            if [ -s "${JUNIT_REPORT_XML}.${i}" ]; then
                JUNIT_AVAILABLE=1
            else
                JUNIT_AVAILABLE=0
            fi
            FAIL_CHECKER_RC="$RC"
            if [ "$JUNIT_AVAILABLE" -eq 0 ] && [ "$FAIL_CHECKER_RC" -eq 0 ]; then
                FAIL_CHECKER_RC=1
            fi
            DO_RETRY=0
            ATTEMPT_TIMED_OUT=0
            if [ "$RC" -eq 124 ]; then
                ATTEMPT_TIMED_OUT=1
            fi
            case $RC in
                0)
                    echo "ya test successfully finished, no errors"
                    ls -la "${JUNIT_REPORT_XML}.${i}" || true
                    date
                    if [ -s "${JUNIT_REPORT_XML}.${i}" ]; then
                        echo "${JUNIT_REPORT_XML}.${i} exists"
                    else
                        echo "${JUNIT_REPORT_XML}.${i} doesn't exist or has zero size"
                    fi
                    ;;
                137)
                    if [ "$YA_MAKE_ATTEMPT_ELAPSED_SECONDS" -ge "$ATTEMPT_TIMEOUT_SECONDS" ]; then
                        echo "ya test returned $RC after timeout killed it with SIGKILL"
                        ATTEMPT_TIMED_OUT=1
                        DO_RETRY=1
                        retry_params=(-X)
                    else
                        echo "ya test returned $RC it was killed by OOM killer"
                        if [ $OOM_CAN_BE_RETRIED -eq 0 ]; then
                            DO_RETRY=0
                        else
                            DO_RETRY=1
                            OOM_CAN_BE_RETRIED=0
                            RETRIES=$((RETRIES + 1))
                            echo "RETRIES=$((RETRIES + 1))" | tee -a "$GITHUB_ENV"
                            echo "Retrying once because OOM killer killed ya-bin, increasing retry count by 1"
                        fi
                    fi
                    ;;
                *)
                    echo "ya test returned $RC"
                    ls -la "${JUNIT_REPORT_XML}.${i}" || true
                    date
                    if [ -s "${JUNIT_REPORT_XML}.${i}" ]; then
                        echo "${JUNIT_REPORT_XML}.${i} exists"
                    else
                        echo "${JUNIT_REPORT_XML}.${i} doesn't exist or has zero size"
                    fi
                    # disabling retries here for now until we will be able to
                    # process junit with retried runs
                    DO_RETRY=1
                    retry_params=(-X)
                    ;;
            esac

            touch "$LOG_DIR/${i}_build_finished"
            TRACE_END_TIME_NS=$(date +%s%N)
            TEST_END_TIME=$(date +%s)
            echo TEST_END_TIME=$TEST_END_TIME | tee -a "$GITHUB_ENV"

            echo "::group::s3-sync-logs"
            if [ "$SYNC_TO_S3" = "true" ]; then
                aws s3 sync --acl public-read --follow-symlinks --no-progress "$RETRY_LOG_DIR/" "$S3_BUCKET_PATH/test_logs/${COMPONENT_DIR}${i}/"
            fi
            echo "::endgroup::"

            touch "$LOG_DIR/${i}_logs_synced"

            NEED_COPYING=0

            # shellcheck disable=SC2024
            if [ "$JUNIT_AVAILABLE" -eq 1 ]; then
                echo "::group::muted-tests"
                cat .github/config/muted_ya.txt
                MUTED_CONFIG=".github/config/muted_ya.txt"
                echo "::endgroup::"
                echo "::group::postprocess-junit-${i}"
                mkdir -p "$ARTIFACTS_DIR/${COMPONENT_DIR}"
                mkdir -p "$REPORTS_ARTIFACTS_DIR/${COMPONENT_DIR}"
                gzip -c "${JUNIT_REPORT_XML}.${i}" > "$REPORTS_ARTIFACTS_DIR/${COMPONENT_DIR}orig_junit.${i}.xml.gz"
                PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                    python3 -m scripts.tests.transform_ya_junit -i \
                    -m "${MUTED_CONFIG}" \
                    --ya-out "$OUT_DIR" \
                    --log-url-prefix "$S3_WEBSITE_PREFIX/logs/${COMPONENT_DIR}${i}/" \
                    --data-url-prefix "$S3_WEBSITE_PREFIX/test_data/${COMPONENT_DIR}${i}$GITHUB_WORKSPACE" \
                    --log-out-dir "$ARTIFACTS_DIR/logs/${COMPONENT_DIR}${i}" \
                    "${JUNIT_REPORT_XML}.${i}"

                echo "::endgroup::"
                # exports BUILD_FAILED_COUNT if it is not zero, we need to save current value to compare it to new one in retries and if it is bigger - update it
                if [ -z "$BUILD_FAILED_COUNT" ]; then
                    BUILD_FAILED_COUNT=0
                fi
                BUILD_FAILED_COUNT_TMP=$BUILD_FAILED_COUNT
                FAIL_CHECKER_TEMP_FILE=$(mktemp -p /home/github)
                FAIL_CHECKER_EXIT_CODE_FILE=$(mktemp)
                set +e
                {
                    PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                        python3 -m scripts.tests.fail_checker "${JUNIT_REPORT_XML}.${i}"
                    echo $? > "$FAIL_CHECKER_EXIT_CODE_FILE"
                } |& tee fail-checker-output.log
                set -e
                if [ -f "$FAIL_CHECKER_TEMP_FILE" ]; then
                    BUILD_FAILED_COUNT=$(grep -E '^BUILD_FAILED_COUNT=' "$FAIL_CHECKER_TEMP_FILE" | cut -d '=' -f2) || true
                    if [ -z "$BUILD_FAILED_COUNT" ]; then
                        BUILD_FAILED_COUNT=0
                    fi
                    rm "$FAIL_CHECKER_TEMP_FILE"
                fi
                if [ "$BUILD_FAILED_COUNT" -gt "$BUILD_FAILED_COUNT_TMP" ]; then
                    echo "BUILD_FAILED_COUNT=$BUILD_FAILED_COUNT" | tee -a "$GITHUB_ENV"
                else
                    echo "BUILD_FAILED_COUNT=$BUILD_FAILED_COUNT_TMP" | tee -a "$GITHUB_ENV"
                    BUILD_FAILED_COUNT=$BUILD_FAILED_COUNT_TMP
                fi
                JUNIT_FAIL_CHECKER_RC=$(< "$FAIL_CHECKER_EXIT_CODE_FILE")
                rm -f "$FAIL_CHECKER_EXIT_CODE_FILE"
                if [ "$JUNIT_FAIL_CHECKER_RC" -ne 0 ]; then
                    NEED_COPYING=1
                    echo "::group::Copy-failed-tests-data"
                    PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                        python3 -m scripts.tests.fail_checker "${JUNIT_REPORT_XML}.${i}" --paths-only | while read -r path; do
                        echo $path
                        find "${GITHUB_WORKSPACE}/${path}" -print0 | xargs -0 cp -L -r --parents -t "$TESTS_DATA_DIR"
                    done
                    echo "::endgroup::"
                fi
                if [ "$JUNIT_FAIL_CHECKER_RC" -eq 237 ]; then
                    BUILD_FAILED=1
                fi
                if [ "$ATTEMPT_TIMED_OUT" -eq 1 ]; then
                    echo "Keeping ya make timeout exit code $RC instead of JUnit checker exit code $JUNIT_FAIL_CHECKER_RC"
                else
                    FAIL_CHECKER_RC="$JUNIT_FAIL_CHECKER_RC"
                fi
            fi
            if [ "$NEED_COPYING" -ne "0" ]; then
                echo "::group::remove-binaries-from-tests-data-dir"
                find "$TESTS_DATA_DIR" -type f -print0 | while IFS= read -r -d '' file; do
                    if file --mime-type "$file" | grep -q 'application/x-executable'; then
                        echo "$file"
                        rm -f "$file"
                    fi
                done
                echo "::endgroup::"
                echo "::group::remove-images-from-tests-data-dir"
                find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name 'NVMENBS0*' -o -name generated_other_big_raw_image
                find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name 'NVMENBS0*' -o -name generated_other_big_raw_image \) -delete
                echo "::endgroup::"
                echo "::group::truncate-err-files"
                find "$TESTS_DATA_DIR" -type f -name '*.err' -o -name '*.log' -o -name '*.out' -size +1G -print0 | while IFS= read -r -d '' file; do
                    orig_size=$(du -h "$file" | cut -f1)
                    echo "$file - $orig_size"
                    # shellcheck disable=SC2193
                    if [ "$TRUNCATE_ENABLED" == "yes" ]; then
                        truncate -s 1G "$file"
                        echo "... truncated (original size was $orig_size) ..." >> "$file"
                    else
                        echo "not truncated"
                    fi
                done
                echo "::endgroup::"
                echo "::group::s3-sync-test-data"
                if [ "$SYNC_TO_S3" = "true" ]; then
                    du -h "$TESTS_DATA_DIR/"
                    aws s3 sync --acl public-read --follow-symlinks --no-progress "$TESTS_DATA_DIR/" "$S3_BUCKET_PATH/test_data/${COMPONENT_DIR}${i}/"
                fi
                touch "$LOG_DIR/${i}_test_data_synced"
                echo "::endgroup::"
            fi
            # shellcheck disable=SC2024
            if [ "$JUNIT_AVAILABLE" -eq 1 ]; then
                gzip -c "${JUNIT_REPORT_XML}.${i}" > "$REPORTS_ARTIFACTS_DIR/${COMPONENT_DIR}transformed_junit.${i}.xml.gz"
            else
                echo "Skipping transformed junit archive: ${JUNIT_REPORT_XML}.${i} is missing or empty"
            fi

            echo "::group::write-summary-${i}"
            mkdir -p $ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}

            cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY

            platform_name=$(uname | tr '[:upper:]' '[:lower:]')-$(arch)
            BUILD_ERROR_LOG_URL_FOR_ITERATION=""

            if [ "$JUNIT_AVAILABLE" -eq 1 ]; then
                SUMMARY_FILE="$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}/build_errors.html"
                BUILD_ERROR_LOG_URL_FOR_ITERATION="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/build_errors.html"
                PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                    python3 -m scripts.tests.ya_build_errors \
                    --junit "${JUNIT_REPORT_XML}.${i}" \
                    --evlog "$LOG_DIR/${i}/ya_evlog.jsonl" \
                    --log "$LOG_DIR/${i}/ya_log.txt" \
                    --ya-log-url "${S3_WEBSITE_PREFIX}/test_logs/${COMPONENT_DIR}${i}/ya_log.txt" \
                    --evlog-url "${S3_WEBSITE_PREFIX}/test_logs/${COMPONENT_DIR}${i}/ya_evlog.jsonl" \
                    --only-if-junit-failed-build \
                    > "$SUMMARY_FILE"

                if [ ! -s "$SUMMARY_FILE" ]; then
                    BUILD_ERROR_LOG_URL_FOR_ITERATION=""
                    rm -f "$SUMMARY_FILE"
                fi
            elif [ "$RC" -ne 0 ]; then
                SUMMARY_FILE="$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}/build_errors.html"
                BUILD_ERROR_LOG_URL_FOR_ITERATION="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/build_errors.html"
                PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                    python3 -m scripts.tests.ya_build_errors \
                    --evlog "$LOG_DIR/${i}/ya_evlog.jsonl" \
                    --log "$LOG_DIR/${i}/ya_log.txt" \
                    --ya-make-output "$LOG_DIR/${i}/ya_make_output.txt" \
                    --ya-make-output-url "${S3_WEBSITE_PREFIX}/test_logs/${COMPONENT_DIR}${i}/ya_make_output.txt" \
                    --ya-log-url "${S3_WEBSITE_PREFIX}/test_logs/${COMPONENT_DIR}${i}/ya_log.txt" \
                    --evlog-url "${S3_WEBSITE_PREFIX}/test_logs/${COMPONENT_DIR}${i}/ya_evlog.jsonl" \
                    > "$SUMMARY_FILE"

                if [ ! -s "$SUMMARY_FILE" ]; then
                    BUILD_ERROR_LOG_URL_FOR_ITERATION=""
                    rm -f "$SUMMARY_FILE"
                fi
            fi

            REPORT_DIR="$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}"
            TRACE_REPORT_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace.html"
            TRACE_OTLP_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace.otlp.jsonl.gz"
            TRACE_INPUTS_URL="$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/trace-inputs.tar.gz"
            if [ -n "${S3_REPORTS_WEBSITE_PREFIX:-}" ]; then
                TRACE_REPORT_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace.html"
                TRACE_OTLP_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace.otlp.jsonl.gz"
                TRACE_INPUTS_URL="$S3_REPORTS_WEBSITE_PREFIX/${COMPONENT_DIR}${i}/trace-inputs.tar.gz"
            fi
            export SUMMARY_OUT_ENV_PATH=$(mktemp -p /home/github)
            set +x
            GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
            export GITHUB_TOKEN
            set -x
            PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                python3 -m scripts.tests.generate_summary \
                --summary-out-path "$ARTIFACTS_DIR/summary/${COMPONENT_DIR}${i}/" \
                --summary-out-env-path "$SUMMARY_OUT_ENV_PATH" \
                --summary-url-prefix "$S3_WEBSITE_PREFIX/summary/${COMPONENT_DIR}${i}/" \
                --build-preset "${platform_name}-${BUILD_PRESET}" \
                --workload-status in_progress \
                --test-target "${TEST_TARGET//\"/}" \
                --test-time $((TEST_END_TIME - TEST_START_TIME)) \
                --build-error-log-url "$BUILD_ERROR_LOG_URL_FOR_ITERATION" \
                --trace-report-url "$TRACE_REPORT_URL" \
                "Tests" ya-test.html "${JUNIT_REPORT_XML}.${i}"
            unset GITHUB_TOKEN
            echo "report_generated=yes" >> "$GITHUB_OUTPUT"

            if [ "$JUNIT_AVAILABLE" -eq 0 ] && [ -n "$BUILD_ERROR_LOG_URL_FOR_ITERATION" ]; then
                {
                    echo
                    echo "[Extracted build errors]($BUILD_ERROR_LOG_URL_FOR_ITERATION)"
                } >> "$SUMMARY_OUT_ENV_PATH"
            fi
            PYTHONPATH="$PYTHONPATH:$GITHUB_WORKSPACE/.github" \
                bash "$GITHUB_WORKSPACE/.github/scripts/tracing/render_ya_trace_bundle.sh" \
                --warning-title "Trace report" \
                --report-dir "$REPORT_DIR" \
                --ya-out "$OUT_DIR" \
                --evlog "$LOG_DIR/${i}/ya_evlog.jsonl" \
                --html-url "$TRACE_REPORT_URL" \
                --otlp-url "$TRACE_OTLP_URL" \
                --inputs-url "$TRACE_INPUTS_URL" \
                --summary "$SUMMARY_OUT_ENV_PATH" \
                -- \
                --attempt-start-ns "$TRACE_START_TIME_NS" \
                --attempt-end-ns "$TRACE_END_TIME_NS" \
                --exit-code "$RC" \
                --result-code "$FAIL_CHECKER_RC" \
                --component "${COMPONENT_DIR%/}" \
                --build-preset "$BUILD_PRESET" \
                --test-target "$TEST_TARGET" \
                --test-type "$TEST_TYPE" \
                --test-size "$TEST_SIZE" \
                --retry "$i" \
                --test-log-url-prefix "$S3_WEBSITE_PREFIX/logs/${COMPONENT_DIR}${i}/" \
                --test-data-url-prefix "$S3_WEBSITE_PREFIX/test_data/${COMPONENT_DIR}${i}$GITHUB_WORKSPACE/"
            cat $SUMMARY_OUT_ENV_PATH | tee -a $GITHUB_STEP_SUMMARY
            echo "::endgroup::"

            if [ -d "$ARTIFACTS_DIR/logs/" ]; then
                echo "::group::cleaning-artifacts-log-dir"
                find "$ARTIFACTS_DIR/logs/" -type f -print0 | while IFS= read -r -d '' file; do
                    if file --mime-type "$file" | grep -q 'application/x-executable'; then
                        echo "$file"
                        rm -f "$file"
                    fi
                done
                echo "::endgroup::"
            fi

            echo "::group::s3-sync"
            aws s3 sync --acl public-read --follow-symlinks --no-progress \
                --exclude "*.trace-inputs.*.tar.gz.tmp" \
                --exclude "*trace-inputs.files" \
                "$ARTIFACTS_DIR/" \
                "$S3_BUCKET_PATH/"
            if [ -n "${S3_REPORTS_BUCKET_PATH:-}" ]; then
                aws s3 sync --acl public-read --follow-symlinks --no-progress \
                    --exclude "*" \
                    --include "summary.json" \
                    --include "trace.html" \
                    --include "trace.manifest.json" \
                    --include "trace.otlp.jsonl.gz" \
                    --include "trace-inputs.tar.gz" \
                    "$REPORT_DIR/" \
                    "$S3_REPORTS_BUCKET_PATH/${COMPONENT_DIR}${i}/"
            fi
            touch "$LOG_DIR/${i}_artifacts_synced"
            echo "::endgroup::"

            if [ $DO_RETRY -eq 1 ]; then
                echo "Retrying..."
                echo "::endgroup::"
                continue
            else
                echo "::endgroup::"
                break
            fi
        done
        echo "exiting with code $FAIL_CHECKER_RC"
        echo "END_EXIT_CODE=${FAIL_CHECKER_RC}" | tee -a "$GITHUB_ENV"
        if [ "$BUILD_FAILED" -eq 1 ]; then
            echo "BUILD_FAILED=1" | tee -a "$GITHUB_ENV"
            {
                echo "failed_tests=yes"
            } >> "$GITHUB_OUTPUT"
        else
            echo "BUILD_FAILED=0" | tee -a "$GITHUB_ENV"
            {
                echo "failed_tests=yes"
            } >> "$GITHUB_OUTPUT"
        fi
    - name: collect VM system logs
      if: ${{ always() }}
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        mkdir -p "$LOG_DIR/system_logs"
        if command -v actions-runner-collect-system-logs.sh > /dev/null; then
            sudo -n actions-runner-collect-system-logs.sh || true
        else
            sudo cp /var/log/kern.log "$LOG_DIR/kern.log" || cp /var/log/kern.log "$LOG_DIR/kern.log" || true
            # shellcheck disable=SC2024
            sudo dmesg -T > "$LOG_DIR/system_logs/dmesg.log" || dmesg -T > "$LOG_DIR/system_logs/dmesg.log" || true
            sudo cp /var/log/atop/atop_* "$LOG_DIR/system_logs/" || cp /var/log/atop/atop_* "$LOG_DIR/system_logs/" || true
            sudo cp /var/log/syslog "$LOG_DIR/system_logs/syslog.log" || cp /var/log/syslog "$LOG_DIR/system_logs/syslog.log" || true
        fi
    - name: dump VM artifacts
      shell: bash --noprofile --norc -eo pipefail -x {0}
      if: ${{ always() }}
      run: |
        echo "::group::dump-vm-artifacts"
        aws s3 sync --acl public-read --follow-symlinks --no-progress "$LOG_DIR/system_logs/" "$S3_BUCKET_PATH/test_logs/${COMPONENT_DIR}"
        echo "::endgroup::"
    - name: sync logs in case of cancel and additional debug info
      if: ${{ cancelled() }}
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        export CANCEL_LOG_DIR="$LOG_DIR/cancel_logs"
        mkdir -p "$CANCEL_LOG_DIR"
        pstree -alpu > "${CANCEL_LOG_DIR}/pstree.log" 2>&1 || true
        ps -eo pid,ppid,stat,etimes,pcpu,pmem,comm,args --sort=-pcpu > "${CANCEL_LOG_DIR}/ps.log" 2>&1 || true
        lsof +c 15  2>&1 | head -n 300 > "${CANCEL_LOG_DIR}/lsof.log" || true
        ss -taop > "${CANCEL_LOG_DIR}/ss.log" 2>&1 || true

        perf record -F 99 -g --call-graph=dwarf -a -o "${CANCEL_LOG_DIR}/perf.data" -- sleep 15 || true
        perf report --stdio -i "${CANCEL_LOG_DIR}/perf.data" > "${CANCEL_LOG_DIR}/perf_report.log" 2>&1 || true
        perf script -i "${CANCEL_LOG_DIR}/perf.data" > "${CANCEL_LOG_DIR}/perf.out" 2>&1 || true

        aws s3 sync --acl public-read --follow-symlinks --no-progress "$LOG_DIR/" "$S3_BUCKET_PATH/test_logs/${COMPONENT_DIR}" || true

        if [ ! -f "${LAST_STEP}_artifacts_synced" ]; then
            echo "Syncing artifacts dir"
            if [ -d "$ARTIFACTS_DIR/logs/" ]; then
                echo "::group::cleaning-artifacts-log-dir"
                find "$ARTIFACTS_DIR/logs/" -type f -print0 | while IFS= read -r -d '' file; do
                    if file --mime-type "$file" | grep -q 'application/x-executable'; then
                        echo "$file"
                        rm -f "$file"
                    fi
                done
                echo "::endgroup::"
            fi
            aws s3 sync --acl private --follow-symlinks --no-progress "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/artifacts/" || true
        fi
    - name: Finish with END_EXIT_CODE
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        echo "Tests ended with exit code $END_EXIT_CODE"
        if [ "$BUILD_FAILED" -eq 1 ]; then
            echo "Build failed, setting END_EXIT_CODE to 237"
            END_EXIT_CODE=237
        fi
        exit $END_EXIT_CODE
    - name: Sync test results to S3
      if: always()
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        echo "::group::s3-sync"
        aws s3 sync --acl public-read --follow-symlinks --no-progress "$ARTIFACTS_DIR/test_reports/" "$S3_BUCKET_PATH/test_reports/"
        echo "::endgroup::"
    - name: Display links to s3 summary
      if: always()
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        echo "::group::s3-report-links"
        for i in $(seq 1 $LAST_STEP); do
            echo ${S3_URL_PREFIX}/summary/${COMPONENT_DIR}${i}/ya-test.html
            echo ${S3_WEBSITE_PREFIX}/summary/${COMPONENT_DIR}${i}/ya-test.html
        done
        echo "::endgroup::"
    - name: upload ya dir to s3 on success
      id: upload_ya_dir
      shell: bash --noprofile --norc -eo pipefail -x {0}
      if: success() && inputs.upload_ya_dir == 'yes'
      run: |
        function grep_ya_tc() {
            # shellcheck disable=SC2009
            ps aux | grep "[y]a-tc" || true
        }
        process_name="ya-tc"
        # by default ya-tc should terminate within 5 minutes
        timeout=360
        start_time=$(date +%s)
        while true; do
            if pgrep -x "$process_name"; then
                echo "$process_name is still running."
                grep_ya_tc
            else
                echo "$process_name is not running."
                grep_ya_tc
                break
            fi

            current_time=$(date +%s)
            elapsed_time=$((current_time - start_time))

            if [ "$elapsed_time" -ge "$timeout" ]; then
                echo "Timeout reached. $process_name is still running. killing it"
                grep_ya_tc
                pkill -f $process_name || true
                grep_ya_tc
                break
            fi

            sleep 5
        done

        # shellcheck disable=SC2009
        ps aux | grep ya
        # checking if files are not corrupted
        echo "::group::ya-files-empty"
        find /home/github/.ya/tools/ -type f -exec file {} \; | grep empty
        echo "::endgroup::"
        ya_bin_path=$(find /home/github/.ya/tools/ -type f -name "ya-bin")
        if [ ! -s "$ya_bin_path" ]; then
            echo "ya-bin exists but is empty"
            echo "empty_ya_bin=yes" >> $GITHUB_OUTPUT
        else
            echo "empty_ya_bin=no" >> $GITHUB_OUTPUT
            echo "::group::upload-ya-dir"

            df -h || true
            EXTENSION=tar.xz
            if [ "$EXTENSION" = "tar.zst" ] || [ "$EXTENSION" = "tar.zstd" ]; then COMPRESS_ARGS='zstd -16 -T60'; fi
            if [ "$EXTENSION" = "tar.xz" ]; then COMPRESS_ARGS='pixz -1'; fi
            if [ "$EXTENSION" = "tar.bz2" ]; then COMPRESS_ARGS=pbzip2; fi
            if [ "$EXTENSION" = "tar.gz" ]; then COMPRESS_ARGS=pigz; fi
            time tar -S -C /home/github -I "$COMPRESS_ARGS" -c -f /home/github/tmp/ya.${EXTENSION} /home/github/.ya
            ls -lsha /home/github/tmp/ya.${EXTENSION}
            time aws s3 cp --no-progress /home/github/tmp/ya.${EXTENSION} "$S3_BUCKET_PATH/ya_archive/${COMPONENT_DIR}ya.${EXTENSION}"
            echo "${S3_WEBSITE_PREFIX}/ya_archive/${COMPONENT_DIR}ya.${EXTENSION}" > /home/github/ya_web_url.txt
            echo "${S3_BUCKET_PATH}/ya_archive/${COMPONENT_DIR}ya.${EXTENSION}" > /home/github/ya_s3_url.txt
            df -h || true
            echo "::endgroup::"
        fi
    - name: upload ya url to s3 on success
      if: success() && inputs.upload_ya_dir == 'yes' && steps.upload_ya_dir.outputs.empty_ya_bin != 'yes'
      uses: actions/upload-artifact@v7.0.1
      with:
        name: ya_web_url_file
        path: /home/github/ya_web_url.txt
    - name: upload ya url to s3 on success
      if: success() && inputs.upload_ya_dir == 'yes' && steps.upload_ya_dir.outputs.empty_ya_bin != 'yes'
      uses: actions/upload-artifact@v7.0.1
      with:
        name: ya_s3_url_file
        path: /home/github/ya_s3_url.txt
    - name: Create directory listing on s3
      if: always()
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        echo "::group::generate-listing"
        python3 .github/scripts/index.py "$S3_BUCKET_PATH/" --generate-indexes --apply
        echo "::endgroup::"
    - name: show free space
      if: always()
      shell: bash --noprofile --norc -eo pipefail -x {0}
      run: |
        mount || true
        df -h || true

    - name: clean up
      if: always()
      shell: bash
      run: |
        echo "::group::clean-up"
        rm -rf $TMP_DIR $SUMMARY_LINKS $BAZEL_REMOTE_PASSWORD_FILE \
               $SUMMARY_OUT_ENV_PATH $SUMMARY_LINKS $GITHUB_TOKEN_FILE
        echo "::endgroup::"