Sandbox/rebase/3.0.20250822 3.0 v3

.github/actions/checkout-with-stable-pkgs/action.yml

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: "Stable specs and manifests checkout"
+description: "Checks out the repo, and a stable version of both specs and manifests."
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Checkout stable specs and manifests
+      shell: bash
+      run: git checkout 3.0-stable -- SPECS/ toolkit/resources/manifests/package/*.txt

.github/workflows/check-files.yml

@@ -0,0 +1,140 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: Check Disallowed Files
+
+on:
+  push:
+    branches: [main, 2.0*, 3.0*, fasttrack/*]
+  pull_request:
+    branches: [main, 2.0*, 3.0*, fasttrack/*]
+
+jobs:
+
+  build:
+    name: Check Disallowed Files
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Get base commit for PRs
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        git fetch origin ${{ github.base_ref }}
+        echo "base_sha=$(git rev-parse origin/${{ github.base_ref }})" >> $GITHUB_ENV
+        echo "Merging ${{ github.sha }} into ${{ github.base_ref }}"
+
+    - name: Get base commit for Pushes
+      if: ${{ github.event_name == 'push' }}
+      run: |
+        git fetch origin ${{ github.event.before }}
+        echo "base_sha=${{ github.event.before }}" >> $GITHUB_ENV
+        echo "Merging ${{ github.sha }} into ${{ github.event.before }}"
+
+    - name: Get the changed files
+      run: |
+        echo "Files changed: '$(git diff-tree --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})'"
+        changed_files=$(git diff-tree --diff-filter=AM --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})
+        echo "Files to validate: '${changed_files}'"
+        echo "changed-files<<EOF" >> $GITHUB_ENV
+        echo "${changed_files}" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
+
+    - name: Check for disallowed file types
+      run: |
+        if [[ -z "${{ env.changed-files }}" ]]; then
+          echo "No files to validate. Exiting."
+          exit 0
+        fi
+
+        echo "Checking files..."
+        error_found=0
+
+        # Read disallowed extensions from the configuration file
+        if [[ ! -f ".github/workflows/disallowed-extensions.txt" ]]; then
+          echo "Configuration file '.github/workflows/disallowed-extensions.txt' not found. Skipping check."
+          exit 0
+        fi
+
+        # Create array of disallowed extensions
+        mapfile -t disallowed_extensions < .github/workflows/disallowed-extensions.txt
+        if [[ $? -ne 0 ]]; then
+          echo "Error occurred while reading disallowed extensions. Exiting."
+          exit 1
+        fi
+
+        # Check each changed file
+        while IFS= read -r file; do
+          if [[ -z "$file" ]]; then
+            continue
+          fi
+
+          echo "Checking file: $file"
+
+          # Get file extension (convert to lowercase for comparison)
+          extension=$(echo "${file##*.}" | tr '[:upper:]' '[:lower:]')
+          filename=$(basename "$file")
+
+          # Check if file should be in blob store
+          should_be_in_blob_store=false
+
+          # Check against disallowed extensions
+          for disallowed_ext in "${disallowed_extensions[@]}"; do
+            # Remove any whitespace and comments
+            clean_ext=$(echo "$disallowed_ext" | sed 's/#.*//' | xargs)
+            if [[ -z "$clean_ext" ]]; then
+              continue
+            fi
+
+            if [[ "$extension" == "$clean_ext" ]]; then
+              should_be_in_blob_store=true
+              break
+            fi
+          done
+
+          # Additional checks for binary files and large files
+          if [[ -f "$file" ]]; then
+            # Check if file is binary (but allow .sh files even if executable)
+            if  file "$file" | grep -q "binary\|archive\|compressed"; then
+              should_be_in_blob_store=true
+            fi
+
+            # Check file size (files > 1MB should be in blob store)
+            file_size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo 0)
+            if [[ $file_size -gt 1048576 ]]; then  # 1MB
+              should_be_in_blob_store=true
+            fi
+          fi
+
+          if [[ "$should_be_in_blob_store" == "true" ]]; then
+            1>&2 echo "**** ERROR ****"
+            1>&2 echo "File '$file' should be stored in blob store, not in git repository."
+            1>&2 echo "Reason: Images, Large files, binaries, tarballs, and non-text files slow down git operations"
+            1>&2 echo "and cannot be efficiently diffed. Please upload to blob store instead."
+            1>&2 echo "**** ERROR ****"
+            error_found=1
+          fi
+        done <<< "${{ env.changed-files }}"
+
+        if [[ $error_found -eq 1 ]]; then
+          echo ""
+          echo "=========================================="
+          echo "FILES THAT SHOULD BE IN BLOB STORE DETECTED"
+          echo "=========================================="
+          echo "The following file types should be stored in blob store:"
+          echo "- Source tarballs (.tar.gz, .tar.xz, .zip, etc.)"
+          echo "- Binary files (.bin, .exe, .so, .dll, etc.)"
+          echo "- Images (.gif, .bmp, etc.)"
+          echo "- Archives (.rar, .7z, .tar, etc.)"
+          echo "- Large files (> 1MB)"
+          echo "- Any non-text files that cannot be efficiently diffed"
+          echo ""
+          echo "Please upload these files to the blob store and reference them"
+          echo "in your spec files or configuration instead of checking them into git."
+          echo "=========================================="
+          exit 1
+        fi
+
+        echo "All files are appropriate for git storage."

.github/workflows/check-package-builds.yml

@@ -0,0 +1,165 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# This check verifies basic package build success and failure cases.
+# It should only be dependent on toolkit changes, not on the specs.
+# This is why each build uses the 3.0-stable version of the specs and manifests.
+
+name: Package build checks
+
+env:
+  REGULAR_PKG: words
+  REGULAR_PKG_SPEC_PATH: SPECS/words/words.spec
+  TOOLCHAIN_PKG: xz
+
+on:
+  push:
+    branches: [3.0*, fasttrack/3.0]
+    paths:
+      - ".github/workflows/check-package-builds.yml"
+      - "toolkit/Makefile"
+      - "toolkit/scripts/*"
+      - "toolkit/tools/*"
+  pull_request:
+    branches: [3.0*, fasttrack/3.0]
+    paths:
+      - ".github/workflows/check-package-builds.yml"
+      - "toolkit/Makefile"
+      - "toolkit/scripts/*"
+      - "toolkit/tools/*"
+
+jobs:
+  package-checks:
+    name: ${{ matrix.check-name }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - check-name: "Simple package build succeeds"
+            package-type: "REGULAR_PKG"
+            extra-args: ""
+
+          - check-name: "Simple package build fails"
+            package-type: "REGULAR_PKG"
+            error-pattern: "Number of failed SRPMs:\\s+1\\s*$"
+            extra-args: ""
+            build-prep: |
+              # Adding an invalid command to the '%prep' section will cause the build to fail.
+              sed -i '/%prep/a this-command-should-fail-because-its-not-a-command-at-all' "$REGULAR_PKG_SPEC_PATH"
+
+          - check-name: "Toolchain package rebuild succeeds"
+            package-type: "TOOLCHAIN_PKG"
+            extra-args: "ALLOW_TOOLCHAIN_REBUILDS=y"
+
+          - check-name: "Toolchain package rebuild fails"
+            package-type: "TOOLCHAIN_PKG"
+            error-pattern: "Number of toolchain SRPM conflicts:\\s+1\\s*$"
+            extra-args: "ALLOW_TOOLCHAIN_REBUILDS=n"
+            build-prep: ""
+
+          - check-name: "None license check does not break the build"
+            package-type: "REGULAR_PKG"
+            extra-args: "LICENSE_CHECK_MODE=none"
+            build-prep: |
+              license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
+              if [[ -z "$license_file_name" ]]; then
+                  echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
+                  exit 1
+              fi
+              # Tagging a license file as a documentation file will not fail the license check on the 'none' level.
+              sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
+
+          - check-name: "Warning-only license check does not break the build"
+            package-type: "REGULAR_PKG"
+            extra-args: "LICENSE_CHECK_MODE=warn"
+            build-prep: |
+              license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
+              if [[ -z "$license_file_name" ]]; then
+                echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
+                exit 1
+              fi
+              # Tagging a license file as a documentation file will not fail the license check on the 'warn' level.
+              sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
+
+          - check-name: "Fatal license check succeeds on duplicated license as documentation"
+            package-type: "REGULAR_PKG"
+            extra-args: "LICENSE_CHECK_MODE=fatal"
+            build-prep: |
+              license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
+              if [[ -z "$license_file_name" ]]; then
+                echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
+                exit 1
+              fi
+              # Tagging a license file as a documentation file will not fail the license check on the 'fatal' level.
+              sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
+
+          - check-name: "Fatal license check fails"
+            package-type: "REGULAR_PKG"
+            error-pattern: "Number of SRPMs with license errors:\\s+1\\s*$"
+            extra-args: "LICENSE_CHECK_MODE=fatal"
+            build-prep: |
+              if ! grep -q '^%license' "$REGULAR_PKG_SPEC_PATH"; then
+                echo "ERROR: no '%license' macro found in the spec $REGULAR_PKG_SPEC_PATH"
+                exit 1
+              fi
+              # Tagging a license file as a documentation file will cause the license check to fail.
+              sed -i "s/^%license/%doc/" "$REGULAR_PKG_SPEC_PATH"
+
+          - check-name: "Pedantic license check fails"
+            package-type: "REGULAR_PKG"
+            error-pattern: "Number of SRPMs with license errors:\\s+1\\s*$"
+            extra-args: "LICENSE_CHECK_MODE=pedantic"
+            build-prep: |
+              license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
+              if [[ -z "$license_file_name" ]]; then
+                echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
+                exit 1
+              fi
+              sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout a stable version of the specs
+        uses: ./.github/actions/checkout-with-stable-pkgs
+
+      - name: Prepare the build environment
+        if: ${{ matrix.build-prep != '' }}
+        run: |
+          set -euo pipefail
+
+          ${{ matrix.build-prep }}
+
+      - name: Run the build
+        run: |
+          set -euo pipefail
+
+          if sudo make -C toolkit -j$(nproc) build-packages \
+            PACKAGE_REBUILD_LIST="${{ env[matrix.package-type] }}" \
+            REBUILD_TOOLS=y \
+            SRPM_PACK_LIST="${{ env[matrix.package-type] }}" \
+            ${{ matrix.extra-args }} 2>&1 | tee build.log; then
+            touch build.succeeded
+          fi
+
+      - name: Check the results
+        run: |
+          set -euo pipefail
+
+          if [[ -z "${{ matrix.error-pattern }}" ]]; then
+            if [[ ! -f build.succeeded ]]; then
+              echo "Build failed, but it was expected to succeed."
+              exit 1
+            fi
+          else
+            if [[ -f build.succeeded ]]; then
+              echo "Build succeeded, but it was expected to fail."
+              exit 1
+            fi
+
+            if ! grep -qP '${{ matrix.error-pattern }}' build.log; then
+              echo "Build failed, but not with the expected error message."
+              exit 1
+            fi
+          fi

.github/workflows/check-package-cgmanifest.yml

@@ -56,8 +56,17 @@ jobs:
           echo "Files to validate: '${changed_specs}'"
           echo "updated-specs=${changed_specs}" >> "$GITHUB_ENV"
 
-      - name: Check each spec
-        run: |
-          .github/workflows/overwrite_shell_link.sh
-          .github/workflows/validate-cg-manifest.sh ${{ env.updated-specs }}
-        shell: bash
+    - name: Get the changed files
+      run: |
+        echo "Files changed: '$(git diff-tree --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})'"
+        changed_specs=$(git diff-tree --diff-filter=d  --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }} | { grep "SPECS.*/.*\.spec$" || test $? = 1; })
+        echo "Files to validate: '${changed_specs}'"
+        echo "updated-specs=$(echo ${changed_specs})" >> $GITHUB_ENV
+
+    - name: Build the worker chroot
+      if: ${{ env.updated-specs != '' }}
+      run: sudo make -C toolkit -j$(nproc) chroot-tools REBUILD_TOOLS=y DAILY_BUILD_ID=lkg
+
+    - name: Check each spec
+      if: ${{ env.updated-specs != '' }}
+      run: .github/workflows/validate-cg-manifest.sh build/worker/worker_chroot.tar.gz ${{ env.updated-specs }}