The OSS Review Toolkit (ORT) #2224
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: The OSS Review Toolkit (ORT) | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| branch_name: | |
| description: "The branch to run against the ORT tool" | |
| required: true | |
| default: "main" | |
| jobs: | |
| run-ort: | |
| name: Create attribution files | |
| runs-on: ubuntu-latest | |
| # 1. For workflow_dispatch, always allow | |
| # 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches | |
| # NOTE: Temporarily removed repository_owner restriction for testing PHP ORT integration | |
| if: > | |
| (github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-'))) | |
| continue-on-error: true | |
| env: | |
| PHP_ATTRIBUTIONS: "THIRD_PARTY_LICENSES_PHP" | |
| INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| HEAD_REF: ${{ github.head_ref }} | |
| steps: | |
| - name: Setup target branch and commit | |
| run: | | |
| if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then | |
| echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV | |
| elif [ "$EVENT_NAME" == 'pull_request' ]; then | |
| echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout target branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.TARGET_BRANCH }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 # Fetch all history for all branches and tags | |
| submodules: recursive | |
| - name: Setup target commit | |
| run: | | |
| echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV | |
| - name: Set up JDK 11 for the ORT package | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: 11 | |
| - name: Cache ORT and Gradle packages | |
| uses: actions/cache@v4 | |
| id: cache-ort | |
| with: | |
| path: | | |
| ./ort | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-ort | |
| - name: Checkout ORT Repository | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: "oss-review-toolkit/ort" | |
| path: "./ort" | |
| ref: "46.0.0" | |
| submodules: recursive | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@c56a35af9328d0bc581dc86c05e58f97f7c38a0e # 1.85 | |
| - name: Build and install ORT | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| working-directory: ./ort/ | |
| run: | | |
| export JAVA_OPTS="$JAVA_OPTS -Xmx8g" | |
| ./gradlew installDist | |
| - name: Create ORT config file for Rust projects | |
| run: | | |
| mkdir -p ~/.ort/config | |
| cat << EOF > ~/.ort/config/config.yml | |
| ort: | |
| analyzer: | |
| skip_excluded: true | |
| allowDynamicVersions: true | |
| enabledPackageManagers: [Cargo, Composer, GoMod, GradleInspector] | |
| excludes: | |
| paths: | |
| - pattern: "**/cpp/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "C++ bindings directory should not be analyzed by ORT" | |
| - pattern: "valkey-glide/cpp/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "C++ client bindings, not part of dependency analysis" | |
| - pattern: "**/cpp/Cargo.toml" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "C++ Cargo.toml should be excluded" | |
| - pattern: "valkey-glide/benchmarks/rust/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "Benchmark code, not core dependencies" | |
| - pattern: "valkey-glide/python/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "Python client bindings, not core Rust dependencies" | |
| - pattern: "valkey-glide/java/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "Java client bindings, not core Rust dependencies" | |
| - pattern: "valkey-glide/csharp/rust/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "C# client bindings, not core Rust dependencies" | |
| - pattern: "valkey-glide/node/rust-client/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "Node.js client bindings, not core Rust dependencies" | |
| scopes: | |
| - pattern: ".*cpp.*" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Exclude any package containing cpp in the name" | |
| EOF | |
| cat ~/.ort/config/config.yml | |
| - name: Create ORT config file for PHP project (exclude Java/Gradle) | |
| run: | | |
| mkdir -p ~/.ort/config | |
| cat << EOF > ~/.ort/config/config-php.yml | |
| ort: | |
| analyzer: | |
| skip_excluded: true | |
| allowDynamicVersions: true | |
| enabledPackageManagers: [Composer] | |
| excludes: | |
| paths: | |
| - pattern: "ort/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "ORT tool directory with test files, not part of PHP project" | |
| - pattern: "**/funTest/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "ORT functional test files should be excluded" | |
| - pattern: "**/src/funTest/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "ORT functional test source files should be excluded" | |
| - pattern: "ort/plugins/package-managers/composer/src/funTest/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "ORT Composer test fixtures should be excluded" | |
| - pattern: "**/synthetic/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "ORT synthetic test projects should be excluded" | |
| - pattern: "valkey-glide/**" | |
| reason: "DOCUMENTATION_OF" | |
| comment: "Submodule not part of PHP project" | |
| - pattern: "**/cpp/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "C++ directories should not be analyzed in PHP context" | |
| - pattern: "build/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Build artifacts" | |
| - pattern: "modules/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Compiled PHP modules" | |
| - pattern: "autom4te.cache/**" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Autotools cache" | |
| - pattern: "**/Cargo.toml" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Rust manifests not relevant for PHP analysis" | |
| - pattern: "**/*.o" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Object files" | |
| - pattern: "**/*.lo" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Libtool object files" | |
| - pattern: "**/*.la" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Libtool archive files" | |
| - pattern: "**/*.dep" | |
| reason: "BUILD_TOOL_OF" | |
| comment: "Dependency files" | |
| EOF | |
| cat ~/.ort/config/config-php.yml | |
| ### Rust glide-core ### | |
| - name: Run ORT tools for glide-core | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/valkey-glide/glide-core" | |
| ### Rust FFI ### | |
| - name: Run ORT tools for FFI | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/valkey-glide/ffi" | |
| ### PHP ### | |
| - name: Set up PHP | |
| uses: shivammathur/setup-php@728c6c6b8cf02c2e48117716a91ee48313958a19 # v2 | |
| with: | |
| php-version: "8.1" | |
| tools: composer | |
| coverage: none | |
| - name: Install PHP dependencies | |
| run: | | |
| composer install --no-dev --no-interaction --prefer-dist --optimize-autoloader | |
| echo "Composer install completed. Checking for composer.lock:" | |
| ls -la composer.lock || echo "Warning: composer.lock not found" | |
| echo "Current directory contents:" | |
| ls -la | |
| - name: Run ORT tools for PHP | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}" | |
| config_file: "config-php.yml" | |
| ### Additional License Scanning with Scancode ### | |
| - name: Install scancode-toolkit | |
| run: | | |
| python3 -m pip install --upgrade pip setuptools wheel | |
| # Use exact versions from scancode-toolkit requirements.txt | |
| python_version=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") | |
| if [[ $(echo "$python_version >= 3.10" | bc -l) -eq 1 ]]; then | |
| pip install "click==8.2.1" | |
| else | |
| pip install "click==8.1.7" | |
| fi | |
| pip install scancode-toolkit | |
| - name: Install and setup aboutcode-toolkit | |
| run: | | |
| git clone https://github.com/nexB/aboutcode-toolkit.git | |
| cd aboutcode-toolkit | |
| ./configure | |
| - name: Run scancode license scanning | |
| run: | | |
| set -e | |
| set -o pipefail | |
| scancode -n 4 --license . \ | |
| --ignore "*valkey-glide/*" \ | |
| --ignore "*bin/*" \ | |
| --ignore "*lib/*" \ | |
| --ignore "*vendor/*" \ | |
| --ignore "*build/*" \ | |
| --ignore "*path/*" \ | |
| --ignore "*modules/*" \ | |
| --ignore "*autom4te.cache/*" \ | |
| --ignore "*venv/*" \ | |
| --ignore "*ort/*" \ | |
| --ignore "*myenv/*" \ | |
| --ignore "*THIRD_PARTY_LICENSES_PHP*" \ | |
| --json report.json --csv report.csv -v 2>&1 | tee scancode_output.log | |
| if [ ! -f "report.json" ] || [ ! -s "report.json" ]; then | |
| echo "ERROR: report.json not created or empty" | |
| cat scancode_output.log | |
| exit 1 | |
| fi | |
| - name: Generate attribution with aboutcode-toolkit | |
| run: | | |
| if [ ! -f "report.json" ]; then | |
| echo "ERROR: report.json not found. Scancode step may have failed." | |
| echo "Checking scancode output:" | |
| cat scancode_output.log || echo "No scancode output log found" | |
| exit 1 | |
| fi | |
| cd aboutcode-toolkit | |
| mkdir -p out | |
| ./about attrib --scancode ../report.json --template ../aboutcode.template out/rep.csv | |
| - name: Clean output and append to PHP licenses | |
| run: | | |
| awk 'NF{blank=0; print; next} blank < 2 {blank++; print}' aboutcode-toolkit/out/rep.csv > output.txt | |
| cat output.txt >> $PHP_ATTRIBUTIONS | |
| ### Get licenses ### | |
| - name: Retrieve licenses list | |
| working-directory: ./utils | |
| run: | | |
| { | |
| echo 'LICENSES_LIST<<EOF' | |
| python3 get_licenses_from_ort.py | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| ### Upload licenses ### | |
| - name: Get current date | |
| id: date | |
| run: | | |
| CURR_DATE=$(date +'%Y-%m-%d-%H') | |
| echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT | |
| - name: Upload the final package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: final-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/final_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the skipped package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skipped-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/skipped_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the unknown/unapproved package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unapproved-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/unapproved_package_list.txt | |
| retention-days: 30 | |
| - name: Check for unapproved packages | |
| run: | | |
| if [ -s utils/unapproved_package_list.txt ]; then | |
| echo "::error::Found unapproved packages. Please review utils/unapproved_package_list.txt" | |
| cat utils/unapproved_package_list.txt | |
| exit 1 | |
| else | |
| echo "No unapproved packages found." | |
| fi | |
| ### Check for attributions diff ### (FFI attributions merged to FFI clients' attributions) | |
| - name: Check for diff | |
| run: | | |
| cat valkey-glide/ffi/ort_results/NOTICE_DEFAULT >> $PHP_ATTRIBUTIONS | |
| GIT_DIFF=`git diff $PHP_ATTRIBUTIONS` | |
| if [ -n "$GIT_DIFF" ]; then | |
| echo "FOUND_DIFF=true" >> $GITHUB_ENV | |
| else | |
| echo "FOUND_DIFF=false" >> $GITHUB_ENV | |
| fi | |
| ### Create PR, Note a potential race on the source branch ### | |
| - name: Create pull request | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} | |
| run: | | |
| export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH" | |
| echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH" | |
| git config --global user.email "valkey-glide@lists.valkey.io" | |
| git config --global user.name "ort-bot" | |
| git checkout -b ${ORT_DIFF_BRANCH_NAME} | |
| git add $PHP_ATTRIBUTIONS | |
| git commit -m "Updated attribution files" -s | |
| git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f | |
| # Check if PR already exists | |
| existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number') | |
| if [ -z "$existing_pr" ]; then | |
| # Create a new PR if none exists | |
| title="Updated attribution files for commit ${TARGET_COMMIT}" | |
| gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" | |
| echo "Pull request created successfully." | |
| else | |
| # Update the existing PR | |
| echo "Pull request #$existing_pr already exists. Updating branch." | |
| gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" | |
| echo "Pull request updated successfully." | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| ### Warn of outdated attributions for PR ### | |
| - name: Warn of outdated attributions due to the PR | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }} | |
| run: | | |
| ATTRIBUTION_FILES=( | |
| "${{ env.PHP_ATTRIBUTIONS }}" | |
| ) | |
| MESSAGE="WARNING! The attribution files differ in this PR. Please ensure an updating PR is issued using a scheduled or manual run of this workflow!" | |
| # Echo the message to the console | |
| echo "$MESSAGE" | |
| # Emit a general warning in the action log | |
| echo "::warning::$MESSAGE" | |
| # Loop through the attribution files | |
| for FILE in "${ATTRIBUTION_FILES[@]}"; do | |
| if git diff --quiet "$FILE"; then | |
| continue | |
| else | |
| # Emit a warning associated with the changed file | |
| echo "::warning file=$FILE::WARNING! The attribution file '$FILE' differs in this PR." | |
| fi | |
| done |