diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5887bdf3..4a235439 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: python-version: "3.13" - name: Install the latest version of uv - uses: astral-sh/setup-uv@eb1897b8dc4b5d5bfe39a428a8f2304605e0983c # v7 + uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2 - name: Sync dependencies run: uv sync diff --git a/.github/workflows/test-and-build.yaml b/.github/workflows/test-and-build.yaml index e4812d0b..1fb0fc5e 100644 --- a/.github/workflows/test-and-build.yaml +++ b/.github/workflows/test-and-build.yaml @@ -27,7 +27,7 @@ jobs: python-version: "3.13" - name: Install the latest version of uv - uses: astral-sh/setup-uv@eb1897b8dc4b5d5bfe39a428a8f2304605e0983c # v7 + uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2 - name: Sync dependencies run: uv sync @@ -47,7 +47,7 @@ jobs: - name: Authenticate to Google Cloud id: auth - uses: google-github-actions/auth@v1.1.1 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: workload_identity_provider: "projects/${{ secrets.GAR_PROJECT_NUMBER }}/locations/global/workloadIdentityPools/gh-actions/providers/gh-actions" service_account: "gh-actions-microdata@${{ secrets.GAR_PROJECT_ID }}.iam.gserviceaccount.com" @@ -55,10 +55,10 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Login to Artifact Registry - uses: docker/login-action@v3 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ env.REGISTRY }} username: "oauth2accesstoken" @@ -66,7 +66,7 @@ jobs: - name: Extract build metadata for Docker id: build_metadata - uses: docker/metadata-action@v5 + uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE }} tags: | @@ -74,7 +74,7 @@ jobs: type=raw,value=latest - name: Build and push docker image to Artifact Registry - uses: docker/build-push-action@v4 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . provenance: false @@ -96,13 +96,16 @@ jobs: steps: - name: Call workflow uses: actions/github-script@v7 + env: + INTEGRATION_TEST_REPO: ${{ secrets.INTEGRATION_TEST_REPO }} + INTEGRATION_TEST_WORKFLOW: "${{ secrets.INTEGRATION_TEST_WORKFLOW }}.yaml" with: github-token: ${{ secrets.INTEGRATION_TEST_PAT }} script: | await github.rest.actions.createWorkflowDispatch({ owner: 'statisticsnorway', - repo: '${{ secrets.INTEGRATION_TEST_REPO }}', - workflow_id: '${{ secrets.INTEGRATION_TEST_WORKFLOW }}.yaml', + repo: process.env.INTEGRATION_TEST_REPO, + workflow_id: process.env.INTEGRATION_TEST_WORKFLOW, ref: 'main' }) @@ -137,13 +140,16 @@ jobs: steps: - name: Call workflow uses: actions/github-script@v7 + env: + DEPLOY_REPO: ${{ secrets.DEPLOY_REPO }} + DEPLOY_WORKFLOW: "${{ secrets.DEPLOY_WORKFLOW }}.yaml" with: github-token: ${{ secrets.DEPLOY_PAT }} script: | await github.rest.actions.createWorkflowDispatch({ owner: 'statisticsnorway', - repo: '${{ secrets.DEPLOY_REPO }}', - workflow_id: '${{ secrets.DEPLOY_WORKFLOW }}.yaml', + repo: process.env.DEPLOY_REPO, + workflow_id: process.env.DEPLOY_WORKFLOW, ref: 'master', inputs: { environment: "PROD", diff --git a/.github/workflows/update-tools-version.yaml b/.github/workflows/update-tools-version.yaml index d846e3b8..88658e38 100644 --- a/.github/workflows/update-tools-version.yaml +++ b/.github/workflows/update-tools-version.yaml @@ -24,7 +24,7 @@ jobs: python-version: "3.13" - name: Install the latest version of uv - uses: astral-sh/setup-uv@eb1897b8dc4b5d5bfe39a428a8f2304605e0983c # v7 + uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2 - name: Sync dependencies run: uv sync @@ -46,24 +46,28 @@ jobs: - name: Update Microdata Tools Dependency if: steps.check_version.outputs.new_version + env: + NEW_VERSION: ${{ steps.check_version.outputs.new_version }} run: | git config --global user.name 'github-actions' git config --global user.email 'github-actions@github.com' git pull origin main - git checkout -b bump-microdata-tools-${{ steps.check_version.outputs.new_version }} - uv add 'microdata-tools==${{ steps.check_version.outputs.new_version }}' + git checkout -b "bump-microdata-tools-$NEW_VERSION" + uv add "microdata-tools==$NEW_VERSION" git add pyproject.toml uv.lock - git commit -m "Update microdata-tools to ${{ steps.check_version.outputs.new_version }}" - git push origin bump-microdata-tools-${{ steps.check_version.outputs.new_version }} + git commit -m "Update microdata-tools to $NEW_VERSION" + git push origin "bump-microdata-tools-$NEW_VERSION" - name: Create Pull Request if: steps.check_version.outputs.new_version + env: + NEW_VERSION: ${{ steps.check_version.outputs.new_version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr create \ --base main \ - --head bump-microdata-tools-${{ steps.check_version.outputs.new_version }} \ - --title "Update microdata-tools to ${{ steps.check_version.outputs.new_version }}" \ - --body "This PR updates microdata-tools to version ${{ steps.check_version.outputs.new_version }}." + --head "bump-microdata-tools-$NEW_VERSION" \ + --title "Update microdata-tools to $NEW_VERSION" \ + --body "This PR updates microdata-tools to version $NEW_VERSION." # --reviewer statisticsnorway/microdata-developers # Reviwer not found [should work according to docs](https://cli.github.com/manual/gh_pr_create) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/job_executor/app.py b/job_executor/app.py index d17049e8..33e47fae 100644 --- a/job_executor/app.py +++ b/job_executor/app.py @@ -1,6 +1,5 @@ import logging import time -from pathlib import Path from job_executor.adapter import datastore_api from job_executor.adapter.fs import LocalStorageAdapter @@ -24,7 +23,9 @@ def initialize_app() -> Manager: try: rollback.fix_interrupted_jobs() for rdn in datastore_api.get_datastores(): - local_storage = LocalStorageAdapter(Path(environment.datastore_dir)) + local_storage = LocalStorageAdapter( + datastore_api.get_datastore_directory(rdn) + ) if local_storage.datastore_dir.temporary_backup_exists(): raise StartupException(f"tmp directory exists for {rdn}") return Manager(