fix: crash-safe atomic JSON read-modify-write #1740
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: metaflow.s3-tests.minio | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - labeled | |
| permissions: read-all | |
| jobs: | |
| test_s3_with_minio: | |
| if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))) | |
| name: metaflow.s3.minio / Python ${{ matrix.ver }} on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| ver: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| ref: refs/pull/${{ github.event.pull_request.number }}/merge | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: ${{ matrix.ver }} | |
| - name: Install Python ${{ matrix.ver }} dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install . kubernetes tox numpy pytest click boto3 requests pylint pytest-benchmark | |
| - name: Start MinIO development environment | |
| run: | | |
| echo "Starting environment in the background..." | |
| MINIKUBE_CPUS=2 metaflow-dev all-up & | |
| # Give time to spin up. Adjust as needed: | |
| sleep 150 | |
| - name: Execute tests | |
| run: | | |
| cat <<EOF | metaflow-dev shell | |
| # Set MinIO environment variables | |
| export AWS_ACCESS_KEY_ID=rootuser | |
| export AWS_SECRET_ACCESS_KEY=rootpass123 | |
| export AWS_DEFAULT_REGION=us-east-1 | |
| export METAFLOW_S3_TEST_ROOT=s3://metaflow-test/metaflow/ | |
| export METAFLOW_DATASTORE_SYSROOT_S3=s3://metaflow-test/metaflow/ | |
| export AWS_ENDPOINT_URL_S3=http://localhost:9000 | |
| export MINIO_TEST=1 | |
| # Run the same test command as the original workflow | |
| cd test/data | |
| PYTHONPATH=\$(pwd)/../../ python3 -m pytest --benchmark-skip -s -v | |
| EOF | |
| - name: Tear down environment | |
| run: | | |
| metaflow-dev down |