Add @tenki compute backend (Tenki Sandbox microVMs) #2725
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: metaflow.s3-tests.minio | |
| # Uses standalone MinIO Docker container instead of minikube devstack | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - labeled | |
| workflow_dispatch: | |
| permissions: read-all | |
| jobs: | |
| test_s3_with_minio: | |
| if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))) | |
| name: metaflow.s3.minio / Python ${{ matrix.ver }} on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-22.04] | |
| ver: ['3.11'] | |
| timeout-minutes: 90 | |
| env: | |
| AWS_ACCESS_KEY_ID: rootuser | |
| AWS_SECRET_ACCESS_KEY: rootpass123 | |
| AWS_DEFAULT_REGION: us-east-1 | |
| METAFLOW_S3_TEST_ROOT: s3://metaflow-test/metaflow/ | |
| METAFLOW_DATASTORE_SYSROOT_S3: s3://metaflow-test/metaflow/ | |
| AWS_ENDPOINT_URL_S3: http://localhost:9000 | |
| MINIO_TEST: "1" | |
| METAFLOW_S3_TRANSIENT_RETRY_COUNT: "7" | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| ref: refs/pull/${{ github.event.pull_request.number }}/merge | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: ${{ matrix.ver }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install . pytest click boto3 requests numpy pytest-benchmark | |
| - name: Start MinIO | |
| run: | | |
| docker run -d --name minio \ | |
| -p 9000:9000 \ | |
| -e MINIO_ROOT_USER=rootuser \ | |
| -e MINIO_ROOT_PASSWORD=rootpass123 \ | |
| minio/minio server /data | |
| for i in $(seq 1 30); do | |
| if curl -sf http://localhost:9000/minio/health/live; then | |
| echo "MinIO is ready" | |
| break | |
| fi | |
| echo "Waiting for MinIO... ($i/30)" | |
| sleep 2 | |
| done | |
| - name: Create test bucket | |
| run: | | |
| python3 -c " | |
| import boto3 | |
| s3 = boto3.client('s3', | |
| endpoint_url='http://localhost:9000', | |
| aws_access_key_id='rootuser', | |
| aws_secret_access_key='rootpass123') | |
| s3.create_bucket(Bucket='metaflow-test') | |
| print('Bucket metaflow-test created') | |
| " | |
| - name: Execute tests | |
| run: | | |
| cd test/data | |
| PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -s -v --tb=long \ | |
| -k "not 5gb_file and not 3000_files" | |
| - name: Stop MinIO | |
| if: always() | |
| run: docker rm -f minio || true |