metaflow/.github/workflows/metaflow.s3_tests.minio.yml at e282c2ea365449e88a3031ba3c5d8ea26ef38b2e · Netflix/metaflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
name: metaflow.s3-tests.minio

on:
  push:
    branches:
    - master
  pull_request:
    types:
      - opened
      - synchronize
      - labeled

permissions: read-all

jobs:
  test_s3_with_minio:
    if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))))
    name: metaflow.s3.minio / Python ${{ matrix.ver }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-22.04]
        ver: ['3.8', '3.9', '3.10', '3.11', '3.12']

    timeout-minutes: 60

    env:
      AWS_ACCESS_KEY_ID: rootuser
      AWS_SECRET_ACCESS_KEY: rootpass123
      AWS_DEFAULT_REGION: us-east-1
      METAFLOW_S3_TEST_ROOT: s3://metaflow-test/metaflow/
      METAFLOW_DATASTORE_SYSROOT_S3: s3://metaflow-test/metaflow/
      AWS_ENDPOINT_URL_S3: http://localhost:9000
      MINIO_TEST: "1"
      METAFLOW_S3_TRANSIENT_RETRY_COUNT: "5"

    steps:
    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: recursive
    - name: Set up Python
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
      with:
        python-version: ${{ matrix.ver }}
    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install . pytest click boto3 requests numpy pytest-benchmark
    - name: Start MinIO
      run: |
        docker run -d --name minio \
          -p 9000:9000 \
          -e MINIO_ROOT_USER=rootuser \
          -e MINIO_ROOT_PASSWORD=rootpass123 \
          minio/minio server /data
        for i in $(seq 1 30); do
          if curl -sf http://localhost:9000/minio/health/live; then
            echo "MinIO is ready"
            break
          fi
          echo "Waiting for MinIO... ($i/30)"
          sleep 2
        done
    - name: Create test bucket
      run: |
        python3 -c "
        import boto3
        s3 = boto3.client('s3',
            endpoint_url='http://localhost:9000',
            aws_access_key_id='rootuser',
            aws_secret_access_key='rootpass123')
        s3.create_bucket(Bucket='metaflow-test')
        print('Bucket metaflow-test created')
        "
    - name: Execute tests
      run: |
        cd test/data
        PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -s -v
    - name: Stop MinIO
      if: always()
      run: docker rm -f minio || true