metaflow-nflx-extensions/.github/workflows/test-fastdata.yml at 5cf35ceb381f9096b71fdac09e05e9173333d887 · Netflix/metaflow-nflx-extensions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
name: test-fastdata

on:
  pull_request:
  push:
    branches:
      - main

jobs:
  # -------------------------------------------------------------------------
  # Unit tests — no external services required
  # -------------------------------------------------------------------------
  unit:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.11", "3.12"]
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          submodules: recursive

      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install metaflow-fastdata + test deps
        working-directory: metaflow-fastdata
        run: |
          pip install metaflow
          pip install -e .
          pip install pytest pytest-html pytest-cov

      - name: Run unit tests
        working-directory: metaflow-fastdata
        run: |
          pytest -m "not integration" -v \
            --html=report.html --self-contained-html \
            --junit-xml=junit.xml \
            --cov=metaflow_fastdata \
            --cov-report=term-missing \
            --cov-report=xml:coverage.xml \
            --cov-report=html:htmlcov \
            --cov-branch

      - name: Publish test results
        uses: dorny/test-reporter@v1
        if: always()
        with:
          name: Unit Tests (py${{ matrix.python-version }})
          path: metaflow-fastdata/junit.xml
          reporter: java-junit

      - name: Post coverage summary
        if: always()
        working-directory: metaflow-fastdata
        run: |
          echo "## Coverage — Unit Tests (py${{ matrix.python-version }})" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          python -m coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
          echo '```' >> $GITHUB_STEP_SUMMARY

      - name: Upload test report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: unit-test-report-py${{ matrix.python-version }}
          path: |
            metaflow-fastdata/report.html
            metaflow-fastdata/junit.xml
            metaflow-fastdata/coverage.xml
            metaflow-fastdata/htmlcov/
          include-hidden-files: true

  # -------------------------------------------------------------------------
  # Integration tests — Hive Metastore + MinIO via docker compose
  # -------------------------------------------------------------------------
  integration:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          submodules: recursive

      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
        with:
          python-version: "3.11"

      - name: Start Hive Metastore + MinIO
        working-directory: metaflow-fastdata
        run: |
          docker compose up -d hive-metastore minio minio-init
          # Wait up to 3 minutes for HMS to be ready
          echo "Waiting for Hive Metastore on port 9083..."
          for i in $(seq 1 36); do
            nc -z localhost 9083 && echo "HMS ready after ${i}x5s" && break
            [ $i -eq 36 ] && echo "Timed out waiting for HMS" && docker compose logs hive-metastore && exit 1
            sleep 5
          done

      - name: Install metaflow-fastdata + test deps + PyHive
        working-directory: metaflow-fastdata
        run: |
          pip install metaflow
          pip install -e .
          pip install pytest pytest-html pytest-cov "PyHive[thrift]" hmsclient boto3

      - name: Run integration tests
        working-directory: metaflow-fastdata
        env:
          METAFLOW_HIVE_METASTORE_URI: thrift://localhost:9083
          MINIO_ENDPOINT: http://localhost:9000
          AWS_ACCESS_KEY_ID: minioadmin
          AWS_SECRET_ACCESS_KEY: minioadmin
          AWS_DEFAULT_REGION: us-east-1
          AWS_ENDPOINT_URL: http://localhost:9000
        run: |
          pytest -m integration -v \
            --html=report.html --self-contained-html \
            --junit-xml=junit.xml \
            --cov=metaflow_fastdata \
            --cov-report=term-missing \
            --cov-report=xml:coverage.xml \
            --cov-report=html:htmlcov \
            --cov-branch

      - name: Publish test results
        uses: dorny/test-reporter@v1
        if: always()
        with:
          name: Integration Tests
          path: metaflow-fastdata/junit.xml
          reporter: java-junit

      - name: Post coverage summary
        if: always()
        working-directory: metaflow-fastdata
        run: |
          echo "## Coverage — Integration Tests" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          python -m coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
          echo '```' >> $GITHUB_STEP_SUMMARY

      - name: Upload test report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: integration-test-report
          path: |
            metaflow-fastdata/report.html
            metaflow-fastdata/junit.xml
            metaflow-fastdata/coverage.xml
            metaflow-fastdata/htmlcov/
          include-hidden-files: true

      - name: Stop services
        if: always()
        working-directory: metaflow-fastdata
        run: docker compose down -v