Add metaflow-functions OSS package

.coveragerc

@@ -0,0 +1,9 @@
+[paths]
+source =
+    .
+    /home/runner/work/metaflow-nflx-extensions/metaflow-nflx-extensions/
+    /Users/runner/work/metaflow-nflx-extensions/metaflow-nflx-extensions/
+
+[report]
+omit =
+    */toplevel/nflxext_version.py

.github/workflows/python-publish.yml

@@ -1,10 +1,7 @@
-# This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
+# Publish packages to PyPI when a release is created.
+#
+# Builds all packages and publishes them with skip-existing, so only
+# packages with new versions (not yet on PyPI) are actually uploaded.
 
 name: Publish
 
@@ -17,25 +14,41 @@ permissions:
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
-    # Specifying a GitHub environment is optional, but strongly encouraged
     environment: pypi
     permissions:
-      # IMPORTANT: this permission is mandatory for Trusted Publishing
       id-token: write
+
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
-    - name: Install dependencies
+
+    - name: Install build dependencies
       run: |
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
-    - name: Build package
+
+    - name: Build packages
       run: |
-        python setup.py sdist bdist_wheel --universal
-    - name: Publish package
+        for pkg_dir in metaflow-netflixext metaflow-functions; do
+          echo "Building $pkg_dir..."
+          cd "$pkg_dir"
+          python setup.py sdist bdist_wheel --universal
+          cd ..
+        done
+
+    - name: Publish metaflow-netflixext
       uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # 1.13.0
+      with:
+        packages-dir: metaflow-netflixext/dist/
+        skip-existing: true
+
+    - name: Publish metaflow-functions
+      uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # 1.13.0
+      with:
+        packages-dir: metaflow-functions/dist/
+        skip-existing: true

.github/workflows/test-functions.yml

@@ -0,0 +1,177 @@
+name: test-functions
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    env:
+      AWS_ENDPOINT_URL: http://localhost:9000
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+      AWS_DEFAULT_REGION: us-east-1
+      METAFLOW_DEFAULT_DATASTORE: s3
+      METAFLOW_DATASTORE_SYSROOT_S3: s3://metaflow-test
+      METAFLOW_DEFAULT_FUNCTIONS_S3_ROOT: s3://metaflow-test/functions
+      METAFLOW_FUNCTION_RUNTIME_PATH: /tmp
+      METAFLOW_CONDA_DEPENDENCY_RESOLVER: micromamba
+      METAFLOW_FEAT_ALWAYS_UPLOAD_CODE_PACKAGE: true
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Start MinIO
+        run: |
+          docker run -d \
+            -p 9000:9000 \
+            -e MINIO_ROOT_USER=minioadmin \
+            -e MINIO_ROOT_PASSWORD=minioadmin \
+            minio/minio server /data
+          until curl -sf http://localhost:9000/minio/health/live; do sleep 1; done
+
+      - uses: mamba-org/setup-micromamba@b09ef9b599704322748535812ca03efb2625677b # v2.0.5
+        with:
+          environment-name: nflxext-dev
+          condarc: |
+            channels:
+              - conda-forge
+            channel_alias: https://prefix.dev
+          create-args: python=${{ matrix.python-version }}
+
+      - name: Install metaflow-functions + test deps
+        shell: bash -eo pipefail -l {0}
+        run: |
+          pip install metaflow
+          pip install -e metaflow-netflixext/
+          pip install -e metaflow-functions/
+          pip install pytest pytest-xdist pytest-cov fastavro psutil pandas pyarrow pytest-mock
+
+      - name: Create minio bucket
+        shell: bash -eo pipefail -l {0}
+        run: |
+          python -c "
+          import boto3, botocore
+          s3 = boto3.client(
+              's3',
+              endpoint_url='http://localhost:9000',
+              aws_access_key_id='minioadmin',
+              aws_secret_access_key='minioadmin',
+              region_name='us-east-1',
+              config=botocore.config.Config(signature_version='s3v4'),
+          )
+          s3.create_bucket(Bucket='metaflow-test')
+          print('Bucket metaflow-test created')
+          "
+
+      - name: Run unit tests
+        shell: bash -eo pipefail -l {0}
+        run: |
+          pytest tests/functions/ -v \
+            --junit-xml=junit.xml \
+            --cov=metaflow-functions/metaflow_extensions \
+            --cov-report=term-missing \
+            --cov-report=xml:coverage.xml \
+            --cov-report=html:htmlcov \
+            --cov-branch
+
+      - name: Publish test results
+        uses: dorny/test-reporter@v1
+        if: always()
+        with:
+          name: Functions Unit Tests (py${{ matrix.python-version }})
+          path: junit.xml
+          reporter: java-junit
+
+      - name: Post coverage summary
+        if: always()
+        run: |
+          echo "## Coverage — Functions Unit Tests (py${{ matrix.python-version }})" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          python -m coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: functions-unit-test-report-py${{ matrix.python-version }}
+          path: |
+            junit.xml
+            coverage.xml
+            htmlcov/
+            .coverage
+          include-hidden-files: true
+
+  coverage-report:
+    name: "Coverage Report"
+    needs: unit
+    if: always()
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        with:
+          python-version: "3.11"
+
+      - name: Install coverage
+        run: pip install coverage[toml]
+
+      - name: Download coverage data from all jobs
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "functions-unit-test-report-*"
+          path: coverage-artifacts/
+
+      - name: Combine coverage data
+        run: |
+          for dir in coverage-artifacts/*/; do
+            label=$(basename "$dir" | sed 's/^functions-unit-test-report-//')
+            src="$dir/.coverage"
+            if [ -f "$src" ]; then
+              cp "$src" ".coverage.$label"
+              echo "Copied $src → .coverage.$label"
+            fi
+          done
+          coverage combine
+          echo "Combined data sources:"
+          coverage debug data
+
+      - name: Generate combined report
+        run: |
+          coverage xml -o combined-coverage.xml
+          coverage html -d combined-htmlcov/ --title="metaflow-functions Coverage"
+
+      - name: Post coverage summary
+        run: |
+          echo "## Coverage — metaflow-functions (combined)" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          total=$(coverage report | tail -1 | awk '{print $NF}')
+          echo "**Total coverage: $total**" >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: combined-coverage.xml
+          name: functions-combined
+
+      - name: Upload combined coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-combined
+          path: |
+            combined-htmlcov/
+            combined-coverage.xml

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
       METAFLOW_CONDA_DEPENDENCY_RESOLVER: ${{ matrix.resolver }}
       METAFLOW_CONDA_TEST: 1
       METAFLOW_DATASTORE_SYSROOT_LOCAL: .metaflow
-      METAFLOW_HACK_CONDA_CHANNEL_ALIAS: prefix.dev
+      METAFLOW_CONDA_HACK_CHANNEL_ALIAS: prefix.dev
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
@@ -53,15 +53,20 @@ jobs:
           condarc: |
             channels:
               - conda-forge
-            channel_alias: https://prefix.dev
           create-args: >-
             python=${{ matrix.python-version }}
             ${{ matrix.mamba-version }}
+
+      - name: Configure channel alias for tests
+        shell: bash -eo pipefail -l {0}
+        run: |
+          echo "channel_alias: https://prefix.dev" >> ~/.condarc
+
       - name: install nflx-extension
         shell: bash -eo pipefail -l {0}
         run: |
           which pip
-          pip install -e . --force-reinstall -U
+          pip install -e metaflow-netflixext/
 
       - name: install bash
         if: runner.os == 'macOS'
@@ -73,4 +78,103 @@ jobs:
           set -x
           which pytest
           mkdir .metaflow
-          pytest -n 4 tests
+          pytest tests/environments tests/plugins/conda -v \
+            --junit-xml=junit.xml \
+            --cov=metaflow_extensions \
+            --cov-report=term-missing \
+            --cov-report=xml:coverage.xml \
+            --cov-report=html:htmlcov \
+            --cov-branch
+
+      - name: Publish test results
+        uses: dorny/test-reporter@v1
+        if: always()
+        with:
+          name: Extensions Unit Tests (py${{ matrix.python-version }} resolver ${{ matrix.resolver }} micromamba ${{ matrix.micromamba-version }} mamba ${{ matrix.mamba-version }} on ${{ matrix.os }})
+          path: junit.xml
+          reporter: java-junit
+
+      - name: Post coverage summary
+        if: always()
+        run: |
+          echo "## Coverage — Extensions Unit Tests (py${{ matrix.python-version }} resolver ${{ matrix.resolver }} micromamba ${{ matrix.micromamba-version }} mamba ${{ matrix.mamba-version }} on ${{ matrix.os }})" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          python -m coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: extensions-unit-test-report-py${{ matrix.python-version }}-re${{ matrix.resolver }}-umamba${{ matrix.micromamba-version }}-mamba${{ matrix.mamba-version }}-os${{ matrix.os }}
+          path: |
+            junit.xml
+            coverage.xml
+            htmlcov/
+            .coverage
+          include-hidden-files: true
+
+  coverage-report-ext:
+    name: "Coverage Report"
+    needs: test
+    if: always()
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        with:
+          python-version: "3.11"
+
+      - name: Install coverage
+        run: pip install coverage[toml]
+
+      - name: Download coverage data from all jobs
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "extensions-unit-test-report-*"
+          path: coverage-artifacts/
+
+      - name: Combine coverage data
+        run: |
+          for dir in coverage-artifacts/*/; do
+            label=$(basename "$dir" | sed 's/^extensions-unit-test-report-//')
+            src="$dir/.coverage"
+            if [ -f "$src" ]; then
+              cp "$src" ".coverage.$label"
+              echo "Copied $src → .coverage.$label"
+            fi
+          done
+          coverage combine
+          echo "Combined data sources:"
+          coverage debug data
+
+      - name: Generate combined report
+        run: |
+          coverage xml -o combined-coverage.xml
+          coverage html -d combined-htmlcov/ --title="metaflow-extensions Coverage"
+
+      - name: Post coverage summary
+        run: |
+          echo "## Coverage — metaflow-extensions (combined)" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          coverage report --sort=miss >> $GITHUB_STEP_SUMMARY || true
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          total=$(coverage report | tail -1 | awk '{print $NF}')
+          echo "**Total coverage: $total**" >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: combined-coverage.xml
+          name: extensions-combined
+
+      - name: Upload combined coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-combined
+          path: |
+            combined-htmlcov/
+            combined-coverage.xml

.gitignore

@@ -1 +1,5 @@
 **/__pycache__
+**/*.egg-info
+**/dist
+**/build
+metaflow-functions/metaflow_extensions/nflx/toplevel/functions_version.py