Added custom function support for kafka and delta tables #370
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| pull_request: | |
| types: [ opened, synchronize ] | |
| push: | |
| branches: [ main ] | |
| jobs: | |
| test-and-results: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| python-version: ['3.9'] | |
| os: [ ubuntu-latest ] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: Set up Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '11' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' # caching pip dependencies | |
| cache-dependency-path: setup.py | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install package and dependencies | |
| run: pip install -U -e ".[dev]" | |
| - name: Install coverage | |
| run: pip install coverage | |
| - name: Install psutil | |
| run: pip install psutil | |
| - name: Lint | |
| run: flake8 | |
| - name: Set environment variables | |
| run: | | |
| echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV | |
| echo "SPARK_EXECUTOR_MEMORY=8g" >> $GITHUB_ENV | |
| echo "SPARK_DRIVER_MEMORY=8g" >> $GITHUB_ENV | |
| echo "JAVA_OPTS=-Xmx10g -XX:+UseG1GC" >> $GITHUB_ENV | |
| - name: Print System Information | |
| run: | | |
| python -c "import psutil; import os; | |
| print(f'Physical Memory: {psutil.virtual_memory().total / 1e9:.2f} GB'); print(f'CPU Cores: {os.cpu_count()}')" | |
| - name: Run Unit Tests | |
| run: python -m coverage run -m pytest tests/ -v | |
| - name: Publish test coverage | |
| if: startsWith(matrix.os,'ubuntu') | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| env_vars: OS,PYTHON | |
| fail_ci_if_error: true | |
| flags: unittests | |
| name: codecov-umbrella | |
| path_to_write_report: ./coverage/codecov_report.txt | |
| verbose: true | |