datafusion-table-providers/.github/workflows/pr.yaml at f42b1f0692e7279da09ea812af548158ceea0f1c · datafusion-contrib/datafusion-table-providers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
---
name: pr

on:
  pull_request:
    branches:
      - main
      - spiceai
      # TODO: remove this once we have fully tested the python integration tests
      - python-integration-test

jobs:
  lint:
    name: Clippy
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - run: cargo clippy --all-features -- -D warnings

  build:
    name: Build
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      # Putting this into a GitHub Actions matrix will run a separate job per matrix item, whereas in theory
      # this can re-use the existing build cache to go faster.
      - name: Build without default features
        run: cargo check --no-default-features

      - name: Build with only duckdb
        run: cargo check --no-default-features --features duckdb

      - name: Build with only postgres
        run: cargo check --no-default-features --features postgres

      - name: Build with only sqlite
        run: cargo check --no-default-features --features sqlite

      - name: Build with only mysql
        run: cargo check --no-default-features --features mysql

      - name: Build with only flight
        run: cargo check --no-default-features --features flight

  integration-test:
    name: Tests
    runs-on: ubuntu-latest

    env:
      PG_DOCKER_IMAGE: ghcr.io/cloudnative-pg/postgresql:16-bookworm
      MYSQL_DOCKER_IMAGE: public.ecr.aws/ubuntu/mysql:8.0-22.04_beta

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - name: Pull the Postgres/MySQL images
        run: |
          docker pull ${{ env.PG_DOCKER_IMAGE }}
          docker pull ${{ env.MYSQL_DOCKER_IMAGE }}

      - name: Free Disk Space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo rm -rf \
            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
            /usr/lib/jvm || true
          echo "some directories deleted"
          sudo apt install aptitude -y >/dev/null 2>&1
          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
            google-cloud-sdk imagemagick \
            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
            mercurial apt-transport-https mono-complete libmysqlclient \
            yarn chrpath libssl-dev libxft-dev \
            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
            -y -f >/dev/null 2>&1
          sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1
          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
          sudo apt-get autoremove -y >/dev/null 2>&1
          sudo apt-get autoclean -y >/dev/null 2>&1
          echo "some packages purged"
          df -h

      - name: Install ODBC & Sqlite
        run: |
          sudo apt-get install -y unixodbc-dev
          sudo apt-get install -y libsqlite3-dev

      - name: Run tests
        run: make test

  python-integration-test:
    name: Python Tests
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: astral-sh/setup-uv@v5
        with:
            enable-cache: true

      - name: Install ODBC & Sqlite
        run: |
          sudo apt-get install -y unixodbc-dev
          sudo apt-get install -y libsqlite3-dev

      - name: Build Python package
        run: |
          cargo install --locked --git https://github.com/roapi/roapi --branch main --bins roapi
          sudo apt-get install -y sqliteodbc
          cd python
          uv sync --dev --no-install-package datafusion
          uv run --no-project maturin develop --uv


      - name: Run Python tests
        run: |
          cd python/python/tests
          uv run --no-project pytest -v .