datafusion-table-providers/.github/workflows/pr.yaml at 09e0a59f1c5c801e63b2ce6ad129cfd6e04a4e43 · datafusion-contrib/datafusion-table-providers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
---
name: pr

on:
  pull_request:
    branches:
      - main
      - spiceai

jobs:
  lint:
    name: Clippy
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - uses: Swatinem/rust-cache@v2
        with:
          key: clippy

      - run: cargo clippy --all-features -- -D warnings

  build:
    name: Build
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        with:
          key: build

      # Putting this into a GitHub Actions matrix will run a separate job per matrix item, whereas in theory
      # this can re-use the existing build cache to go faster.
      - name: Build without default features
        run: cargo check --no-default-features

      - name: Build with only duckdb
        run: cargo check --no-default-features --features duckdb

      - name: Build with only postgres
        run: cargo check --no-default-features --features postgres

      - name: Build with only sqlite
        run: cargo check --no-default-features --features sqlite

      - name: Build with only mysql
        run: cargo check --no-default-features --features mysql

      - name: Build with only mongodb
        run: cargo check --no-default-features --features mongodb

  integration-test:
    name: Tests postgres,postgres-federation,sqlite,sqlite-federation,mysql
    runs-on: ubuntu-latest

    env:
      PG_DOCKER_IMAGE: ghcr.io/cloudnative-pg/postgresql:16-bookworm
      MYSQL_DOCKER_IMAGE: public.ecr.aws/ubuntu/mysql:8.0-22.04_beta
      MONGODB_DOCKER_IMAGE: public.ecr.aws/docker/library/mongo:7

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        with:
          key: test

      - name: Pull the Postgres/MySQL images
        run: |
          docker pull ${{ env.PG_DOCKER_IMAGE }}
          docker pull ${{ env.MYSQL_DOCKER_IMAGE }}
          docker pull ${{ env.MONGODB_DOCKER_IMAGE }}

      - name: Free Disk Space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo rm -rf \
            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
            /usr/lib/jvm || true
          echo "some directories deleted"
          sudo apt install aptitude -y >/dev/null 2>&1
          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
            google-cloud-sdk imagemagick \
            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
            mercurial apt-transport-https mono-complete libmysqlclient \
            yarn chrpath libssl-dev libxft-dev \
            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
            -y -f >/dev/null 2>&1
          sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1
          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
          sudo apt-get autoremove -y >/dev/null 2>&1
          sudo apt-get autoclean -y >/dev/null 2>&1
          echo "some packages purged"
          df -h

      - name: Install ODBC & Sqlite
        run: |
          sudo apt-get install -y unixodbc-dev
          sudo apt-get install -y libsqlite3-dev

      - name: Run tests
        run: cargo test --features postgres,postgres-federation,sqlite,sqlite-federation,mysql

  integration-test-2:
    name: Tests duckdb,duckdb-federation,flight,mongodb
    runs-on: ubuntu-latest

    env:
      PG_DOCKER_IMAGE: ghcr.io/cloudnative-pg/postgresql:16-bookworm
      MYSQL_DOCKER_IMAGE: public.ecr.aws/ubuntu/mysql:8.0-22.04_beta
      MONGODB_DOCKER_IMAGE: public.ecr.aws/docker/library/mongo:7

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        with:
          key: test

      - name: Pull the Postgres/MySQL images
        run: |
          docker pull ${{ env.PG_DOCKER_IMAGE }}
          docker pull ${{ env.MYSQL_DOCKER_IMAGE }}
          docker pull ${{ env.MONGODB_DOCKER_IMAGE }}

      - name: Free Disk Space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo rm -rf \
            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
            /usr/lib/jvm || true
          echo "some directories deleted"
          sudo apt install aptitude -y >/dev/null 2>&1
          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
            google-cloud-sdk imagemagick \
            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
            mercurial apt-transport-https mono-complete libmysqlclient \
            yarn chrpath libssl-dev libxft-dev \
            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
            -y -f >/dev/null 2>&1
          sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1
          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
          sudo apt-get autoremove -y >/dev/null 2>&1
          sudo apt-get autoclean -y >/dev/null 2>&1
          echo "some packages purged"
          df -h

      - name: Install ODBC & Sqlite
        run: |
          sudo apt-get install -y unixodbc-dev
          sudo apt-get install -y libsqlite3-dev

      - name: Run tests
        run: cargo test --features duckdb,duckdb-federation,flight,mongodb