datafusion-distributed/.github/workflows/ci.yml at 1927b0384704a1d07bf23c4d5aaf9b452161a6bc · paradedb/datafusion-distributed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: CI

on:
  pull_request:
  push:
    branches:
      - main
      - 'target-patch-*'

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  TPCDS_SCALE_FACTOR: 0.5 # 0.5 scale factor produces a data dir which is 124MB

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo build

  clippy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
        with:
          components: clippy
      - run: cargo clippy --all-targets --all-features -- -D warnings

  unit-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true
      - uses: ./.github/actions/setup
      - run: cargo test --features integration

  # Runs the whole lib unit suite with `grpc` off, the proof that the abstraction builds and runs
  # without gRPC. `lfs: true` because some planner unit tests load parquet testdata.
  unit-test-no-grpc:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true
      - uses: ./.github/actions/setup
      - run: cargo test --no-default-features --lib

  tpch-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_correctness_test
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  tpch-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_plans_test

  tpcds-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        shard: [ "01", "02", "03", "04", "05", "06", "07", "08", "09", "10" ]
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: benchmarks/testdata/tpcds/main.zip
          key: "main.zip"
      - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }}
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  tpcds-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: benchmarks/testdata/tpcds/main.zip
          key: "main.zip"
      - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_plans_test

  clickbench-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: benchmarks/testdata/clickbench/
          key: "data"
      - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_correctness_test
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  clickbench-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: benchmarks/testdata/clickbench/
          key: "data"
      - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_plans_test

  format-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
        with:
          components: rustfmt
      - run: cargo fmt --all -- --check