vortex/.github/workflows/nightly-bench.yml at b9c87f19696317e3cfa006f829e42cd60c4a216d · spiceai/vortex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
name: Nightly Benchmarks

on:
  schedule:
    - cron: "0 0 * * *"  # daily
  workflow_dispatch:

permissions:
  id-token: write  # enables AWS-GitHub OIDC
  actions: read
  contents: write
  deployments: write

jobs:
  sql:
    uses: ./.github/workflows/sql-benchmarks.yml
    secrets: inherit
    with:
      mode: "develop"
      machine_type: ${{ matrix.machine_type.instance_name }}
      # datafusion:vortex uses a lot of memory
      benchmark_matrix: |
        [
          {
            "id": "clickbench-nvme",
            "subcommand": "clickbench",
            "name": "Clickbench on NVME",
            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
            "build_args": "--features lance"
          },
          {
            "id": "tpch-nvme",
            "subcommand": "tpch",
            "name": "TPC-H on NVME",
            "targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
            "scale_factor": "--scale-factor 10.0",
            "build_args": "--features lance"
          },
          {
            "id": "tpch-s3",
            "subcommand": "tpch",
            "name": "TPC-H on S3",
            "local_dir": "bench-vortex/data/tpch/10.0",
            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
            "targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex",
            "scale_factor": "--scale-factor 10.0",
            "build_args": "--features lance"
          },
          {
            "id": "tpch-nvme",
            "subcommand": "tpch",
            "name": "TPC-H on NVME",
            "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
            "scale_factor": "--scale-factor 100"
          },
          {
            "id": "tpch-s3",
            "subcommand": "tpch",
            "name": "TPC-H on S3",
            "local_dir": "bench-vortex/data/tpch/100.0",
            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
            "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
            "scale_factor": "--scale-factor 100.0"
          },
        ]
    strategy:
      # A single run not should kill the others
      fail-fast: false
      matrix:
        machine_type:
          - id: x86
            instance_name: c6id.8xlarge
# TODO(joe): support other arch
#          - id: arm64
#            instance_name: c6gd.8xlarge