Skip to content

Commit b53cd00

Browse files
authored
Add hypertable on/off oracle for LLM fuzzer (#10295)
Detects mismatch of query results on hypertable vs plain postgres table.
1 parent 49cac59 commit b53cd00

3 files changed

Lines changed: 165 additions & 15 deletions

File tree

.github/workflows/llm-fuzzer.yaml

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,16 @@ jobs:
7979
display_name: "Internal Program Errors"
8080
- oracle: "optimizations"
8181
display_name: "Optimizations ON-OFF"
82+
- oracle: "hypertable"
83+
display_name: "Hypertable ON-OFF"
8284

8385
steps:
8486
- name: Install Linux Dependencies
8587
timeout-minutes: 15
8688
run: |
8789
sudo apt-get update
88-
sudo apt-get install ccache cmake flex bison systemd-coredump gdb \
89-
jq postgresql-client ${{ env.extra_packages }}
90+
sudo apt-get install ccache cmake flex bison icu-devtools systemd-coredump \
91+
gdb jq postgresql-client ${{ env.extra_packages }}
9092
9193
- name: Checkout TimescaleDB
9294
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -314,6 +316,14 @@ jobs:
314316
printf '\\restrict %s\n' "${RANDOM}" > restricted-repro.sql
315317
cat ~/llm-fuzzer-repro.sql >> restricted-repro.sql
316318
319+
PSQLRC=oracle-psqlrc
320+
export PSQLRC
321+
echo "
322+
\set QUIET on
323+
\set ON_ERROR_STOP on
324+
set client_min_messages = error;
325+
" > "${PSQLRC}"
326+
317327
.github/workflows/llm-fuzzer/oracle/${{ matrix.oracle }}/verify.sh restricted-repro.sql &> repro_result.txt
318328
319329
psql -c "select 1;"
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#!/usr/bin/env bash
2+
3+
# This oracle runs the reproducer script with the psql variable :hyper set
4+
# to false and true, and compares the output. The script passes the variable
5+
# into CREATE TABLE ... WITH (tsdb.hypertable = :hyper, ...), so the same
6+
# statements run against a plain Postgres table and a TimescaleDB
7+
# hypertable. The admissible bug must lead to a change in script output
8+
# between the two.
9+
#
10+
# An admissible repro script:
11+
#
12+
# Must be runnable on same database multiple times in sequence.
13+
#
14+
# Must run exactly the same statements no matter if it runs against the
15+
# plain table or the hypertable.
16+
#
17+
#
18+
# Must not use the psql meta-commands.
19+
#
20+
# Must not require superuser privileges.
21+
#
22+
# The output of an admissible repro script:
23+
#
24+
# Must be sufficiently ordered to prevent false positives (i.e. ORDER BY, no
25+
# ties).
26+
#
27+
# Must not depend on floating point precision or numeric stability.
28+
#
29+
# Must be independent from arbitrary environmental influence like the OID values
30+
# or chunk identifiers.
31+
#
32+
# Must not change when the script runs on the same database multiple times in
33+
# sequence.
34+
set -eu
35+
36+
psql <<<'alter database :"DBNAME" set client_min_messages to error'
37+
38+
if ! psql -v hyper=false -f "$1" > result_plain.txt
39+
then
40+
echo "Repro errors out, not admissible"
41+
exit 0
42+
fi
43+
44+
if psql -v hyper=maybe -f "$1" &> result_probe.txt
45+
then
46+
echo "Repro does not use :hyper for tsdb.hypertable, not admissible"
47+
exit 0
48+
fi
49+
50+
if ! psql -v hyper=off -f "$1" > result_plain_synonym.txt
51+
then
52+
echo "Repro errors out, not admissible"
53+
exit 0
54+
fi
55+
56+
57+
if ! psql -v hyper=true -f "$1" > result_hyper.txt
58+
then
59+
echo "Repro errors out, not admissible"
60+
exit 0
61+
fi
62+
63+
if ! psql -v hyper=true -c "set enable_seqscan to off;" -f "$1" > result_hyper_noseq.txt
64+
then
65+
echo "Repro errors out, not admissible"
66+
exit 0
67+
fi
68+
69+
if ! psql -v hyper=true -c "set enable_indexscan to off;" -f "$1" > result_hyper_noindex.txt
70+
then
71+
echo "Repro errors out, not admissible"
72+
exit 0
73+
fi
74+
75+
if ! psql -v hyper=true -c "set enable_hashagg to off;" -f "$1" > result_hyper_nohashagg.txt
76+
then
77+
echo "Repro errors out, not admissible"
78+
exit 0
79+
fi
80+
81+
if ! psql -v hyper=true -c "
82+
set max_parallel_workers_per_gather = 8;
83+
set parallel_setup_cost = 0;
84+
set parallel_tuple_cost = 0;
85+
set min_parallel_table_scan_size = 0;
86+
set min_parallel_index_scan_size = 0;
87+
" -f "$1" > result_hyper_para.txt
88+
then
89+
echo "Repro errors out, not admissible"
90+
exit 0
91+
fi
92+
93+
if ! psql -v hyper=true -c "set work_mem = '4GB'" -f "$1" > result_hyper_mem.txt
94+
then
95+
echo "Repro errors out, not admissible"
96+
exit 0
97+
fi
98+
99+
if ! psql -v hyper=true -c "
100+
set enable_hashagg to off;
101+
set enable_sort to off;
102+
set timescaledb.enable_chunkwise_aggregation to off;
103+
" -f "$1" > result_hyper_rowsort.txt 2> result_hyper_rowsort.err
104+
then
105+
echo "Repro errors out, not admissible"
106+
exit 0
107+
fi
108+
109+
if ! diff -u result_plain.txt result_plain_synonym.txt \
110+
|| ! diff -u result_hyper.txt result_hyper_noseq.txt \
111+
|| ! diff -u result_hyper.txt result_hyper_noindex.txt \
112+
|| ! diff -u result_hyper.txt result_hyper_nohashagg.txt \
113+
|| ! diff -u result_hyper.txt result_hyper_para.txt \
114+
|| ! diff -u result_hyper.txt result_hyper_mem.txt \
115+
|| ! diff -u result_hyper.txt result_hyper_rowsort.txt
116+
then
117+
echo "Repro gives different results between runs, not admissible"
118+
exit 0
119+
fi
120+
121+
echo
122+
echo '```diff'
123+
echo
124+
125+
if diff -u result_plain.txt result_hyper.txt
126+
then
127+
result=0
128+
else
129+
result=$?
130+
fi
131+
132+
echo
133+
echo '```'
134+
echo
135+
136+
if [ "${result}" -eq 0 ]
137+
then
138+
echo "Same result with hypertable false/true, error not reproduced"
139+
exit 0
140+
fi
141+
142+
echo "Reproduced"
143+
exit 1

.github/workflows/llm-fuzzer/oracle/optimizations/verify.sh

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,37 +29,34 @@
2929
# sequence.
3030
set -eu
3131

32-
PGOPTIONS='-c client_min_messages=error'
33-
export PGOPTIONS
32+
psql <<<'alter database :"DBNAME" set client_min_messages to error'
33+
psql <<<'alter database :"DBNAME" set timescaledb.enable_optimizations to off'
3434

35-
psql -q <<<'alter database :"DBNAME" set client_min_messages to error'
36-
psql -q <<<'alter database :"DBNAME" set timescaledb.enable_optimizations to off'
37-
38-
if ! psql -q -f "$1" > result_noopt.txt
35+
if ! psql -f "$1" > result_noopt.txt
3936
then
4037
echo "Repro errors out, not admissible"
4138
exit 0
4239
fi
4340

44-
if ! psql -q -c "set enable_seqscan to off;" -f "$1" > result_noopt_noseq.txt
41+
if ! psql -c "set enable_seqscan to off;" -f "$1" > result_noopt_noseq.txt
4542
then
4643
echo "Repro errors out, not admissible"
4744
exit 0
4845
fi
4946

50-
if ! psql -q -c "set enable_indexscan to off;" -f "$1" > result_noopt_noindex.txt
47+
if ! psql -c "set enable_indexscan to off;" -f "$1" > result_noopt_noindex.txt
5148
then
5249
echo "Repro errors out, not admissible"
5350
exit 0
5451
fi
5552

56-
if ! psql -q -c "set enable_hashagg to off;" -f "$1" > result_noopt_nohashagg.txt
53+
if ! psql -c "set enable_hashagg to off;" -f "$1" > result_noopt_nohashagg.txt
5754
then
5855
echo "Repro errors out, not admissible"
5956
exit 0
6057
fi
6158

62-
if ! psql -q -c "
59+
if ! psql -c "
6360
set max_parallel_workers_per_gather = 8;
6461
set parallel_setup_cost = 0;
6562
set parallel_tuple_cost = 0;
@@ -71,7 +68,7 @@ then
7168
exit 0
7269
fi
7370

74-
if ! psql -q -c "set work_mem = '4GB'" -f "$1" > result_noopt_mem.txt
71+
if ! psql -c "set work_mem = '4GB'" -f "$1" > result_noopt_mem.txt
7572
then
7673
echo "Repro errors out, not admissible"
7774
exit 0
@@ -87,9 +84,9 @@ then
8784
exit 0
8885
fi
8986

90-
psql -q <<<'alter database :"DBNAME" set timescaledb.enable_optimizations to on'
87+
psql <<<'alter database :"DBNAME" set timescaledb.enable_optimizations to on'
9188

92-
if ! psql -q -f "$1" > result_opt.txt
89+
if ! psql -f "$1" > result_opt.txt
9390
then
9491
echo "Repro errors out, not admissible"
9592
exit 0

0 commit comments

Comments
 (0)