Skip to content

Commit a3c8829

Browse files
filipeomfilipeom
committed
Artifact fixes
1 parent 2f430ed commit a3c8829

File tree

6 files changed

+30
-16
lines changed

6 files changed

+30
-16
lines changed

‎bench/README.md‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ solver usage, with notable improvements for batched solver interactions.
1010

1111
## Getting the Artifact
1212

13-
You can download the artifact through [Zenodo](https://zenodo.org/records/13959671/files/smtml_0.3.tar.gz?download=1).
13+
You can download the artifact through [Zenodo](https://zenodo.org/records/14104822).
1414

1515
## Tested platforms
1616

@@ -29,7 +29,7 @@ modification, and distribution.
2929

3030
## MD5 Hash of the Artifact
3131

32-
8edce799971395168bdf728c9a136399
32+
46c6e43adf9bd5d9a922b3b7db307d12
3333

3434
## Building and Running the Docker Image
3535

@@ -191,10 +191,14 @@ $ git submodule update --remote -- smt-comp/
191191
```
192192

193193
2. Run the `eq1.sh` script. Note that, this script can take up multiple
194-
hours to complete.
194+
hours to complete. Therefore, to avoid running the entire dataset specify
195+
a number of benchmark to run with the `-n` flag. This will run the benchmarks
196+
quicker, but only provide an aproximation of the papers results.
195197

196198
```sh
197199
$ ./eq1.sh
200+
# Or, to just run 20 benchmarks to finish quickly
201+
$ ./eq1.sh -n 20
198202
```
199203

200204
The script will generate the results in directory `eq1`, which
@@ -237,10 +241,14 @@ $ git submodule update --remote -- smt-comp/
237241
```
238242

239243
2. Run the `eq2.sh` script. Note that, this script can take up multiple
240-
hours to complete.
244+
hours to complete. Therefore, to avoid running the entire dataset specify
245+
a number of benchmark to run with the `-n` flag. This will run the benchmarks
246+
quicker, but only provide an aproximation of the papers results.
241247

242248
```sh
243249
$ ./eq2.sh
250+
# Or, to just run 20 benchmarks to finish quickly
251+
$ ./eq2.sh -n 20
244252
```
245253

246254
The script will generate the following files:
@@ -315,10 +323,14 @@ $ git submodule update --remote -- smt-testcomp23/
315323
```
316324

317325
2. Run the `testcomp.sh` script. Note that, this script can take up multiple
318-
hours to complete.
326+
hours to complete. Therefore, to avoid running the entire dataset specify
327+
a number of benchmark to run with the `-n` flag. This will run the benchmarks
328+
quicker, but only provide an aproximation of the papers results.
319329

320330
```sh
321331
$ ./testcomp.sh
332+
# Or, to just run 20 benchmarks to finish quickly
333+
$ ./testcomp.sh -n 20
322334
```
323335

324336
The script will generate the following files:

‎bench/eq1.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/sh
1+
#!/bin/bash
22

33
TEMPLATE=./tmp.XXXXXXXXXX
44

‎bench/eq2.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ python3 run_benchmarks.py --single --dir $QF_LIA_DIR --output-dir csvs_single --
100100
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_z3 --prover smtml-z3
101101
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_bitwuzla --prover smtml-bitwuzla
102102
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_z3_solver --prover z3
103-
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_bitwuzla --prover smtml-bitwuzla
103+
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_bitwuzla_solver --prover bitwuzla
104104
#### QF_S ####
105105
python3 run_benchmarks.py --single --dir $QF_S_DIR --output-dir csvs_single --output-filename QF_S_z3 --prover smtml-z3
106106
python3 run_benchmarks.py --single --dir $QF_S_DIR --output-dir csvs_single --output-filename QF_S_z3_solver --prover z3

‎bench/eq3.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ fi
127127

128128
$dry_value && exit 0
129129

130+
mkdir csvs_multi
131+
130132
opam sw z3-bitwuzla
131133
eval $(opam env)
132134

@@ -147,7 +149,7 @@ else
147149
python3 run_benchmarks.py --single --dir $QF_FP_DIR --output-dir csvs_single --output-filename QF_FP_bitwuzla_solver --prover bitwuzla
148150
fi
149151

150-
python3 run_benchmarks.py --multi -F $QF_FP_LIST --output-dir csvs_multi --output-filename QF_FP_bitwuzla_solver --prover bitwuzla
152+
python3 run_benchmarks.py --multi -F $QF_FP_LIST --output-dir csvs_multi --output-filename QF_FP_bitwuzla --prover smtml-bitwuzla
151153

152154
#### QF_LIA ####
153155
## Z3 ##
@@ -176,7 +178,7 @@ else
176178
python3 run_benchmarks.py --single --dir $QF_BV_DIR --output-dir csvs_single --output-filename QF_BV_bitwuzla_solver --prover bitwuzla
177179
fi
178180

179-
python3 run_benchmarks.py --multi -F $QF_BV_LIST --output-dir csvs_multi --output-filename QF_BV_bitwuzla_solver --prover bitwuzla
181+
python3 run_benchmarks.py --multi -F $QF_BV_LIST --output-dir csvs_multi --output-filename QF_BV_bitwuzla --prover smtml-bitwuzla
180182

181183
#### QF_S ####
182184
## Z3 ##

‎bench/run_benchmarks.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def get_latest_sqlite_file(benchpress_data_dir):
3232
files = [f for f in os.listdir(benchpress_data_dir) if f.endswith('.sqlite')]
3333
if not files:
3434
raise FileNotFoundError("No .sqlite files found in benchpress data directory.")
35-
35+
3636
latest_file = max(files, key=lambda x: os.path.getmtime(os.path.join(benchpress_data_dir, x)))
3737
return os.path.join(benchpress_data_dir, latest_file)
3838

@@ -65,13 +65,13 @@ def automate_benchpress_single(directory_to_benchmark, output_csv_dir, output_cs
6565
print_error(f"Error during SQLite to CSV process: {e}")
6666

6767
def get_latest_n_csvs(n):
68-
result = subprocess.run(['ls', '-d', os.path.join('/home/smtml/smtml/bench', 'res-multi-query-*')],
68+
result = subprocess.run(['ls', '-d', os.path.join('/home/smtml/smtml/bench', 'res-multi-query-*')],
6969
stdout=subprocess.PIPE, text=True, shell=True)
7070
query_dirs = result.stdout.splitlines()
7171
query_dirs = query_dirs[-n:]
7272
csv_files = []
7373
for query_dir in query_dirs:
74-
csv_file = glob(os.path.join(query_dir, '*.csv'))
74+
csv_file = glob.glob(os.path.join(query_dir, '*.csv'))
7575
if csv_file:
7676
csv_files.append(csv_file[0])
7777
return csv_files
@@ -135,4 +135,4 @@ def automate_multi_query(list_file, output_csv_dir, output_csv_name, prover):
135135
elif args.multi:
136136
automate_multi_query(args.F, args.output_dir, args.output_filename, args.prover)
137137
else:
138-
print_error("Please specify --single or --multi.")
138+
print_error("Please specify --single or --multi.")

‎bench/runner/multi_query.ml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,11 @@ let main _hook provers from_file dirs =
7575
let results =
7676
List.map
7777
(fun p ->
78-
let ((status, stdout, _stderr, _) as result) =
78+
let ((status, stdout, stderr, _) as result) =
7979
Tool.fork_and_run ?from_file p dirs
8080
in
81-
Fmt.pr "@[<v 2>Run %a@;Exited: %a@;Stdout:@; %a@]@." Tool.pp_prover p
82-
pp_exit_status status Fmt.string (String.escaped stdout);
81+
Fmt.pr "@[<v 2>Run %a@;Exited: %a@;Stdout:@; %a@;Stderr:@; %a@]@." Tool.pp_prover p
82+
pp_exit_status status Fmt.string (String.escaped stdout) Fmt.string (String.escaped stderr);
8383
(p, result) )
8484
provers
8585
in

0 commit comments

Comments
 (0)