Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions _report/report.R
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ get_report_status_file = function(path=getwd()) {
file.path(path, "report-done")
}
get_report_solutions = function() {
c("collapse", "data.table", "dplyr", "pandas", "pydatatable", "spark", "dask", "juliadf", "juliads", "clickhouse", "cudf", "polars", "duckdb", "datafusion", "arrow", "R-arrow")
c("duckdb-latest", "collapse", "data.table", "dplyr", "pandas", "pydatatable", "spark", "dask", "juliadf", "juliads", "clickhouse", "cudf", "polars", "duckdb", "datafusion", "arrow", "R-arrow")
}
get_data_levels = function() {
## groupby
Expand Down Expand Up @@ -273,11 +273,9 @@ time_logs = function(path=getwd()) {
lt <- load_time(path=getwd())

ct = clean_time(lt)

# ct = ct %>% filter(machine_type != 'c6id.metal')

# remove duckdb-latest for now
ct = ct %>% filter(!(solution == 'duckdb-latest'))
ct = ct %>% filter(!(solution == 'duckdb-latest'))
d = model_time(ct)
ll <- load_logs(path=path)

Expand Down
2 changes: 1 addition & 1 deletion duckdb-latest/VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.99.9000
1.3.1
134 changes: 72 additions & 62 deletions duckdb-latest/groupby-duckdb-latest.R

Large diffs are not rendered by default.

150 changes: 65 additions & 85 deletions duckdb-latest/join-duckdb-latest.R

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion duckdb-latest/setup-duckdb-latest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@ Rscript -e 'install.packages("DBI", lib="./duckdb-latest/r-duckdb-latest", repos
cd duckdb-latest
rm -rf duckdb-r
git clone https://github.com/duckdb/duckdb-r.git
cd duckdb-r
git checkout next
cd ..
ncores=`python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'`
MAKE="make -j$ncores" R CMD INSTALL -l "./r-duckdb-latest" duckdb-r
MAKEFLAGS="-j$ncores" R CMD INSTALL -l "./r-duckdb-latest" duckdb-r
rm -rf duckdb-r
cd ..

Expand Down
2 changes: 2 additions & 0 deletions duckdb/groupby-duckdb.R
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ if (machine_type == 'c6id.4xlarge' && on_disk) {
dbExecute(con, "pragma memory_limit='25G'")
}

dbExecute(con, "SET enable_progress_bar = false;")

ncores = parallel::detectCores()
invisible(dbExecute(con, sprintf("PRAGMA THREADS=%d", ncores)))
git = dbGetQuery(con, "SELECT source_id FROM pragma_version()")[[1L]]
Expand Down
4 changes: 4 additions & 0 deletions duckdb/join-duckdb.R
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ invisible(dbExecute(con, "DROP TABLE IF EXISTS ans"))

dbDisconnect(con, shutdown=TRUE)

if (on_disk) {
unlink(db_file)
}

cat(sprintf("joining finished, took %.0fs\n", proc.time()[["elapsed"]]-task_init))

if( !interactive() ) q("no", status=0)