fix pagination (#244)

Co-authored-by: Zac Davies <zachary.davies+data@databricks.com>

Author: zacdav-db

NEWS.md

@@ -4,6 +4,7 @@
 -   Fixed Unity Catalog volume file requests so `db_volume_*` paths containing spaces are encoded correctly (#231)
 -   `db_cluster_events()` now forwards the `event_types` argument to the API, which was previously ignored
 -   `db_sql_warehouse_create()` and `db_sql_warehouse_edit()` now forward the `tags` argument to the API, which was previously ignored
+-   Unity Catalog list helpers now preserve `next_page_token` metadata in list responses for catalogs, schemas, tables, and volumes; their list arguments are sent as documented query parameters
 -   `db_uc_volumes_list()` now forwards the `max_results`, `include_browse`, and `page_token` arguments to the API, which were previously ignored
 -   `db_vs_indexes_query()` now sends the `score_threshold` argument to the API, which was previously ignored
 -   Added `show_progress` to `dbConnect()` for the DBI backend; `dbGetQuery()`, `dbFetch()`, `dbWriteTable()`, and dbplyr `collect()` now use the connection default while preserving per-call `show_progress` overrides (#223)

R/connection-pane.R

@@ -33,7 +33,7 @@ readable_time <- function(x) {
 }
 
 get_catalogs <- function(host, token) {
-  catalogs <- db_uc_catalogs_list(host = host, token = token)
+  catalogs <- db_uc_catalogs_list(host = host, token = token)$catalogs
   if (length(catalogs) > 0) {
     data.frame(
       name = purrr::map_chr(catalogs, "name"),
@@ -50,7 +50,7 @@ get_schemas <- function(catalog, host, token) {
     catalog = catalog,
     host = host,
     token = token
-  )
+  )$schemas
   if (length(schemas) > 0) {
     data.frame(
       name = purrr::map_chr(schemas, "name"),
@@ -68,7 +68,7 @@ get_tables <- function(catalog, schema, host, token) {
     schema = schema,
     host = host,
     token = token
-  )
+  )$tables
   if (length(tables) > 0) {
     data.frame(
       name = purrr::map_chr(tables, "name"),
@@ -238,7 +238,7 @@ get_uc_volumes <- function(catalog, schema, host, token) {
     schema = schema,
     host = host,
     token = token
-  )
+  )$volumes
   if (length(volumes) > 0) {
     data.frame(
       name = purrr::map_chr(volumes, "name"),
@@ -256,7 +256,7 @@ get_uc_volume <- function(catalog, schema, host, volume, token) {
     schema = schema,
     host = host,
     token = token
-  )
+  )$volumes
 
   volume <- purrr::keep(volumes, \(x) x$name == volume)[[1]]
 
@@ -394,7 +394,7 @@ get_table_data <- function(catalog, schema, table, host, token, metadata = TRUE)
 }
 
 get_experiments <- function(host, token) {
-  experiments <- db_experiments_list(host = host, token = token)
+  experiments <- db_experiments_list(host = host, token = token)$experiments
   exp_names <- purrr::map_chr(experiments, "name")
   exp_ids <-  purrr::map_chr(experiments, "experiment_id")
   data.frame(

R/experiments.R

@@ -22,7 +22,7 @@ db_experiments_list <- function(view_type = c("ACTIVE_ONLY", "DELETED_ONLY", "AL
   )
 
   if (perform_request) {
-    db_perform_request(req)$experiments
+    db_perform_request(req)
   } else {
     req
   }
@@ -63,4 +63,3 @@ db_experiments_get <- function(name = NULL, id = NULL,
     req
   }
 }
-

R/uc-catalogs.R

@@ -9,7 +9,8 @@
 #'
 #' @family Unity Catalog Management
 #'
-#' @returns List
+#' @returns Full API response list, including `next_page_token` when present,
+#' or an `httr2_request` when `perform_request = FALSE`.
 #' @export
 db_uc_catalogs_list <- function(max_results = 1000,
                                 include_browse = TRUE,
@@ -19,23 +20,21 @@ db_uc_catalogs_list <- function(max_results = 1000,
 
   stopifnot(max_results <= 1000)
 
-  body <- list(
-    max_results = max_results,
-    include_browse = include_browse,
-    page_token = page_token
-  )
-
   req <- db_request(
     endpoint = "unity-catalog/catalogs",
     method = "GET",
     version = "2.1",
     host = host,
-    token = token,
-    body = body
-  )
+    token = token
+  ) |>
+    httr2::req_url_query(
+      max_results = max_results,
+      include_browse = from_logical(include_browse),
+      page_token = page_token
+    )
 
   if (perform_request) {
-    db_perform_request(req)$catalogs
+    db_perform_request(req)
   } else {
     req
   }

R/uc-schemas.R

@@ -8,7 +8,8 @@
 #'
 #' @family Unity Catalog Management
 #'
-#' @returns List
+#' @returns Full API response list, including `next_page_token` when present,
+#' or an `httr2_request` when `perform_request = FALSE`.
 #' @export
 db_uc_schemas_list <- function(catalog,
                                max_results = 1000,
@@ -18,23 +19,21 @@ db_uc_schemas_list <- function(catalog,
 
   stopifnot(max_results <= 1000)
 
-  body <- list(
-    max_results = max_results,
-    page_token = page_token
-  )
-
   req <- db_request(
     endpoint = "unity-catalog/schemas",
     method = "GET",
     version = "2.1",
     host = host,
-    token = token,
-    body = body
+    token = token
   ) |>
-    httr2::req_url_query(catalog_name = catalog)
+    httr2::req_url_query(
+      catalog_name = catalog,
+      max_results = max_results,
+      page_token = page_token
+    )
 
   if (perform_request) {
-    db_perform_request(req)$schemas
+    db_perform_request(req)
   } else {
     req
   }
@@ -78,4 +77,3 @@ db_uc_schemas_get <- function(catalog, schema,
     req
   }
 }
-

R/uc-tables.R

@@ -21,7 +21,8 @@
 #'
 #' @family Unity Catalog Table Management
 #'
-#' @returns List
+#' @returns Full API response list, including `next_page_token` when present,
+#' or an `httr2_request` when `perform_request = FALSE`.
 #' @export
 db_uc_tables_list <- function(catalog, schema, max_results = 50,
                               omit_columns = TRUE,
@@ -47,6 +48,7 @@ db_uc_tables_list <- function(catalog, schema, max_results = 50,
       catalog_name = catalog,
       schema_name = schema,
       include_delta_metadata = from_logical(include_delta_metadata),
+      max_results = max_results,
       omit_columns = from_logical(omit_columns),
       omit_properties = from_logical(omit_properties),
       omit_username = from_logical(omit_username),
@@ -56,7 +58,7 @@ db_uc_tables_list <- function(catalog, schema, max_results = 50,
     )
 
   if (perform_request) {
-    db_perform_request(req)$tables
+    db_perform_request(req)
   } else {
     req
   }
@@ -228,4 +230,3 @@ db_uc_tables_summaries <- function(catalog,
     req
   }
 }
-

R/uc-volumes.R

@@ -11,7 +11,8 @@
 #'
 #' @family Unity Catalog Volume Management
 #'
-#' @returns List
+#' @returns Full API response list, including `next_page_token` when present,
+#' or an `httr2_request` when `perform_request = FALSE`.
 #' @export
 db_uc_volumes_list <- function(catalog, schema,
                                max_results = 10000,
@@ -36,7 +37,7 @@ db_uc_volumes_list <- function(catalog, schema,
     )
 
   if (perform_request) {
-    db_perform_request(req)$volumes
+    db_perform_request(req)
   } else {
     req
   }
@@ -214,4 +215,3 @@ db_uc_volumes_create <- function(catalog, schema, volume,
 }
 
 
-