From 20016ec41d78d8b0f9343dad6b55bc9b062a73ec Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 12:40:49 +0000 Subject: [PATCH 01/20] chore: sync Rust client with Apify OpenAPI spec v2-2026-07-10T105921Z Bumps API_SPEC_VERSION to v2-2026-07-10T105921Z and crate version to 0.5.1. The spec delta (added 401/402 error responses and relaxed field nullability/optionality) requires no code change: errors are handled generically and response models are forward-compatible. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 8 ++++++++ Cargo.toml | 2 +- src/version.rs | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cad9cc..ff5a723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to the Rust Apify API client are documented here. The format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.5.1] - 2026-07-10 + +### Changed +- Bumped `API_SPEC_VERSION` to `v2-2026-07-10T105921Z`. The spec delta (added `401`/`402` + error responses and relaxed field nullability/optionality) needs no code change: error + responses are handled generically and response models are forward-compatible. +- Bumped crate version to `0.5.1`. + ## [0.5.0] - 2026-07-10 ### Added diff --git a/Cargo.toml b/Cargo.toml index a7712a9..6bec260 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apify-client" -version = "0.5.0" +version = "0.5.1" authors = ["Apify Technologies "] description = "An official, but experimental, AI-generated and AI-maintained Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" diff --git a/src/version.rs b/src/version.rs index 2bbb258..38205ca 100644 --- a/src/version.rs +++ b/src/version.rs @@ -10,4 +10,4 @@ pub const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// and verified against. /// /// This corresponds to the `info.version` field of the Apify OpenAPI document. -pub const API_SPEC_VERSION: &str = "v2-2026-07-08T143931Z"; +pub const API_SPEC_VERSION: &str = "v2-2026-07-10T105921Z"; From 2c5e045b2f0aac78d9beb3f18ebe34cf8e3d3398 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 13:30:11 +0000 Subject: [PATCH 02/20] feat: add lazy pagination iterators to all collection clients Adds a shared generic ListIterator (src/clients/pagination.rs, exported at the crate root) and an iterate() method on every collection client the reference JS client iterates: actors, actor versions, env vars, builds, runs, datasets, key-value stores, request queues, schedules, tasks, webhooks, webhook dispatches, plus DatasetClient::iterate_items(). Store iteration is refactored onto the shared iterator (StoreActorIterator kept as a type alias). Termination is short-page based, robust to the dataset-items endpoint reporting total=0. Adds one item-iteration integration test per collection and corrects the src/models.rs module doc. Minor version bump to 0.6.0. Addresses review items in notes.md (iteration helpers, iteration tests, module-doc accuracy). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 15 ++- Cargo.toml | 2 +- docs/README.md | 29 +++++- src/clients/actor_collection.rs | 19 ++++ src/clients/actor_env_var_collection.rs | 17 ++++ src/clients/actor_version_collection.rs | 16 +++ src/clients/build_collection.rs | 16 +++ src/clients/dataset.rs | 23 +++++ src/clients/dataset_collection.rs | 16 +++ src/clients/key_value_store_collection.rs | 16 +++ src/clients/mod.rs | 1 + src/clients/pagination.rs | 112 +++++++++++++++++++++ src/clients/request_queue_collection.rs | 16 +++ src/clients/run_collection.rs | 20 ++++ src/clients/schedule_collection.rs | 16 +++ src/clients/store_collection.rs | 81 ++++----------- src/clients/task_collection.rs | 16 +++ src/clients/webhook_collection.rs | 16 +++ src/clients/webhook_dispatch_collection.rs | 16 +++ src/lib.rs | 1 + src/models.rs | 7 +- tests/actor.rs | 96 ++++++++++++++++++ tests/actor_run.rs | 27 +++++ tests/build.rs | 58 +++++++++++ tests/common/mod.rs | 26 +++++ tests/dataset.rs | 71 +++++++++++++ tests/key_value_store.rs | 31 ++++++ tests/request_queue.rs | 31 ++++++ tests/schedule.rs | 29 ++++++ tests/task.rs | 29 ++++++ tests/webhook.rs | 66 ++++++++++++ 31 files changed, 868 insertions(+), 67 deletions(-) create mode 100644 src/clients/pagination.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ff5a723..3e6016b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,24 @@ All notable changes to the Rust Apify API client are documented here. The format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). -## [0.5.1] - 2026-07-10 +## [0.6.0] - 2026-07-10 + +### Added +- Lazy async pagination iterators on every collection client, via a shared generic + `ListIterator` (exported at the crate root). New `iterate()` methods on the actor, actor + version, environment-variable, build, run, dataset, key-value-store, request-queue, schedule, + task, webhook, and webhook-dispatch collection clients, plus `DatasetClient::iterate_items()` + for dataset items. Each yields one item at a time, fetching pages on demand — the idiomatic + counterpart to the reference client's async-iterable list results. ### Changed - Bumped `API_SPEC_VERSION` to `v2-2026-07-10T105921Z`. The spec delta (added `401`/`402` error responses and relaxed field nullability/optionality) needs no code change: error responses are handled generically and response models are forward-compatible. -- Bumped crate version to `0.5.1`. +- `StoreCollectionClient::iterate` now uses the shared `ListIterator`; `StoreActorIterator` is + a type alias for `ListIterator` (existing usage is unaffected). +- Corrected the `src/models.rs` module doc to describe forward-compatibility accurately. +- Bumped crate version to `0.6.0`. ## [0.5.0] - 2026-07-10 diff --git a/Cargo.toml b/Cargo.toml index 6bec260..f4fa1fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apify-client" -version = "0.5.1" +version = "0.6.0" authors = ["Apify Technologies "] description = "An official, but experimental, AI-generated and AI-maintained Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" diff --git a/docs/README.md b/docs/README.md index ad7dcc5..b73e001 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,7 +35,7 @@ Add the crate and an async runtime: ```toml [dependencies] -apify-client = "0.5" +apify-client = "0.6" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` @@ -180,6 +180,33 @@ lists/creates, and the single-resource accessor (singular) operates on one resou Each resource has a dedicated page, linked under **Resource clients** in the [Contents](#contents) above (Actors, runs, builds, tasks, storages, schedules, webhooks, and store/users/logs). +### Iterating collections + +A collection's `list(...)` method returns a single `PaginationList` page. To walk every item +across all pages without tracking offsets yourself, call `iterate(...)` instead: it returns a +lazy `ListIterator` (re-exported at the crate root) that fetches the next page from the API on +demand as you consume items. Every collection client provides it (`actors`, `builds`, `runs`, +`tasks`, `datasets`, `key_value_stores`, `request_queues`, `schedules`, `webhooks`, +`webhook_dispatches`, `store`, and the nested Actor `versions`/`env_vars`), and `DatasetClient` +exposes `iterate_items()` for dataset items. Any per-call `limit` is used as the page size. + +```rust,no_run +use apify_client::{ApifyClient, ActorListOptions}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let client = ApifyClient::new("my-api-token"); + let mut actors = client.actors().iterate(ActorListOptions { + my: Some(true), + ..Default::default() + }); + while let Some(actor) = actors.next().await? { + println!("{}", actor.id); + } + Ok(()) +} +``` + ## Error handling Every fallible method returns `Result`. The variants are: diff --git a/src/clients/actor_collection.rs b/src/clients/actor_collection.rs index 5634fb6..34a1742 100644 --- a/src/clients/actor_collection.rs +++ b/src/clients/actor_collection.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -53,6 +54,24 @@ impl ActorCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all Actors matching `options`, fetching pages on demand. + /// + /// Returns a [`ListIterator`] whose `next()` yields one Actor at a time, transparently + /// fetching subsequent pages until the listing is exhausted. + pub fn iterate(&self, options: ActorListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Creates a new Actor with the given definition. /// /// `actor` is any JSON-serializable Actor definition (at minimum a `name`). diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 9c16a5b..4d3f494 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -1,6 +1,7 @@ //! Client for an Actor version's environment variable collection. use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -24,6 +25,22 @@ impl ActorEnvVarCollectionClient { list_resource(&self.ctx, None, &QueryParams::new()).await } + /// Lazily iterates over the Actor version's environment variables. + /// + /// The env-var listing is not offset-paginated (the API returns every variable in a single + /// page), so this yields all variables from that one page and then completes. It exists for + /// interface parity with the other collection clients and the reference client. + pub fn iterate(&self) -> ListIterator { + let client = self.clone(); + ListIterator::new( + 0, + Box::new(move |_offset| { + let client = client.clone(); + Box::pin(async move { client.list().await }) + }), + ) + } + /// Creates a new environment variable. pub async fn create(&self, env_var: &ActorEnvVar) -> ApifyClientResult { create_resource(&self.ctx, &QueryParams::new(), env_var).await diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index 4b8ef08..1a8685b 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -34,6 +35,21 @@ impl ActorVersionCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all versions matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Creates a new Actor version. pub async fn create(&self, version: &T) -> ApifyClientResult { create_resource(&self.ctx, &QueryParams::new(), version).await diff --git a/src/clients/build_collection.rs b/src/clients/build_collection.rs index bd0c898..1eaf307 100644 --- a/src/clients/build_collection.rs +++ b/src/clients/build_collection.rs @@ -1,6 +1,7 @@ //! Client for an Actor-build collection (`/v2/actor-builds`, `/v2/actors/{id}/builds`). use crate::clients::base::{list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -35,4 +36,19 @@ impl BuildCollectionClient { .add_bool("desc", options.desc); list_resource(&self.ctx, None, ¶ms).await } + + /// Lazily iterates over all builds matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } } diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index a4b2ee7..d460b11 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -5,6 +5,7 @@ use serde::Serialize; use serde_json::Value; use crate::clients::base::{delete_resource, get_resource, update_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{parse_data_envelope, sign_storage_content, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; @@ -232,6 +233,28 @@ impl DatasetClient { }) } + /// Lazily iterates over all items in the dataset, fetching pages on demand. + /// + /// The idiomatic-Rust counterpart of the reference client's async-iterable + /// `listItems`/`iterateItems`: yields one deserialized item of type `T` at a time, + /// transparently paging with the caller's `options` (its `limit` acts as the page size). + pub fn iterate_items( + &self, + options: DatasetListItemsOptions, + ) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list_items::(options).await }) + }), + ) + } + /// Downloads dataset items serialized in the given `format`, returning the raw bytes. /// /// Unlike [`list_items`](Self::list_items), which returns parsed items, this returns the diff --git a/src/clients/dataset_collection.rs b/src/clients/dataset_collection.rs index 8b39d86..9c7c39b 100644 --- a/src/clients/dataset_collection.rs +++ b/src/clients/dataset_collection.rs @@ -1,6 +1,7 @@ //! Client for the dataset collection (`/v2/datasets`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -29,6 +30,21 @@ impl DatasetCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all datasets matching `options`, fetching pages on demand. + pub fn iterate(&self, options: StorageListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Gets the dataset with the given `name`, creating it if it does not exist. /// /// Passing `None` for `name` creates an unnamed dataset. diff --git a/src/clients/key_value_store_collection.rs b/src/clients/key_value_store_collection.rs index ec8cd1a..cd18a64 100644 --- a/src/clients/key_value_store_collection.rs +++ b/src/clients/key_value_store_collection.rs @@ -1,6 +1,7 @@ //! Client for the key-value store collection (`/v2/key-value-stores`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -30,6 +31,21 @@ impl KeyValueStoreCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all key-value stores matching `options`, fetching pages on demand. + pub fn iterate(&self, options: StorageListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Gets the store with the given `name`, creating it if it does not exist. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { get_or_create_named(&self.ctx, name).await diff --git a/src/clients/mod.rs b/src/clients/mod.rs index 1486be1..91308a1 100644 --- a/src/clients/mod.rs +++ b/src/clients/mod.rs @@ -19,6 +19,7 @@ pub mod dataset_collection; pub mod key_value_store; pub mod key_value_store_collection; pub mod log; +pub mod pagination; pub mod request_queue; pub mod request_queue_collection; pub mod run; diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs new file mode 100644 index 0000000..5f619ab --- /dev/null +++ b/src/clients/pagination.rs @@ -0,0 +1,112 @@ +//! Generic lazy pagination shared by every collection client. +//! +//! The reference JavaScript client returns an `AsyncIterable` from every collection `list()` +//! (via its base `_listPaginated`), so callers can iterate across all pages without manually +//! tracking offsets. Rust cannot return a value that is simultaneously a `Future` and a +//! `Stream`, so the idiomatic equivalent here is a dedicated `iterate()` method on each +//! collection client that returns a [`ListIterator`]. This module implements the paging logic +//! once (the DRY principle the requirements call for) so every client stays a thin wrapper. + +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; + +use crate::common::PaginationList; +use crate::error::ApifyClientResult; + +/// A boxed future yielding one page of results. Boxed so [`ListIterator`] can hold a fetcher +/// for any concrete collection client without being generic over its (unnameable) future type. +type PageFuture = Pin>> + Send>>; + +/// Fetches the page starting at the given absolute `offset`. Implementations capture a clone of +/// the collection client and the caller's list options, overriding only the offset per page. +type PageFetcher = Box PageFuture + Send + Sync>; + +/// A lazy, page-fetching async iterator over an offset/limit-paginated list endpoint. +/// +/// Created by a collection client's `iterate()` method. Each call to [`next`](Self::next) +/// returns the next item, transparently fetching the following page from the API once the +/// local buffer drains, until every item across all pages has been yielded. The caller's +/// per-page `limit` (if any) is honoured as the page size; iteration always walks the full +/// result set regardless. +/// +/// # Example +/// ```no_run +/// use apify_client::ApifyClient; +/// +/// # async fn run() -> Result<(), Box> { +/// let client = ApifyClient::new("my-api-token"); +/// let mut it = client.actors().iterate(Default::default()); +/// while let Some(actor) = it.next().await? { +/// println!("{}", actor.id); +/// } +/// # Ok(()) +/// # } +/// ``` +pub struct ListIterator { + fetch: PageFetcher, + buffer: VecDeque, + /// Absolute offset of the next page to request. + next_offset: i64, + /// Set once the listing has been fully consumed. + exhausted: bool, +} + +impl ListIterator { + /// Builds an iterator that starts at `start_offset` and fetches pages via `fetch`. + pub(crate) fn new(start_offset: i64, fetch: PageFetcher) -> Self { + Self { + fetch, + buffer: VecDeque::new(), + next_offset: start_offset, + exhausted: false, + } + } + + /// Returns the next item, or `None` when the listing is exhausted. Fetches another page from + /// the API when the local buffer is empty. + pub async fn next(&mut self) -> ApifyClientResult> { + if let Some(item) = self.buffer.pop_front() { + return Ok(Some(item)); + } + if self.exhausted { + return Ok(None); + } + + let page = (self.fetch)(self.next_offset).await?; + let received = page.items.len() as i64; + if received == 0 { + self.exhausted = true; + return Ok(None); + } + self.next_offset += received; + + // Decide whether more pages remain. Primary signal is a "short" page: the API returns + // fewer items than the effective page size it reports (`page.limit`), which only happens + // on the final page. This is robust even where `total` is unreliable — the dataset-items + // endpoint, for instance, reports `total = 0`. A non-positive `limit` means the endpoint + // is not offset-paginated (it returned everything at once), so stop after this page to + // avoid refetching it forever. `total`, when the endpoint reports it (> 0), is used only + // as an early stop so a full final page does not cost one extra empty request. + let effective_limit = page.limit; + let reached_total = page.total > 0 && self.next_offset >= page.total; + if effective_limit <= 0 || received < effective_limit || reached_total { + self.exhausted = true; + } + + self.buffer.extend(page.items); + Ok(self.buffer.pop_front()) + } + + /// Eagerly drains the iterator into a single `Vec`, fetching every remaining page. + /// + /// Convenience for callers that want all items at once; prefer [`next`](Self::next) to + /// process items as they stream in without buffering the whole result set. + pub async fn collect_all(mut self) -> ApifyClientResult> { + let mut out = Vec::new(); + while let Some(item) = self.next().await? { + out.push(item); + } + Ok(out) + } +} diff --git a/src/clients/request_queue_collection.rs b/src/clients/request_queue_collection.rs index 32145f8..bbfadd0 100644 --- a/src/clients/request_queue_collection.rs +++ b/src/clients/request_queue_collection.rs @@ -1,6 +1,7 @@ //! Client for the request queue collection (`/v2/request-queues`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -30,6 +31,21 @@ impl RequestQueueCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all request queues matching `options`, fetching pages on demand. + pub fn iterate(&self, options: StorageListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Gets the queue with the given `name`, creating it if it does not exist. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { get_or_create_named(&self.ctx, name).await diff --git a/src/clients/run_collection.rs b/src/clients/run_collection.rs index c52283b..ac5ec38 100644 --- a/src/clients/run_collection.rs +++ b/src/clients/run_collection.rs @@ -1,6 +1,7 @@ //! Client for an Actor-run collection (`/v2/actor-runs`, `/v2/actors/{id}/runs`, etc.). use crate::clients::base::{list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -77,4 +78,23 @@ impl RunCollectionClient { .add_str("startedBefore", filter.started_before); list_resource(&self.ctx, None, ¶ms).await } + + /// Lazily iterates over all runs matching `options`/`filter`, fetching pages on demand. + /// + /// The idiomatic-Rust counterpart of the reference client's async-iterable run listing; + /// yields one [`ActorRun`] at a time across all pages. + pub fn iterate(&self, options: ListOptions, filter: RunListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + let filter = filter.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options, filter).await }) + }), + ) + } } diff --git a/src/clients/schedule_collection.rs b/src/clients/schedule_collection.rs index 1cc671c..83ef4e5 100644 --- a/src/clients/schedule_collection.rs +++ b/src/clients/schedule_collection.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -31,6 +32,21 @@ impl ScheduleCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all schedules matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Creates a new schedule from the given definition. pub async fn create(&self, schedule: &T) -> ApifyClientResult { create_resource(&self.ctx, &QueryParams::new(), schedule).await diff --git a/src/clients/store_collection.rs b/src/clients/store_collection.rs index b43c7ad..2654aac 100644 --- a/src/clients/store_collection.rs +++ b/src/clients/store_collection.rs @@ -1,11 +1,19 @@ //! Client for browsing the Apify Store (`/v2/store`). use crate::clients::base::{list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorStoreListItem; +/// A lazy, page-fetching iterator over Apify Store Actors. +/// +/// Returned by [`StoreCollectionClient::iterate`]. This is an alias for the shared +/// [`ListIterator`]; call its `next()` to yield one Actor at a time, fetching further pages +/// from the API transparently until the listing is exhausted. +pub type StoreActorIterator = ListIterator; + /// Options for searching the Apify Store. #[derive(Debug, Default, Clone)] pub struct StoreListOptions { @@ -58,17 +66,20 @@ impl StoreCollectionClient { /// Lazily iterates all Store Actors matching `options`, fetching pages on demand. /// - /// Returns a [`StoreActorIterator`] whose [`next`](StoreActorIterator::next) method - /// yields one Actor at a time, transparently fetching subsequent pages. + /// Returns a [`StoreActorIterator`] whose `next()` method yields one Actor at a time, + /// transparently fetching subsequent pages. pub fn iterate(&self, options: StoreListOptions) -> StoreActorIterator { - StoreActorIterator { - client: self.clone(), - options, - buffer: std::collections::VecDeque::new(), - next_offset: 0, - total: None, - exhausted: false, - } + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) } fn build_params(&self, options: &StoreListOptions) -> QueryParams { @@ -87,53 +98,3 @@ impl StoreCollectionClient { params } } - -/// A lazy, page-fetching iterator over Apify Store Actors. -/// -/// Created by [`StoreCollectionClient::iterate`]. Each call to [`next`](Self::next) -/// returns the next Actor, fetching another page from the API when the local buffer is -/// exhausted, until all matching Actors have been yielded. -pub struct StoreActorIterator { - client: StoreCollectionClient, - options: StoreListOptions, - buffer: std::collections::VecDeque, - next_offset: i64, - total: Option, - exhausted: bool, -} - -impl StoreActorIterator { - /// Returns the next Store Actor, or `None` when the listing is exhausted. - pub async fn next(&mut self) -> ApifyClientResult> { - if let Some(item) = self.buffer.pop_front() { - return Ok(Some(item)); - } - if self.exhausted { - return Ok(None); - } - - // Honour a caller-provided starting offset on the first fetch. - let start_offset = self.options.offset.unwrap_or(0) + self.next_offset; - let mut page_options = self.options.clone(); - page_options.offset = Some(start_offset); - - let page = self.client.list(page_options).await?; - if self.total.is_none() { - self.total = Some(page.total); - } - if page.items.is_empty() { - self.exhausted = true; - return Ok(None); - } - - self.next_offset += page.items.len() as i64; - // Stop once we have walked past the total number of available items. - if let Some(total) = self.total { - if start_offset + page.items.len() as i64 >= total { - self.exhausted = true; - } - } - self.buffer.extend(page.items); - Ok(self.buffer.pop_front()) - } -} diff --git a/src/clients/task_collection.rs b/src/clients/task_collection.rs index b277a72..86b8e81 100644 --- a/src/clients/task_collection.rs +++ b/src/clients/task_collection.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -31,6 +32,21 @@ impl TaskCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all tasks matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Creates a new task from the given definition. pub async fn create(&self, task: &T) -> ApifyClientResult { create_resource(&self.ctx, &QueryParams::new(), task).await diff --git a/src/clients/webhook_collection.rs b/src/clients/webhook_collection.rs index 3e9b7be..4a2fc16 100644 --- a/src/clients/webhook_collection.rs +++ b/src/clients/webhook_collection.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -38,6 +39,21 @@ impl WebhookCollectionClient { list_resource(&self.ctx, None, ¶ms).await } + /// Lazily iterates over all webhooks matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } + /// Creates a new webhook from the given definition. pub async fn create(&self, webhook: &T) -> ApifyClientResult { create_resource(&self.ctx, &QueryParams::new(), webhook).await diff --git a/src/clients/webhook_dispatch_collection.rs b/src/clients/webhook_dispatch_collection.rs index 2508a91..578341a 100644 --- a/src/clients/webhook_dispatch_collection.rs +++ b/src/clients/webhook_dispatch_collection.rs @@ -1,6 +1,7 @@ //! Client for the webhook dispatch collection (`/v2/webhook-dispatches`). use crate::clients::base::{list_resource, ResourceContext}; +use crate::clients::pagination::ListIterator; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -38,4 +39,19 @@ impl WebhookDispatchCollectionClient { .add_bool("desc", options.desc); list_resource(&self.ctx, None, ¶ms).await } + + /// Lazily iterates over all webhook dispatches matching `options`, fetching pages on demand. + pub fn iterate(&self, options: ListOptions) -> ListIterator { + let client = self.clone(); + let start = options.offset.unwrap_or(0); + ListIterator::new( + start, + Box::new(move |offset| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + Box::pin(async move { client.list(options).await }) + }), + ) + } } diff --git a/src/lib.rs b/src/lib.rs index 247d32b..1ea0d24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,7 @@ pub use clients::actor_collection::ActorListOptions; pub use clients::dataset::{DatasetDownloadOptions, DatasetListItemsOptions, DownloadItemsFormat}; pub use clients::key_value_store::{GetRecordOptions, GetRecordsOptions, ListKeysOptions}; pub use clients::log::LogOptions; +pub use clients::pagination::ListIterator; pub use clients::request_queue::ListRequestsOptions; pub use clients::run::{ LastRunOptions, RunChargeOptions, RunMetamorphOptions, RunResurrectOptions, diff --git a/src/models.rs b/src/models.rs index c6fc9a0..679cad2 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,9 +1,10 @@ //! Data models for Apify API resources. //! //! Each resource is modelled with the fields most commonly used by clients, mirroring -//! the reference JavaScript client. To remain forward-compatible with additive changes -//! to the API, every model captures any unknown fields in an `extra` map via -//! `#[serde(flatten)]`, so new API fields never break deserialization. +//! the reference JavaScript client. To remain forward-compatible with additive changes to +//! the API, none of the models set `deny_unknown_fields`, so unknown API fields are ignored +//! rather than breaking deserialization. Most resource models additionally capture any unknown +//! fields in an `extra` map via `#[serde(flatten)]` so they remain accessible to callers. use std::collections::HashMap; diff --git a/tests/actor.rs b/tests/actor.rs index 773e712..1e88e58 100644 --- a/tests/actor.rs +++ b/tests/actor.rs @@ -74,6 +74,102 @@ async fn get_actor() { assert_eq!(fetched.id, actor.id); } +/// Iteration: the Actor collection iterator yields a just-created Actor across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_actors() { + let client = require_client!(); + let name = actor_name("actor-iter"); + let actor = client + .actors() + .create(&actor_definition(&name)) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&id).delete().await; + }); + + // Restrict to the caller's own Actors, newest-first, with a small page size. + let iter = client.actors().iterate(apify_client::ActorListOptions { + my: Some(true), + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = actor.id.clone(); + assert!( + common::iter_contains(iter, move |a| a.id == target).await, + "actor iteration should yield the created actor" + ); +} + +/// Iteration: the Actor version iterator yields the Actor's initial version. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_actor_versions() { + let client = require_client!(); + let name = actor_name("ver-iter"); + let actor = client + .actors() + .create(&actor_definition(&name)) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&id).delete().await; + }); + + let iter = client + .actor(&actor.id) + .versions() + .iterate(Default::default()); + assert!( + common::iter_contains(iter, |v| v.version_number == "0.0").await, + "version iteration should yield the initial 0.0 version" + ); +} + +/// Iteration: the environment-variable iterator yields a variable we just created. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_actor_env_vars() { + use apify_client::models::ActorEnvVar; + + let client = require_client!(); + let name = actor_name("envit"); + let actor = client + .actors() + .create(&actor_definition(&name)) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&id).delete().await; + }); + + let version_client = client.actor(&actor.id).version("0.0"); + version_client + .env_vars() + .create(&ActorEnvVar { + name: "ITER_VAR".to_string(), + value: Some("v".to_string()), + is_secret: Some(false), + extra: Default::default(), + }) + .await + .expect("create env var"); + + let iter = version_client.env_vars().iterate(); + assert!( + common::iter_contains(iter, |e| e.name == "ITER_VAR").await, + "env-var iteration should yield the created variable" + ); +} + /// Complex flow: create an Actor with a single version, get it, update it, list builds, /// and delete it. #[tokio::test(flavor = "multi_thread")] diff --git a/tests/actor_run.rs b/tests/actor_run.rs index c5ccc35..27da6bf 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -83,6 +83,33 @@ async fn run_actor_and_read_outputs() { ); } +/// Iteration: the run collection iterator yields a run we just started across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_runs() { + let client = require_client!(); + // Ensure at least one run exists on the account by calling the public hello-world Actor. + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + + // Newest-first with a small page size so the just-finished run is near the front. + let iter = client.runs().iterate( + apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }, + Default::default(), + ); + let target = run.id.clone(); + assert!( + common::iter_contains(iter, move |r| r.id == target).await, + "run iteration should yield the started run" + ); +} + /// Convenience: access the Actor's last run. #[tokio::test(flavor = "multi_thread")] async fn last_run_access() { diff --git a/tests/build.rs b/tests/build.rs index a1c61d7..5a06206 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -16,6 +16,64 @@ async fn list_builds() { assert!(page.total >= 0); } +/// Iteration: the build collection iterator yields a build we just started. +/// +/// Scoped to a fresh Actor's builds so the collection is small and deterministic. The build is +/// started but not awaited — it appears in the listing immediately regardless of its state. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_builds() { + let client = require_client!(); + let name = common::unique_name("build-iter").replace('-', ""); + let name = format!("b{}", &name[..name.len().min(20)]); + + let definition = json!({ + "name": name, + "isPublic": false, + "versions": [{ + "versionNumber": "0.0", + "sourceType": "SOURCE_FILES", + "buildTag": "latest", + "sourceFiles": [ + { + "name": "Dockerfile", + "format": "TEXT", + "content": "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" + }, + { "name": "main.js", "format": "TEXT", "content": "console.log('iter');" } + ] + }] + }); + + let actor = client + .actors() + .create(&definition) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let cleanup_id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&cleanup_id).delete().await; + }); + + let actor_client = client.actor(&actor.id); + let build = actor_client + .build("0.0", Default::default()) + .await + .expect("start build"); + + let iter = actor_client.builds().iterate(apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = build.id.clone(); + assert!( + common::iter_contains(iter, move |b| b.id == target).await, + "build iteration should yield the started build" + ); +} + /// Complex flow: create an Actor, build it, wait for the build to finish, fetch the build /// and its log, then clean up. #[tokio::test(flavor = "multi_thread")] diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a61c8dc..25e45e8 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -116,6 +116,32 @@ impl Drop for Cleanup { } } +/// Upper bound on how many items an iteration test pulls while searching for a specific +/// just-created resource. Iteration tests sort newest-first, so the target is normally in the +/// first page; the cap only guards against an unbounded scan on a busy shared account. +pub const ITER_SEARCH_CAP: usize = 1000; + +/// Drives a lazy [`ListIterator`](apify_client::ListIterator) looking for an item matching +/// `pred`, pulling at most [`ITER_SEARCH_CAP`] items. Returns `true` as soon as a match is +/// found. Used by the per-collection iteration tests to confirm a just-created resource is +/// reachable through the iterator (exercising the transparent page-fetching path). +pub async fn iter_contains(mut iter: apify_client::ListIterator, mut pred: F) -> bool +where + F: FnMut(&T) -> bool, +{ + let mut pulled = 0usize; + while let Some(item) = iter.next().await.expect("iteration should not error") { + if pred(&item) { + return true; + } + pulled += 1; + if pulled >= ITER_SEARCH_CAP { + break; + } + } + false +} + /// Generates a unique, collision-resistant resource name for test isolation. /// /// The name embeds the test-specific `prefix`, a random UUID fragment, and is kept short diff --git a/tests/dataset.rs b/tests/dataset.rs index 0c403d0..bed3f30 100644 --- a/tests/dataset.rs +++ b/tests/dataset.rs @@ -44,6 +44,77 @@ async fn get_dataset() { assert_eq!(fetched.id, dataset.id); } +/// Iteration: the dataset collection iterator yields a just-created dataset across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_datasets() { + let client = require_client!(); + let name = common::unique_name("dataset-iter"); + let dataset = client + .datasets() + .get_or_create(Some(&name)) + .await + .expect("create dataset"); + + let cleanup_client = client.clone(); + let id = dataset.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.dataset(&id).delete().await; + }); + + // Newest-first with a small page size so the iterator must fetch at least one page. + let iter = client.datasets().iterate(apify_client::StorageListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = dataset.id.clone(); + assert!( + common::iter_contains(iter, move |d| d.id == target).await, + "dataset iteration should yield the created dataset" + ); +} + +/// Iteration: dataset item iterator yields every pushed item exactly once across multiple pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_dataset_items() { + let client = require_client!(); + let name = common::unique_name("dataset-items-iter"); + let dataset = client + .datasets() + .get_or_create(Some(&name)) + .await + .expect("create dataset"); + + let cleanup_client = client.clone(); + let id = dataset.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.dataset(&id).delete().await; + }); + + let dataset_client = client.dataset(&dataset.id); + // Push 5 items so a page size of 2 forces three pages. + dataset_client + .push_items(&json!([{ "n": 0 }, { "n": 1 }, { "n": 2 }, { "n": 3 }, { "n": 4 }])) + .await + .expect("push items"); + + let mut iter = + dataset_client.iterate_items::(apify_client::DatasetListItemsOptions { + limit: Some(2), + ..Default::default() + }); + let mut seen = std::collections::HashSet::new(); + while let Some(item) = iter.next().await.expect("iterate items") { + let n = item["n"].as_i64().expect("item has n"); + assert!(seen.insert(n), "item {n} yielded more than once"); + } + assert_eq!( + seen, + (0..5).collect::>(), + "item iteration must yield every pushed item exactly once across pages" + ); +} + /// Complex flow: create -> get -> push items -> read items -> update -> delete. #[tokio::test(flavor = "multi_thread")] async fn dataset_crud_flow() { diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 2ec5ae0..e222478 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -42,6 +42,37 @@ async fn get_key_value_store() { assert_eq!(fetched.id, store.id); } +/// Iteration: the key-value store collection iterator yields a just-created store across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_key_value_stores() { + let client = require_client!(); + let name = common::unique_name("kvs-iter"); + let store = client + .key_value_stores() + .get_or_create(Some(&name)) + .await + .expect("create store"); + + let cleanup_client = client.clone(); + let id = store.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.key_value_store(&id).delete().await; + }); + + let iter = client + .key_value_stores() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = store.id.clone(); + assert!( + common::iter_contains(iter, move |s| s.id == target).await, + "key-value store iteration should yield the created store" + ); +} + /// Record keys containing characters that are valid for the API (`!`, `'`, `(`, `)`) but /// reserved in a URL path must round-trip correctly, proving the path segment is /// percent-encoded rather than interpolated raw. diff --git a/tests/request_queue.rs b/tests/request_queue.rs index c8a7801..a3d5504 100644 --- a/tests/request_queue.rs +++ b/tests/request_queue.rs @@ -43,6 +43,37 @@ async fn get_request_queue() { assert_eq!(fetched.id, queue.id); } +/// Iteration: the request queue collection iterator yields a just-created queue across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_request_queues() { + let client = require_client!(); + let name = common::unique_name("rq-iter"); + let queue = client + .request_queues() + .get_or_create(Some(&name)) + .await + .expect("create queue"); + + let cleanup_client = client.clone(); + let id = queue.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.request_queue(&id).delete().await; + }); + + let iter = client + .request_queues() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = queue.id.clone(); + assert!( + common::iter_contains(iter, move |q| q.id == target).await, + "request queue iteration should yield the created queue" + ); +} + /// Complex flow: create -> get -> add request -> read request -> list head -> update -> delete. #[tokio::test(flavor = "multi_thread")] async fn request_queue_crud_flow() { diff --git a/tests/schedule.rs b/tests/schedule.rs index 837cba4..f3c3b92 100644 --- a/tests/schedule.rs +++ b/tests/schedule.rs @@ -52,6 +52,35 @@ async fn get_schedule() { assert_eq!(fetched.id, schedule.id); } +/// Iteration: the schedule collection iterator yields a just-created schedule across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_schedules() { + let client = require_client!(); + let name = common::unique_name("schedule-iter"); + let schedule = client + .schedules() + .create(&schedule_definition(&name)) + .await + .expect("create schedule"); + + let cleanup_client = client.clone(); + let id = schedule.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.schedule(&id).delete().await; + }); + + let iter = client.schedules().iterate(apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = schedule.id.clone(); + assert!( + common::iter_contains(iter, move |s| s.id == target).await, + "schedule iteration should yield the created schedule" + ); +} + /// Complex flow: create -> get -> update -> delete a schedule. #[tokio::test(flavor = "multi_thread")] async fn schedule_crud_flow() { diff --git a/tests/task.rs b/tests/task.rs index bf37d0b..98ea475 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -51,6 +51,35 @@ async fn get_task() { assert_eq!(fetched.id, task.id); } +/// Iteration: the task collection iterator yields a just-created task across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_tasks() { + let client = require_client!(); + let name = common::unique_name("task-iter"); + let task = client + .tasks() + .create(&task_definition(&name)) + .await + .expect("create task"); + + let cleanup_client = client.clone(); + let id = task.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.task(&id).delete().await; + }); + + let iter = client.tasks().iterate(apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = task.id.clone(); + assert!( + common::iter_contains(iter, move |t| t.id == target).await, + "task iteration should yield the created task" + ); +} + /// Complex flow: create a task for the public hello-world Actor, get it, update its input, /// list its runs, and delete it. #[tokio::test(flavor = "multi_thread")] diff --git a/tests/webhook.rs b/tests/webhook.rs index ab16d6b..2538ad8 100644 --- a/tests/webhook.rs +++ b/tests/webhook.rs @@ -98,6 +98,72 @@ async fn get_webhook_dispatch() { assert_eq!(fetched.id, dispatch.id); } +/// Iteration: the webhook collection iterator yields a just-created webhook across pages. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_webhooks() { + let client = require_client!(); + let webhook = client + .webhooks() + .create(&webhook_definition()) + .await + .expect("create webhook"); + + let cleanup_client = client.clone(); + let id = webhook.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.webhook(&id).delete().await; + }); + + let iter = client.webhooks().iterate(apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = webhook.id.clone(); + assert!( + common::iter_contains(iter, move |w| w.id == target).await, + "webhook iteration should yield the created webhook" + ); +} + +/// Iteration: the webhook-dispatch collection iterator yields a dispatch we just triggered. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_webhook_dispatches() { + let client = require_client!(); + let webhook = client + .webhooks() + .create(&webhook_definition()) + .await + .expect("create webhook"); + + let cleanup_client = client.clone(); + let id = webhook.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.webhook(&id).delete().await; + }); + + // Trigger a real dispatch so there is a known dispatch id to find. + let dispatch = client + .webhook(&webhook.id) + .test() + .await + .expect("test webhook"); + assert!(!dispatch.id.is_empty()); + + let iter = client + .webhook_dispatches() + .iterate(apify_client::ListOptions { + desc: Some(true), + limit: Some(10), + ..Default::default() + }); + let target = dispatch.id.clone(); + assert!( + common::iter_contains(iter, move |d| d.id == target).await, + "webhook-dispatch iteration should yield the triggered dispatch" + ); +} + /// Complex flow: create -> get -> update -> delete a webhook. #[tokio::test(flavor = "multi_thread")] async fn webhook_crud_flow() { From c8efe6e999a5480f4d59639ad7678d8d550fe37f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 13:42:14 +0000 Subject: [PATCH 03/20] test: add hermetic ListIterator unit tests; single-page env-var iterator Addresses staff-review nits from notes.md: - Add ListIterator::new_single_page for non-paginated endpoints (env-vars), removing the reliance on the API omitting a page limit. - Reword pagination.rs module doc to drop the task/requirements reference. - Add token-free unit tests covering all ListIterator termination branches (short-page, total=0, empty-page, reached_total early-stop, caller offset, single-page); tests exercise collect_all as their drain path. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/clients/actor_env_var_collection.rs | 14 +- src/clients/pagination.rs | 164 ++++++++++++++++++++++-- 2 files changed, 157 insertions(+), 21 deletions(-) diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 4d3f494..19fae35 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -29,16 +29,14 @@ impl ActorEnvVarCollectionClient { /// /// The env-var listing is not offset-paginated (the API returns every variable in a single /// page), so this yields all variables from that one page and then completes. It exists for - /// interface parity with the other collection clients and the reference client. + /// interface parity with the other collection clients and the reference client. Built with + /// [`ListIterator::new_single_page`], which fetches exactly once and never re-requests. pub fn iterate(&self) -> ListIterator { let client = self.clone(); - ListIterator::new( - 0, - Box::new(move |_offset| { - let client = client.clone(); - Box::pin(async move { client.list().await }) - }), - ) + ListIterator::new_single_page(Box::new(move |_offset| { + let client = client.clone(); + Box::pin(async move { client.list().await }) + })) } /// Creates a new environment variable. diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 5f619ab..976114f 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -4,8 +4,8 @@ //! (via its base `_listPaginated`), so callers can iterate across all pages without manually //! tracking offsets. Rust cannot return a value that is simultaneously a `Future` and a //! `Stream`, so the idiomatic equivalent here is a dedicated `iterate()` method on each -//! collection client that returns a [`ListIterator`]. This module implements the paging logic -//! once (the DRY principle the requirements call for) so every client stays a thin wrapper. +//! collection client that returns a [`ListIterator`]. The paging logic lives here once so every +//! client stays a thin wrapper over it (don't-repeat-yourself). use std::collections::VecDeque; use std::future::Future; @@ -48,21 +48,37 @@ pub struct ListIterator { buffer: VecDeque, /// Absolute offset of the next page to request. next_offset: i64, + /// When `true`, the underlying endpoint is not offset-paginated: the first fetch returns the + /// whole result set, so the iterator stops after it rather than requesting a second page. + single_page: bool, /// Set once the listing has been fully consumed. exhausted: bool, } impl ListIterator { - /// Builds an iterator that starts at `start_offset` and fetches pages via `fetch`. + /// Builds an iterator that starts at `start_offset` and fetches offset-paginated pages via + /// `fetch`, walking every page until the listing is exhausted. pub(crate) fn new(start_offset: i64, fetch: PageFetcher) -> Self { Self { fetch, buffer: VecDeque::new(), next_offset: start_offset, + single_page: false, exhausted: false, } } + /// Builds an iterator over an endpoint that is **not** offset-paginated (it returns every + /// item in one response, e.g. an Actor version's environment variables). `fetch` is called + /// exactly once; the iterator does not attempt a second page, so it does not depend on the + /// endpoint reporting a page `limit` and cannot refetch the same items. + pub(crate) fn new_single_page(fetch: PageFetcher) -> Self { + Self { + single_page: true, + ..Self::new(0, fetch) + } + } + /// Returns the next item, or `None` when the listing is exhausted. Fetches another page from /// the API when the local buffer is empty. pub async fn next(&mut self) -> ApifyClientResult> { @@ -81,17 +97,22 @@ impl ListIterator { } self.next_offset += received; - // Decide whether more pages remain. Primary signal is a "short" page: the API returns - // fewer items than the effective page size it reports (`page.limit`), which only happens - // on the final page. This is robust even where `total` is unreliable — the dataset-items - // endpoint, for instance, reports `total = 0`. A non-positive `limit` means the endpoint - // is not offset-paginated (it returned everything at once), so stop after this page to - // avoid refetching it forever. `total`, when the endpoint reports it (> 0), is used only - // as an early stop so a full final page does not cost one extra empty request. - let effective_limit = page.limit; - let reached_total = page.total > 0 && self.next_offset >= page.total; - if effective_limit <= 0 || received < effective_limit || reached_total { + // Decide whether more pages remain. A single-page endpoint returned everything at once, + // so stop unconditionally after the first fetch. Otherwise the primary signal is a + // "short" page: the API returned fewer items than the effective page size it reports + // (`page.limit`), which only happens on the final page. Short-page detection is robust + // even where `total` is unreliable — the dataset-items endpoint, for instance, reports + // `total = 0`. A non-positive reported `limit` is treated as a final page as a safety net. + // `total`, when the endpoint reports it (> 0), is used only as an early stop so a full + // final page does not cost one extra empty request. + if self.single_page { self.exhausted = true; + } else { + let effective_limit = page.limit; + let reached_total = page.total > 0 && self.next_offset >= page.total; + if effective_limit <= 0 || received < effective_limit || reached_total { + self.exhausted = true; + } } self.buffer.extend(page.items); @@ -110,3 +131,120 @@ impl ListIterator { Ok(out) } } + +#[cfg(test)] +mod tests { + use super::{ListIterator, PageFetcher}; + use crate::common::PaginationList; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + /// Builds a fetcher that serves `all` as offset-paginated pages of at most `page_size` + /// items, reporting `limit = page_size` and `total = all.len()` only when `report_total`. + /// Counts how many times it is called so tests can assert page-request behaviour. + fn slicing_fetcher( + all: Vec, + page_size: i64, + report_total: bool, + calls: Arc, + ) -> PageFetcher { + let all = Arc::new(all); + Box::new(move |offset| { + calls.fetch_add(1, Ordering::SeqCst); + let all = all.clone(); + Box::pin(async move { + let start = (offset.max(0) as usize).min(all.len()); + let end = (start + page_size.max(0) as usize).min(all.len()); + let items = all[start..end].to_vec(); + Ok(PaginationList { + total: if report_total { all.len() as i64 } else { 0 }, + offset, + limit: page_size, + count: items.len() as i64, + desc: false, + items, + }) + }) + }) + } + + #[tokio::test] + async fn walks_all_pages_using_reported_total() { + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new(0, slicing_fetcher((0..5).collect(), 2, true, calls.clone())); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3, 4]); + // Pages: [0,1] [2,3] [4]. The last page is short, so no extra empty request. + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn walks_all_pages_when_total_is_zero() { + // Emulates the dataset-items endpoint, which reports total = 0. + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new(0, slicing_fetcher((0..5).collect(), 2, false, calls)); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn total_zero_exact_multiple_terminates_on_empty_page() { + // 4 items, page size 2, no usable total: pages [0,1] [2,3] [] — the empty page ends it. + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new( + 0, + slicing_fetcher((0..4).collect(), 2, false, calls.clone()), + ); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3]); + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn reported_total_avoids_extra_request_on_exact_multiple() { + // 4 items, page size 2, total known: stops after the second full page (no empty fetch). + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new(0, slicing_fetcher((0..4).collect(), 2, true, calls.clone())); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3]); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn honours_caller_start_offset() { + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new(2, slicing_fetcher((0..5).collect(), 10, true, calls)); + assert_eq!(iter.collect_all().await.unwrap(), vec![2, 3, 4]); + } + + #[tokio::test] + async fn single_page_fetches_once_and_stops() { + // A non-paginated endpoint: the fetcher ignores offset and returns everything every call, + // reporting limit == count. Multi-page mode would loop forever here; single-page must not. + let calls = Arc::new(AtomicUsize::new(0)); + let counter = calls.clone(); + let fetch: PageFetcher = Box::new(move |offset| { + counter.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + Ok(PaginationList { + total: 0, + offset, + limit: 3, + count: 3, + desc: false, + items: vec![10, 20, 30], + }) + }) + }); + let iter = ListIterator::new_single_page(fetch); + assert_eq!(iter.collect_all().await.unwrap(), vec![10, 20, 30]); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "single-page must fetch exactly once" + ); + } + + #[tokio::test] + async fn empty_first_page_yields_nothing() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut iter = ListIterator::new(0, slicing_fetcher(vec![], 5, true, calls)); + assert!(iter.next().await.unwrap().is_none()); + } +} From 84a9f3d0b986341cf9323269e78925a82b00176e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 14:05:20 +0000 Subject: [PATCH 04/20] docs: clarify iterate() limit semantics; drop process reference in base.rs comment Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/clients/base.rs | 2 +- src/clients/pagination.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/clients/base.rs b/src/clients/base.rs index 1439c4e..65e258f 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -3,7 +3,7 @@ //! [`ResourceContext`] holds the resolved URL and the [`HttpClient`] for a single //! resource (or sub-resource). The free functions in this module implement the CRUD //! and wait-for-finish primitives once, so that every resource client stays small and -//! consistent (the DRY principle the requirements call for). +//! consistent (the DRY principle). use std::time::Duration; diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 976114f..171770d 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -26,9 +26,14 @@ type PageFetcher = Box PageFuture + Send + Sync>; /// /// Created by a collection client's `iterate()` method. Each call to [`next`](Self::next) /// returns the next item, transparently fetching the following page from the API once the -/// local buffer drains, until every item across all pages has been yielded. The caller's -/// per-page `limit` (if any) is honoured as the page size; iteration always walks the full -/// result set regardless. +/// local buffer drains, until every item across all pages has been yielded. +/// +/// The caller's `limit` (if any) is the **page size**, not a cap on the total number of items +/// yielded: iteration always walks the full result set regardless. This mirrors the reference +/// JavaScript client's `AsyncIterable`, where `limit` likewise controls page size while the +/// iterable spans every page. There is intentionally no first-class "take N items" option — a +/// caller wanting only the first N should stop calling [`next`](Self::next) after N items +/// rather than expect `limit` to bound the stream. /// /// # Example /// ```no_run From 28b49d1642ecbc8999a1a075d5d5d9525d30c133 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 14:34:15 +0000 Subject: [PATCH 05/20] fix: align iterate() limit with reference (total cap) and stop short-page truncation - ListIterator: limit is now a total-item cap; page size set via with_chunk_size, matching the reference _listPaginatedFromCallback (total-driven termination). - iterate_items no longer truncates when skip_empty/clean/skip_hidden shorten a non-final page (short-page detection confined to the total==0 case). - Corrected misleading limit-as-page-size docs (pagination.rs, dataset.rs, docs/README.md). - Added hermetic unit tests for non-final short pages and total-cap behaviour. - README: bump apify-client 0.5 -> 0.6 to match Cargo.toml. - Iteration integration tests use with_chunk_size for page size (limit unset). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 4 +- README.md | 4 +- docs/README.md | 4 +- examples/iterate_store.rs | 13 +- src/clients/actor_collection.rs | 5 +- src/clients/actor_env_var_collection.rs | 2 +- src/clients/actor_version_collection.rs | 5 +- src/clients/build_collection.rs | 5 +- src/clients/dataset.rs | 10 +- src/clients/dataset_collection.rs | 5 +- src/clients/key_value_store_collection.rs | 5 +- src/clients/pagination.rs | 247 +++++++++++++++++---- src/clients/request_queue_collection.rs | 5 +- src/clients/run_collection.rs | 5 +- src/clients/schedule_collection.rs | 5 +- src/clients/store_collection.rs | 7 +- src/clients/task_collection.rs | 5 +- src/clients/webhook_collection.rs | 5 +- src/clients/webhook_dispatch_collection.rs | 5 +- tests/actor.rs | 14 +- tests/actor_run.rs | 21 +- tests/build.rs | 12 +- tests/dataset.rs | 22 +- tests/key_value_store.rs | 4 +- tests/request_queue.rs | 4 +- tests/schedule.rs | 12 +- tests/task.rs | 12 +- tests/webhook.rs | 16 +- 28 files changed, 337 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e6016b..dc9e74f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,9 @@ to [Semantic Versioning](https://semver.org/). version, environment-variable, build, run, dataset, key-value-store, request-queue, schedule, task, webhook, and webhook-dispatch collection clients, plus `DatasetClient::iterate_items()` for dataset items. Each yields one item at a time, fetching pages on demand — the idiomatic - counterpart to the reference client's async-iterable list results. + counterpart to the reference client's async-iterable list results. The options' `limit` caps + the total number of items yielded (matching the reference client), and `ListIterator::with_chunk_size` + sets the per-request page size. ### Changed - Bumped `API_SPEC_VERSION` to `v2-2026-07-10T105921Z`. The spec delta (added `401`/`402` diff --git a/README.md b/README.md index 1794eb7..70839c0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ It provides a resource-oriented, async interface that mirrors the official ```toml [dependencies] -apify-client = "0.5" +apify-client = "0.6" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } serde_json = "1" # for the `serde_json::Value` responses used in the Quick start ``` @@ -49,7 +49,7 @@ project needs `serde_json`. Two more dependencies are needed only for specific f By default the client uses the system TLS (`native-tls`). To use rustls instead: ```toml -apify-client = { version = "0.5", default-features = false, features = ["rustls"] } +apify-client = { version = "0.6", default-features = false, features = ["rustls"] } ``` ## Quick start diff --git a/docs/README.md b/docs/README.md index b73e001..2352632 100644 --- a/docs/README.md +++ b/docs/README.md @@ -188,7 +188,9 @@ lazy `ListIterator` (re-exported at the crate root) that fetches the next page f demand as you consume items. Every collection client provides it (`actors`, `builds`, `runs`, `tasks`, `datasets`, `key_value_stores`, `request_queues`, `schedules`, `webhooks`, `webhook_dispatches`, `store`, and the nested Actor `versions`/`env_vars`), and `DatasetClient` -exposes `iterate_items()` for dataset items. Any per-call `limit` is used as the page size. +exposes `iterate_items()` for dataset items. The options' `limit` caps the total number of items +yielded (unset iterates everything); to control the per-request page size, call +`.with_chunk_size(n)` on the returned iterator. ```rust,no_run use apify_client::{ApifyClient, ActorListOptions}; diff --git a/examples/iterate_store.rs b/examples/iterate_store.rs index 3e2cf0b..c2b9be5 100644 --- a/examples/iterate_store.rs +++ b/examples/iterate_store.rs @@ -9,12 +9,13 @@ async fn main() -> Result<(), Box> { let token = std::env::var("APIFY_TOKEN").expect("set APIFY_TOKEN"); let client = ApifyClient::new(token); - // Iterate the store, fetching pages of 5 on demand (`limit` is the per-page size, not a - // total cap). The loop below stops after the first 10 actors regardless of page size. - let mut iter = client.store().iterate(StoreListOptions { - limit: Some(5), - ..Default::default() - }); + // Iterate the store, fetching pages of 5 on demand. `with_chunk_size` sets the per-request + // page size; the options' `limit` (left unset here) would instead cap the total number of + // items yielded. The loop below stops after the first 10 actors regardless of page size. + let mut iter = client + .store() + .iterate(StoreListOptions::default()) + .with_chunk_size(5); let mut count = 0; while let Some(actor) = iter.next().await? { diff --git a/src/clients/actor_collection.rs b/src/clients/actor_collection.rs index 34a1742..67621fa 100644 --- a/src/clients/actor_collection.rs +++ b/src/clients/actor_collection.rs @@ -61,12 +61,15 @@ impl ActorCollectionClient { pub fn iterate(&self, options: ActorListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 19fae35..14830b7 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -33,7 +33,7 @@ impl ActorEnvVarCollectionClient { /// [`ListIterator::new_single_page`], which fetches exactly once and never re-requests. pub fn iterate(&self) -> ListIterator { let client = self.clone(); - ListIterator::new_single_page(Box::new(move |_offset| { + ListIterator::new_single_page(Box::new(move |_offset, _page_limit| { let client = client.clone(); Box::pin(async move { client.list().await }) })) diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index 1a8685b..47f7b95 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -39,12 +39,15 @@ impl ActorVersionCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/build_collection.rs b/src/clients/build_collection.rs index 1eaf307..53fa245 100644 --- a/src/clients/build_collection.rs +++ b/src/clients/build_collection.rs @@ -41,12 +41,15 @@ impl BuildCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index d460b11..5239ed7 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -237,19 +237,25 @@ impl DatasetClient { /// /// The idiomatic-Rust counterpart of the reference client's async-iterable /// `listItems`/`iterateItems`: yields one deserialized item of type `T` at a time, - /// transparently paging with the caller's `options` (its `limit` acts as the page size). + /// transparently paging. The caller's `options.limit` caps the total number of items yielded + /// (unset = all); use [`ListIterator::with_chunk_size`] to control the per-page fetch size. + /// Filtering options such as `skip_empty`/`clean`/`skip_hidden` are honoured across pages + /// without truncating the result. pub fn iterate_items( &self, options: DatasetListItemsOptions, ) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list_items::(options).await }) }), ) diff --git a/src/clients/dataset_collection.rs b/src/clients/dataset_collection.rs index 9c7c39b..170e092 100644 --- a/src/clients/dataset_collection.rs +++ b/src/clients/dataset_collection.rs @@ -34,12 +34,15 @@ impl DatasetCollectionClient { pub fn iterate(&self, options: StorageListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/key_value_store_collection.rs b/src/clients/key_value_store_collection.rs index cd18a64..728d04d 100644 --- a/src/clients/key_value_store_collection.rs +++ b/src/clients/key_value_store_collection.rs @@ -35,12 +35,15 @@ impl KeyValueStoreCollectionClient { pub fn iterate(&self, options: StorageListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 171770d..ec22bf1 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -1,11 +1,11 @@ //! Generic lazy pagination shared by every collection client. //! //! The reference JavaScript client returns an `AsyncIterable` from every collection `list()` -//! (via its base `_listPaginated`), so callers can iterate across all pages without manually -//! tracking offsets. Rust cannot return a value that is simultaneously a `Future` and a -//! `Stream`, so the idiomatic equivalent here is a dedicated `iterate()` method on each -//! collection client that returns a [`ListIterator`]. The paging logic lives here once so every -//! client stays a thin wrapper over it (don't-repeat-yourself). +//! (via its base `_listPaginatedFromCallback`), so callers can iterate across all pages without +//! manually tracking offsets. Rust cannot return a value that is simultaneously a `Future` and a +//! `Stream`, so the idiomatic equivalent here is a dedicated `iterate()` method on each collection +//! client that returns a [`ListIterator`]. The paging logic lives here once so every client stays +//! a thin wrapper over it (don't-repeat-yourself). use std::collections::VecDeque; use std::future::Future; @@ -18,22 +18,41 @@ use crate::error::ApifyClientResult; /// for any concrete collection client without being generic over its (unnameable) future type. type PageFuture = Pin>> + Send>>; -/// Fetches the page starting at the given absolute `offset`. Implementations capture a clone of -/// the collection client and the caller's list options, overriding only the offset per page. -type PageFetcher = Box PageFuture + Send + Sync>; +/// Fetches one page: the arguments are the absolute `offset` to start at and the per-page `limit` +/// to request (`None` = let the API pick its default page size). Implementations capture a clone of +/// the collection client and the caller's list options, overriding only the offset and limit per +/// page. +type PageFetcher = Box) -> PageFuture + Send + Sync>; + +/// Returns the smaller of two optional positive limits, treating a non-positive value as "no +/// limit" (`None`). Mirrors the reference client's `minForLimitParam`, where the API treats `0` +/// as an absent limit. +fn min_positive_limit(a: Option, b: Option) -> Option { + let a = a.filter(|&x| x > 0); + let b = b.filter(|&x| x > 0); + match (a, b) { + (Some(x), Some(y)) => Some(x.min(y)), + (Some(x), None) | (None, Some(x)) => Some(x), + (None, None) => None, + } +} /// A lazy, page-fetching async iterator over an offset/limit-paginated list endpoint. /// /// Created by a collection client's `iterate()` method. Each call to [`next`](Self::next) /// returns the next item, transparently fetching the following page from the API once the -/// local buffer drains, until every item across all pages has been yielded. +/// local buffer drains, until the listing is exhausted (or the caller's total-item cap is hit). +/// +/// # `limit` vs. page size /// -/// The caller's `limit` (if any) is the **page size**, not a cap on the total number of items -/// yielded: iteration always walks the full result set regardless. This mirrors the reference -/// JavaScript client's `AsyncIterable`, where `limit` likewise controls page size while the -/// iterable spans every page. There is intentionally no first-class "take N items" option — a -/// caller wanting only the first N should stop calling [`next`](Self::next) after N items -/// rather than expect `limit` to bound the stream. +/// The caller's `limit` (from the list options passed to `iterate()`) is a **cap on the total +/// number of items the iterator yields**, matching the reference JavaScript client's +/// `_listPaginatedFromCallback`, where `options.limit` bounds the whole async-iterable and a +/// separate `chunkSize` controls page size. Leaving `limit` unset (or `0`) iterates the entire +/// listing. The page size is a distinct concern: set it with [`with_chunk_size`](Self::with_chunk_size); +/// when unset, the API's default page size is used. So `iterate(opts{ limit: 10 })` yields at most +/// 10 items, and `iterate(opts).with_chunk_size(50)` fetches 50 per request while yielding +/// everything. /// /// # Example /// ```no_run @@ -53,6 +72,12 @@ pub struct ListIterator { buffer: VecDeque, /// Absolute offset of the next page to request. next_offset: i64, + /// Number of items still allowed under the caller's total-item cap; `None` = uncapped. + /// Decremented by each page's item count as pages are fetched. + remaining: Option, + /// Page size to request per fetch (the reference client's `chunkSize`); `None` = let the API + /// choose its default page size. + chunk_size: Option, /// When `true`, the underlying endpoint is not offset-paginated: the first fetch returns the /// whole result set, so the iterator stops after it rather than requesting a second page. single_page: bool, @@ -61,13 +86,16 @@ pub struct ListIterator { } impl ListIterator { - /// Builds an iterator that starts at `start_offset` and fetches offset-paginated pages via - /// `fetch`, walking every page until the listing is exhausted. - pub(crate) fn new(start_offset: i64, fetch: PageFetcher) -> Self { + /// Builds an iterator that starts at `start_offset`, yields at most `total_limit` items across + /// all pages (`None`/`0` = uncapped), and fetches offset-paginated pages via `fetch`. + pub(crate) fn new(start_offset: i64, total_limit: Option, fetch: PageFetcher) -> Self { + let cap = total_limit.filter(|&l| l > 0); Self { fetch, buffer: VecDeque::new(), next_offset: start_offset, + remaining: cap, + chunk_size: None, single_page: false, exhausted: false, } @@ -80,10 +108,19 @@ impl ListIterator { pub(crate) fn new_single_page(fetch: PageFetcher) -> Self { Self { single_page: true, - ..Self::new(0, fetch) + ..Self::new(0, None, fetch) } } + /// Sets the page size (items requested per API call) for this iteration — the reference + /// client's `chunkSize`. This controls only how many items each page fetch requests, never how + /// many the iterator yields in total (that is the caller's `limit`; see the type docs). A + /// non-positive value lets the API choose its default page size. + pub fn with_chunk_size(mut self, chunk_size: i64) -> Self { + self.chunk_size = (chunk_size > 0).then_some(chunk_size); + self + } + /// Returns the next item, or `None` when the listing is exhausted. Fetches another page from /// the API when the local buffer is empty. pub async fn next(&mut self) -> ApifyClientResult> { @@ -94,33 +131,59 @@ impl ListIterator { return Ok(None); } - let page = (self.fetch)(self.next_offset).await?; + // Request the smaller of the items still allowed under the caller's cap (`remaining`) and + // the configured page size (`chunk_size`); `None` lets the API pick its default page size. + // On the first page `remaining` is the full cap, matching the reference client's initial + // `minForLimitParam(options.limit, options.chunkSize)`. + let page_limit = min_positive_limit(self.remaining, self.chunk_size); + let page = (self.fetch)(self.next_offset, page_limit).await?; let received = page.items.len() as i64; - if received == 0 { - self.exhausted = true; - return Ok(None); + + // Enforce the caller's total-item cap exactly, even if the API returns more than the + // requested page limit. + let mut items = page.items; + if let Some(rem) = self.remaining { + if received > rem { + items.truncate(rem.max(0) as usize); + } } + self.next_offset += received; + if let Some(rem) = self.remaining.as_mut() { + *rem -= received; + } - // Decide whether more pages remain. A single-page endpoint returned everything at once, - // so stop unconditionally after the first fetch. Otherwise the primary signal is a - // "short" page: the API returned fewer items than the effective page size it reports - // (`page.limit`), which only happens on the final page. Short-page detection is robust - // even where `total` is unreliable — the dataset-items endpoint, for instance, reports - // `total = 0`. A non-positive reported `limit` is treated as a final page as a safety net. - // `total`, when the endpoint reports it (> 0), is used only as an early stop so a full - // final page does not cost one extra empty request. + // Decide whether more pages remain. if self.single_page { + // Non-paginated endpoint: everything came back in one response. + self.exhausted = true; + } else if received == 0 { + // Empty page: nothing more to read. This is the primary backstop and matches the + // reference client, whose loop stops as soon as a page returns no items. + self.exhausted = true; + } else if matches!(self.remaining, Some(r) if r <= 0) { + // Reached the caller's total-item cap. self.exhausted = true; + } else if page.total > 0 { + // The endpoint reports a usable total, so drive termination by position, like the + // reference `_listPaginatedFromCallback`. A short page is deliberately NOT treated as + // terminal here: with dataset item filters (`skip_empty`/`clean`/`skip_hidden`) a full, + // non-final window can return fewer items than requested while more remain at higher + // offsets, so short-page detection would silently truncate. The empty-page backstop + // above ends the walk instead. + if self.next_offset >= page.total { + self.exhausted = true; + } } else { - let effective_limit = page.limit; - let reached_total = page.total > 0 && self.next_offset >= page.total; - if effective_limit <= 0 || received < effective_limit || reached_total { + // No usable total (endpoint reports `total == 0`): fall back to short-page detection — + // a page shorter than the size the API says it served (`page.limit`) is the last one. + // A non-positive reported limit means the endpoint is not offset-paginated at all. + if page.limit <= 0 || received < page.limit { self.exhausted = true; } } - self.buffer.extend(page.items); + self.buffer.extend(items); Ok(self.buffer.pop_front()) } @@ -144,19 +207,22 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; - /// Builds a fetcher that serves `all` as offset-paginated pages of at most `page_size` - /// items, reporting `limit = page_size` and `total = all.len()` only when `report_total`. - /// Counts how many times it is called so tests can assert page-request behaviour. + /// Builds a fetcher that serves `all` as offset-paginated pages, honouring the per-page `limit` + /// requested by the iterator (falling back to `default_page` when the iterator requests none). + /// Reports `total = all.len()` only when `report_total`, and echoes the effective page size back + /// as the response `limit`. Counts how many times it is called so tests can assert page-request + /// behaviour. fn slicing_fetcher( all: Vec, - page_size: i64, + default_page: i64, report_total: bool, calls: Arc, ) -> PageFetcher { let all = Arc::new(all); - Box::new(move |offset| { + Box::new(move |offset, limit| { calls.fetch_add(1, Ordering::SeqCst); let all = all.clone(); + let page_size = limit.filter(|&l| l > 0).unwrap_or(default_page); Box::pin(async move { let start = (offset.max(0) as usize).min(all.len()); let end = (start + page_size.max(0) as usize).min(all.len()); @@ -176,17 +242,24 @@ mod tests { #[tokio::test] async fn walks_all_pages_using_reported_total() { let calls = Arc::new(AtomicUsize::new(0)); - let iter = ListIterator::new(0, slicing_fetcher((0..5).collect(), 2, true, calls.clone())); + let iter = ListIterator::new( + 0, + None, + slicing_fetcher((0..5).collect(), 2, true, calls.clone()), + ) + .with_chunk_size(2); assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3, 4]); - // Pages: [0,1] [2,3] [4]. The last page is short, so no extra empty request. + // Pages: [0,1] [2,3] [4]. next_offset reaches total (5) on the third page, so no extra + // empty request. assert_eq!(calls.load(Ordering::SeqCst), 3); } #[tokio::test] async fn walks_all_pages_when_total_is_zero() { - // Emulates the dataset-items endpoint, which reports total = 0. + // Emulates an endpoint that does not report a usable total: short-page detection ends it. let calls = Arc::new(AtomicUsize::new(0)); - let iter = ListIterator::new(0, slicing_fetcher((0..5).collect(), 2, false, calls)); + let iter = ListIterator::new(0, None, slicing_fetcher((0..5).collect(), 2, false, calls)) + .with_chunk_size(2); assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3, 4]); } @@ -196,8 +269,10 @@ mod tests { let calls = Arc::new(AtomicUsize::new(0)); let iter = ListIterator::new( 0, + None, slicing_fetcher((0..4).collect(), 2, false, calls.clone()), - ); + ) + .with_chunk_size(2); assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3]); assert_eq!(calls.load(Ordering::SeqCst), 3); } @@ -206,15 +281,89 @@ mod tests { async fn reported_total_avoids_extra_request_on_exact_multiple() { // 4 items, page size 2, total known: stops after the second full page (no empty fetch). let calls = Arc::new(AtomicUsize::new(0)); - let iter = ListIterator::new(0, slicing_fetcher((0..4).collect(), 2, true, calls.clone())); + let iter = ListIterator::new( + 0, + None, + slicing_fetcher((0..4).collect(), 2, true, calls.clone()), + ) + .with_chunk_size(2); assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3]); assert_eq!(calls.load(Ordering::SeqCst), 2); } + #[tokio::test] + async fn non_final_short_page_with_reported_total_does_not_truncate() { + // Regression test for the dataset-items filtering case (Finding 1): every page is "short" + // — it returns fewer items than the page size the API reports (as `skip_empty`/`clean` do, + // where a full raw window omits filtered-out items) — while the endpoint reports a large + // total. The old `received < page.limit` termination stopped after page 1 and silently + // dropped the rest; total-driven termination must keep going and yield every item, ending + // only on the empty page. + let all: Vec = (0..6).collect(); + let calls = Arc::new(AtomicUsize::new(0)); + let counter = calls.clone(); + let all = Arc::new(all); + let fetch: PageFetcher = Box::new(move |offset, _limit| { + counter.fetch_add(1, Ordering::SeqCst); + let all = all.clone(); + Box::pin(async move { + // Serve at most 2 items per page but report a page limit of 4, so every non-final + // page is strictly short (received 2 < reported limit 4). `total` is reported large + // (100) so termination can only come from the empty page, never a short page. + let start = (offset.max(0) as usize).min(all.len()); + let end = (start + 2).min(all.len()); + let items = all[start..end].to_vec(); + Ok(PaginationList { + total: 100, + offset, + limit: 4, + count: items.len() as i64, + desc: false, + items, + }) + }) + }); + let iter = ListIterator::new(0, None, fetch).with_chunk_size(4); + let got = iter.collect_all().await.unwrap(); + // Every item must be yielded despite each page being short. + assert_eq!(got, vec![0, 1, 2, 3, 4, 5]); + // Pages: [0,1] [2,3] [4,5] [] — four calls, none terminated early by short-page detection. + assert_eq!(calls.load(Ordering::SeqCst), 4); + } + + #[tokio::test] + async fn total_limit_caps_items_yielded() { + // `limit` is a total-item cap: with 100 items available, iterate should yield exactly 3. + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new( + 0, + Some(3), + slicing_fetcher((0..100).collect(), 1000, true, calls.clone()), + ); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2]); + // Requesting limit=3 up front means a single page suffices. + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn total_limit_with_smaller_chunk_size_pages_and_caps() { + // limit=5 total cap, chunk_size=2 page size: pages of 2 until 5 items are yielded. + let calls = Arc::new(AtomicUsize::new(0)); + let iter = ListIterator::new( + 0, + Some(5), + slicing_fetcher((0..100).collect(), 1000, true, calls.clone()), + ) + .with_chunk_size(2); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2, 3, 4]); + // Pages: [0,1] [2,3] [4] — the last page is trimmed to the remaining budget of 1. + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + #[tokio::test] async fn honours_caller_start_offset() { let calls = Arc::new(AtomicUsize::new(0)); - let iter = ListIterator::new(2, slicing_fetcher((0..5).collect(), 10, true, calls)); + let iter = ListIterator::new(2, None, slicing_fetcher((0..5).collect(), 10, true, calls)); assert_eq!(iter.collect_all().await.unwrap(), vec![2, 3, 4]); } @@ -224,7 +373,7 @@ mod tests { // reporting limit == count. Multi-page mode would loop forever here; single-page must not. let calls = Arc::new(AtomicUsize::new(0)); let counter = calls.clone(); - let fetch: PageFetcher = Box::new(move |offset| { + let fetch: PageFetcher = Box::new(move |offset, _limit| { counter.fetch_add(1, Ordering::SeqCst); Box::pin(async move { Ok(PaginationList { @@ -249,7 +398,7 @@ mod tests { #[tokio::test] async fn empty_first_page_yields_nothing() { let calls = Arc::new(AtomicUsize::new(0)); - let mut iter = ListIterator::new(0, slicing_fetcher(vec![], 5, true, calls)); + let mut iter = ListIterator::new(0, None, slicing_fetcher(vec![], 5, true, calls)); assert!(iter.next().await.unwrap().is_none()); } } diff --git a/src/clients/request_queue_collection.rs b/src/clients/request_queue_collection.rs index bbfadd0..bb0245e 100644 --- a/src/clients/request_queue_collection.rs +++ b/src/clients/request_queue_collection.rs @@ -35,12 +35,15 @@ impl RequestQueueCollectionClient { pub fn iterate(&self, options: StorageListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/run_collection.rs b/src/clients/run_collection.rs index ac5ec38..545fdec 100644 --- a/src/clients/run_collection.rs +++ b/src/clients/run_collection.rs @@ -86,13 +86,16 @@ impl RunCollectionClient { pub fn iterate(&self, options: ListOptions, filter: RunListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); let filter = filter.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options, filter).await }) }), ) diff --git a/src/clients/schedule_collection.rs b/src/clients/schedule_collection.rs index 83ef4e5..31a2577 100644 --- a/src/clients/schedule_collection.rs +++ b/src/clients/schedule_collection.rs @@ -36,12 +36,15 @@ impl ScheduleCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/store_collection.rs b/src/clients/store_collection.rs index 2654aac..f28c3da 100644 --- a/src/clients/store_collection.rs +++ b/src/clients/store_collection.rs @@ -19,7 +19,7 @@ pub type StoreActorIterator = ListIterator; pub struct StoreListOptions { /// Number of items to skip. pub offset: Option, - /// Maximum number of items to return per page. + /// Maximum number of items to return. pub limit: Option, /// Full-text search query. pub search: Option, @@ -71,12 +71,15 @@ impl StoreCollectionClient { pub fn iterate(&self, options: StoreListOptions) -> StoreActorIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/task_collection.rs b/src/clients/task_collection.rs index 86b8e81..b22eef4 100644 --- a/src/clients/task_collection.rs +++ b/src/clients/task_collection.rs @@ -36,12 +36,15 @@ impl TaskCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/webhook_collection.rs b/src/clients/webhook_collection.rs index 4a2fc16..d2b13af 100644 --- a/src/clients/webhook_collection.rs +++ b/src/clients/webhook_collection.rs @@ -43,12 +43,15 @@ impl WebhookCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/src/clients/webhook_dispatch_collection.rs b/src/clients/webhook_dispatch_collection.rs index 578341a..c5f4da2 100644 --- a/src/clients/webhook_dispatch_collection.rs +++ b/src/clients/webhook_dispatch_collection.rs @@ -44,12 +44,15 @@ impl WebhookDispatchCollectionClient { pub fn iterate(&self, options: ListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); + let total_limit = options.limit; ListIterator::new( start, - Box::new(move |offset| { + total_limit, + Box::new(move |offset, page_limit| { let client = client.clone(); let mut options = options.clone(); options.offset = Some(offset); + options.limit = page_limit; Box::pin(async move { client.list(options).await }) }), ) diff --git a/tests/actor.rs b/tests/actor.rs index 1e88e58..83928a4 100644 --- a/tests/actor.rs +++ b/tests/actor.rs @@ -92,12 +92,14 @@ async fn iterate_actors() { }); // Restrict to the caller's own Actors, newest-first, with a small page size. - let iter = client.actors().iterate(apify_client::ActorListOptions { - my: Some(true), - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = client + .actors() + .iterate(apify_client::ActorListOptions { + my: Some(true), + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = actor.id.clone(); assert!( common::iter_contains(iter, move |a| a.id == target).await, diff --git a/tests/actor_run.rs b/tests/actor_run.rs index 27da6bf..e708b46 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -94,15 +94,18 @@ async fn iterate_runs() { .await .expect("call hello-world actor"); - // Newest-first with a small page size so the just-finished run is near the front. - let iter = client.runs().iterate( - apify_client::ListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }, - Default::default(), - ); + // Newest-first with a small page size so the just-finished run is near the front. `limit` + // is a total-item cap, so it is left unset here; page size is set via `with_chunk_size`. + let iter = client + .runs() + .iterate( + apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }, + Default::default(), + ) + .with_chunk_size(5); let target = run.id.clone(); assert!( common::iter_contains(iter, move |r| r.id == target).await, diff --git a/tests/build.rs b/tests/build.rs index 5a06206..bfc96e3 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -62,11 +62,13 @@ async fn iterate_builds() { .await .expect("start build"); - let iter = actor_client.builds().iterate(apify_client::ListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = actor_client + .builds() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = build.id.clone(); assert!( common::iter_contains(iter, move |b| b.id == target).await, diff --git a/tests/dataset.rs b/tests/dataset.rs index bed3f30..118005e 100644 --- a/tests/dataset.rs +++ b/tests/dataset.rs @@ -62,11 +62,13 @@ async fn iterate_datasets() { }); // Newest-first with a small page size so the iterator must fetch at least one page. - let iter = client.datasets().iterate(apify_client::StorageListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = client + .datasets() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = dataset.id.clone(); assert!( common::iter_contains(iter, move |d| d.id == target).await, @@ -98,11 +100,11 @@ async fn iterate_dataset_items() { .await .expect("push items"); - let mut iter = - dataset_client.iterate_items::(apify_client::DatasetListItemsOptions { - limit: Some(2), - ..Default::default() - }); + // A page size of 2 forces three pages; `limit` is a total-item cap (left unset here) rather + // than the page size, so every pushed item must still be yielded across pages. + let mut iter = dataset_client + .iterate_items::(apify_client::DatasetListItemsOptions::default()) + .with_chunk_size(2); let mut seen = std::collections::HashSet::new(); while let Some(item) = iter.next().await.expect("iterate items") { let n = item["n"].as_i64().expect("item has n"); diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index e222478..e341001 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -63,9 +63,9 @@ async fn iterate_key_value_stores() { .key_value_stores() .iterate(apify_client::StorageListOptions { desc: Some(true), - limit: Some(10), ..Default::default() - }); + }) + .with_chunk_size(5); let target = store.id.clone(); assert!( common::iter_contains(iter, move |s| s.id == target).await, diff --git a/tests/request_queue.rs b/tests/request_queue.rs index a3d5504..219e670 100644 --- a/tests/request_queue.rs +++ b/tests/request_queue.rs @@ -64,9 +64,9 @@ async fn iterate_request_queues() { .request_queues() .iterate(apify_client::StorageListOptions { desc: Some(true), - limit: Some(10), ..Default::default() - }); + }) + .with_chunk_size(5); let target = queue.id.clone(); assert!( common::iter_contains(iter, move |q| q.id == target).await, diff --git a/tests/schedule.rs b/tests/schedule.rs index f3c3b92..b9b1170 100644 --- a/tests/schedule.rs +++ b/tests/schedule.rs @@ -69,11 +69,13 @@ async fn iterate_schedules() { let _ = cleanup_client.schedule(&id).delete().await; }); - let iter = client.schedules().iterate(apify_client::ListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = client + .schedules() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = schedule.id.clone(); assert!( common::iter_contains(iter, move |s| s.id == target).await, diff --git a/tests/task.rs b/tests/task.rs index 98ea475..9155567 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -68,11 +68,13 @@ async fn iterate_tasks() { let _ = cleanup_client.task(&id).delete().await; }); - let iter = client.tasks().iterate(apify_client::ListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = client + .tasks() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = task.id.clone(); assert!( common::iter_contains(iter, move |t| t.id == target).await, diff --git a/tests/webhook.rs b/tests/webhook.rs index 2538ad8..ea27073 100644 --- a/tests/webhook.rs +++ b/tests/webhook.rs @@ -114,11 +114,13 @@ async fn iterate_webhooks() { let _ = cleanup_client.webhook(&id).delete().await; }); - let iter = client.webhooks().iterate(apify_client::ListOptions { - desc: Some(true), - limit: Some(10), - ..Default::default() - }); + let iter = client + .webhooks() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5); let target = webhook.id.clone(); assert!( common::iter_contains(iter, move |w| w.id == target).await, @@ -154,9 +156,9 @@ async fn iterate_webhook_dispatches() { .webhook_dispatches() .iterate(apify_client::ListOptions { desc: Some(true), - limit: Some(10), ..Default::default() - }); + }) + .with_chunk_size(5); let target = dispatch.id.clone(); assert!( common::iter_contains(iter, move |d| d.id == target).await, From 4d84c411a6c0212354d705ee95977b98a4abf37e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 14:42:50 +0000 Subject: [PATCH 06/20] test: force multi-page fetching in iterate_store via with_chunk_size Migrate the last iteration test off limit-as-page-size (now a total cap) so its seen>=12 break is reachable and it exercises page-crossing, matching siblings. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- tests/store.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/store.rs b/tests/store.rs index b36540e..26e507d 100644 --- a/tests/store.rs +++ b/tests/store.rs @@ -23,10 +23,12 @@ async fn list_store() { #[tokio::test(flavor = "multi_thread")] async fn iterate_store() { let client = require_client!(); - let mut iter = client.store().iterate(StoreListOptions { - limit: Some(5), - ..Default::default() - }); + // Page size of 5 (via `with_chunk_size`) so pulling 12 items forces multiple page fetches; + // `limit` is a total-item cap and is left unset so iteration is not bounded to one page. + let mut iter = client + .store() + .iterate(StoreListOptions::default()) + .with_chunk_size(5); // Pull a handful of items; the iterator should fetch pages transparently. let mut seen = 0; From 44b8f7042714f6fbc594b2d84f59c1634020b5ed Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 15:06:17 +0000 Subject: [PATCH 07/20] docs: correct iterate_items/pagination docs and add cap-truncation test Address 5 open review items (behaviour unchanged; reference-consistent): - iterate_items docstring now describes real offset-advance behaviour (may re-yield filtered items), matching the reference JS client, instead of overpromising filters are honoured without truncation. - Reword non_final_short_page test comment: it guards termination logic, not filter de-duplication. - Add cap_truncates_page_that_exceeds_remaining_budget test for the received > remaining truncation branch. - README timestamp prose: Option>. - docs/README imports list: add ListIterator (a return type). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- README.md | 2 +- docs/README.md | 3 ++- src/clients/dataset.rs | 10 +++++++-- src/clients/pagination.rs | 45 +++++++++++++++++++++++++++++++++------ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 70839c0..7e7858d 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ project needs `serde_json`. Two more dependencies are needed only for specific f [`raw_log`](examples/raw_log.rs) examples import `futures_util::StreamExt` for this. See [`docs/misc.md`](docs/misc.md#logs--clientlogbuild_or_run_id). - `chrono = "0.4"` — only if you construct or read timestamp values yourself. Model timestamp - fields (e.g. `Actor::created_at`, `ActorRun::started_at`) are typed as `chrono::DateTime` + fields (e.g. `Actor::created_at`, `ActorRun::started_at`) are typed as `Option>` and `chrono` is **not** re-exported, so snippets that call `chrono::Utc::now()` (e.g. the `monthly_usage` example in [`docs/misc.md`](docs/misc.md#users--clientme--clientuserid) and the account example) need it as a direct dependency. diff --git a/docs/README.md b/docs/README.md index 2352632..c4e8552 100644 --- a/docs/README.md +++ b/docs/README.md @@ -85,7 +85,8 @@ types is: - Shared: `ListOptions`, `StorageListOptions` - Client configuration: `RequestCompression` -plus the common container `PaginationList` and the query helper `QueryParams`. Import any of them +plus the common container `PaginationList`, the query helper `QueryParams`, and `ListIterator` +(the return type of every collection client's `iterate()` method). Import any of them directly from `apify_client`: ```rust,no_run diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 5239ed7..4c6a740 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -239,8 +239,14 @@ impl DatasetClient { /// `listItems`/`iterateItems`: yields one deserialized item of type `T` at a time, /// transparently paging. The caller's `options.limit` caps the total number of items yielded /// (unset = all); use [`ListIterator::with_chunk_size`] to control the per-page fetch size. - /// Filtering options such as `skip_empty`/`clean`/`skip_hidden` are honoured across pages - /// without truncating the result. + /// + /// Server-side filters (`skip_empty`/`clean`/`skip_hidden`) are forwarded on every page + /// request. Paging advances the offset by the number of items each page returns, exactly + /// like the reference JavaScript client (`currentOffset += items.length`). When a filter + /// drops items from a page, that post-filter count is smaller than the raw window, so the + /// next page starts at an offset that overlaps the previous window and some items may be + /// yielded more than once. If you need every filtered item exactly once, apply the filter + /// client-side over an unfiltered iteration instead. pub fn iterate_items( &self, options: DatasetListItemsOptions, diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index ec22bf1..3aae578 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -293,12 +293,13 @@ mod tests { #[tokio::test] async fn non_final_short_page_with_reported_total_does_not_truncate() { - // Regression test for the dataset-items filtering case (Finding 1): every page is "short" - // — it returns fewer items than the page size the API reports (as `skip_empty`/`clean` do, - // where a full raw window omits filtered-out items) — while the endpoint reports a large - // total. The old `received < page.limit` termination stopped after page 1 and silently - // dropped the rest; total-driven termination must keep going and yield every item, ending - // only on the empty page. + // Guards the termination logic, NOT filter de-duplication: every page is "short" — it + // returns fewer items than the page size the API reports — while the endpoint reports a + // large total. The old `received < page.limit` termination stopped after page 1 and + // silently dropped the rest; total-driven termination must keep going and yield every + // item, ending only on the empty page. (This fetcher advances offset by the page size + // and never overlaps windows, so it does not model the sparse-window duplicate behaviour + // documented on `iterate_items`; it only proves a short page is not treated as terminal.) let all: Vec = (0..6).collect(); let calls = Arc::new(AtomicUsize::new(0)); let counter = calls.clone(); @@ -360,6 +361,38 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 3); } + #[tokio::test] + async fn cap_truncates_page_that_exceeds_remaining_budget() { + // Exercises the cap-truncation branch (`received > rem`): the endpoint ignores the + // requested page limit and returns MORE items than the caller's total cap allows. The + // iterator must trim the over-long page to the remaining budget and yield exactly `limit` + // items, fetching only once. + let calls = Arc::new(AtomicUsize::new(0)); + let counter = calls.clone(); + let fetch: PageFetcher = Box::new(move |offset, _limit| { + counter.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + // Always return 5 items regardless of the requested limit. + Ok(PaginationList { + total: 100, + offset, + limit: 5, + count: 5, + desc: false, + items: vec![0, 1, 2, 3, 4], + }) + }) + }); + // Total cap of 3, but the page delivers 5 → must be truncated to [0, 1, 2]. + let iter = ListIterator::new(0, Some(3), fetch); + assert_eq!(iter.collect_all().await.unwrap(), vec![0, 1, 2]); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "cap reached on the first (over-long) page, so no second fetch" + ); + } + #[tokio::test] async fn honours_caller_start_offset() { let calls = Arc::new(AtomicUsize::new(0)); From d61a7fe36205402693ef658ad5ea1b64c8ac8b31 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 15:28:38 +0000 Subject: [PATCH 08/20] docs: clarify iterate() limit is total cap; harden missing-total header; extract iterate boilerplate - Document on all 11 collection iterate() methods that options.limit is a total-item cap (page size via with_chunk_size), pointing to ListIterator. - list_items reports total=0 (not count) when the pagination-total header is absent, so iterate_items no longer falsely terminates after page 1; add a hermetic header-less test in unit_http.rs. - Remove orphaned historical narration from a pagination test comment. - Extract the duplicated iterate() closure into a crate-internal list_iterator! macro used by the 10 uniform collections (run/dataset-items/env-var keep their explicit bodies). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/clients/actor_collection.rs | 22 ++++-------- src/clients/actor_version_collection.rs | 22 ++++-------- src/clients/build_collection.rs | 22 ++++-------- src/clients/dataset.rs | 8 ++++- src/clients/dataset_collection.rs | 22 ++++-------- src/clients/key_value_store_collection.rs | 22 ++++-------- src/clients/pagination.rs | 42 ++++++++++++++++++---- src/clients/request_queue_collection.rs | 22 ++++-------- src/clients/run_collection.rs | 5 +++ src/clients/schedule_collection.rs | 22 ++++-------- src/clients/store_collection.rs | 22 ++++-------- src/clients/webhook_collection.rs | 22 ++++-------- src/clients/webhook_dispatch_collection.rs | 22 ++++-------- tests/unit_http.rs | 37 +++++++++++++++++++ 14 files changed, 155 insertions(+), 157 deletions(-) diff --git a/src/clients/actor_collection.rs b/src/clients/actor_collection.rs index 67621fa..12c3fc7 100644 --- a/src/clients/actor_collection.rs +++ b/src/clients/actor_collection.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -58,21 +58,13 @@ impl ActorCollectionClient { /// /// Returns a [`ListIterator`] whose `next()` yields one Actor at a time, transparently /// fetching subsequent pages until the listing is exhausted. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ActorListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Creates a new Actor with the given definition. diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index 47f7b95..b7e2b9f 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -36,21 +36,13 @@ impl ActorVersionCollectionClient { } /// Lazily iterates over all versions matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Creates a new Actor version. diff --git a/src/clients/build_collection.rs b/src/clients/build_collection.rs index 53fa245..e458ade 100644 --- a/src/clients/build_collection.rs +++ b/src/clients/build_collection.rs @@ -1,7 +1,7 @@ //! Client for an Actor-build collection (`/v2/actor-builds`, `/v2/actors/{id}/builds`). use crate::clients::base::{list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -38,20 +38,12 @@ impl BuildCollectionClient { } /// Lazily iterates over all builds matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } } diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 4c6a740..14b984f 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -210,10 +210,16 @@ impl DatasetClient { let items: Vec = serde_json::from_slice(&response.body)?; let count = items.len() as i64; + // When the endpoint omits the total header, report `0` ("total unknown"), not the current + // page's own item count. A fallback of `count` would look like a genuine total equal to + // the number of items already returned, which [`ListIterator`] reads as "listing complete" + // and would stop after the first page, silently dropping later items. `0` instead routes + // iteration to the short-page / empty-page backstop, which walks every page. The live + // dataset-items endpoint always sends this header, so this only guards a degenerate case. let total = response .header("x-apify-pagination-total") .and_then(|v| v.parse().ok()) - .unwrap_or(count); + .unwrap_or(0); let offset = response .header("x-apify-pagination-offset") .and_then(|v| v.parse().ok()) diff --git a/src/clients/dataset_collection.rs b/src/clients/dataset_collection.rs index 170e092..a2399e3 100644 --- a/src/clients/dataset_collection.rs +++ b/src/clients/dataset_collection.rs @@ -1,7 +1,7 @@ //! Client for the dataset collection (`/v2/datasets`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -31,21 +31,13 @@ impl DatasetCollectionClient { } /// Lazily iterates over all datasets matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: StorageListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Gets the dataset with the given `name`, creating it if it does not exist. diff --git a/src/clients/key_value_store_collection.rs b/src/clients/key_value_store_collection.rs index 728d04d..65a0803 100644 --- a/src/clients/key_value_store_collection.rs +++ b/src/clients/key_value_store_collection.rs @@ -1,7 +1,7 @@ //! Client for the key-value store collection (`/v2/key-value-stores`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -32,21 +32,13 @@ impl KeyValueStoreCollectionClient { } /// Lazily iterates over all key-value stores matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: StorageListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Gets the store with the given `name`, creating it if it does not exist. diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 3aae578..7ee2f73 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -24,6 +24,37 @@ type PageFuture = Pin = Box) -> PageFuture + Send + Sync>; +/// Generates the body of a collection client's `iterate()` method for the common shape: an +/// options struct with `offset`/`limit` fields plus a `list(options)` method. It clones the +/// client, reads the caller's start offset and total-item cap from the options, and builds a +/// [`ListIterator`] whose per-page fetcher overrides only `offset`/`limit` before calling +/// `list`. This removes the ~15 lines of identical closure boilerplate that would otherwise be +/// copied into every collection client (the don't-repeat-yourself goal of this module). +/// +/// Collections whose listing does not fit this shape build their iterator directly instead: +/// the run listing takes a separate `filter` argument, dataset items use `list_items::`, and +/// an Actor's env vars are non-paginated ([`ListIterator::new_single_page`]). +macro_rules! list_iterator { + ($self:expr, $options:expr, $list:ident) => {{ + let client = $self.clone(); + let options = $options; + let start = options.offset.unwrap_or(0); + let total_limit = options.limit; + $crate::clients::pagination::ListIterator::new( + start, + total_limit, + Box::new(move |offset, page_limit| { + let client = client.clone(); + let mut options = options.clone(); + options.offset = Some(offset); + options.limit = page_limit; + Box::pin(async move { client.$list(options).await }) + }), + ) + }}; +} +pub(crate) use list_iterator; + /// Returns the smaller of two optional positive limits, treating a non-positive value as "no /// limit" (`None`). Mirrors the reference client's `minForLimitParam`, where the API treats `0` /// as an absent limit. @@ -293,12 +324,11 @@ mod tests { #[tokio::test] async fn non_final_short_page_with_reported_total_does_not_truncate() { - // Guards the termination logic, NOT filter de-duplication: every page is "short" — it - // returns fewer items than the page size the API reports — while the endpoint reports a - // large total. The old `received < page.limit` termination stopped after page 1 and - // silently dropped the rest; total-driven termination must keep going and yield every - // item, ending only on the empty page. (This fetcher advances offset by the page size - // and never overlaps windows, so it does not model the sparse-window duplicate behaviour + // Guards the termination logic, NOT filter de-duplication: every page is "short" (it + // returns fewer items than the page size the API reports) while the endpoint reports a + // large total. Total-driven termination must keep going and yield every item, ending + // only on the empty page. (This fetcher advances offset by the page size and never + // overlaps windows, so it does not model the sparse-window duplicate behaviour // documented on `iterate_items`; it only proves a short page is not treated as terminal.) let all: Vec = (0..6).collect(); let calls = Arc::new(AtomicUsize::new(0)); diff --git a/src/clients/request_queue_collection.rs b/src/clients/request_queue_collection.rs index bb0245e..360faa5 100644 --- a/src/clients/request_queue_collection.rs +++ b/src/clients/request_queue_collection.rs @@ -1,7 +1,7 @@ //! Client for the request queue collection (`/v2/request-queues`). use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -32,21 +32,13 @@ impl RequestQueueCollectionClient { } /// Lazily iterates over all request queues matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: StorageListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Gets the queue with the given `name`, creating it if it does not exist. diff --git a/src/clients/run_collection.rs b/src/clients/run_collection.rs index 545fdec..fccc0ec 100644 --- a/src/clients/run_collection.rs +++ b/src/clients/run_collection.rs @@ -83,6 +83,11 @@ impl RunCollectionClient { /// /// The idiomatic-Rust counterpart of the reference client's async-iterable run listing; /// yields one [`ActorRun`] at a time across all pages. + /// + /// `options.limit` caps the *total* number of runs yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions, filter: RunListOptions) -> ListIterator { let client = self.clone(); let start = options.offset.unwrap_or(0); diff --git a/src/clients/schedule_collection.rs b/src/clients/schedule_collection.rs index 31a2577..722c12a 100644 --- a/src/clients/schedule_collection.rs +++ b/src/clients/schedule_collection.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -33,21 +33,13 @@ impl ScheduleCollectionClient { } /// Lazily iterates over all schedules matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Creates a new schedule from the given definition. diff --git a/src/clients/store_collection.rs b/src/clients/store_collection.rs index f28c3da..dd5df29 100644 --- a/src/clients/store_collection.rs +++ b/src/clients/store_collection.rs @@ -1,7 +1,7 @@ //! Client for browsing the Apify Store (`/v2/store`). use crate::clients::base::{list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -68,21 +68,13 @@ impl StoreCollectionClient { /// /// Returns a [`StoreActorIterator`] whose `next()` method yields one Actor at a time, /// transparently fetching subsequent pages. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: StoreListOptions) -> StoreActorIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } fn build_params(&self, options: &StoreListOptions) -> QueryParams { diff --git a/src/clients/webhook_collection.rs b/src/clients/webhook_collection.rs index d2b13af..b4e4362 100644 --- a/src/clients/webhook_collection.rs +++ b/src/clients/webhook_collection.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -40,21 +40,13 @@ impl WebhookCollectionClient { } /// Lazily iterates over all webhooks matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Creates a new webhook from the given definition. diff --git a/src/clients/webhook_dispatch_collection.rs b/src/clients/webhook_dispatch_collection.rs index c5f4da2..2ac21ab 100644 --- a/src/clients/webhook_dispatch_collection.rs +++ b/src/clients/webhook_dispatch_collection.rs @@ -1,7 +1,7 @@ //! Client for the webhook dispatch collection (`/v2/webhook-dispatches`). use crate::clients::base::{list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -41,20 +41,12 @@ impl WebhookDispatchCollectionClient { } /// Lazily iterates over all webhook dispatches matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } } diff --git a/tests/unit_http.rs b/tests/unit_http.rs index ea055c5..2dc1205 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -520,3 +520,40 @@ async fn last_run_sends_status_and_origin_query_params() { "default last_run must not send status/origin params, got {url}" ); } + +/// `iterate_items` must keep paging even when the dataset-items response omits the +/// `X-Apify-Pagination-Total` header. With the header absent, `list_items` reports `total = 0` +/// ("unknown"), so the iterator falls through to the short-page/empty-page backstop and walks +/// every page. The `MockBackend` returns no headers, reproducing the missing-total case. Before +/// the fix, `list_items` fell back to `total = count`, which the iterator read as a completed +/// total after the first full page (`next_offset == total`) and silently dropped every later +/// item — this test scripts three data pages then an empty one and asserts all items are yielded. +#[tokio::test] +async fn iterate_items_without_total_header_walks_all_pages() { + let backend = MockBackend::new(vec![ + MockOutcome::Status(200, br#"[{"i":0},{"i":1}]"#.to_vec()), + MockOutcome::Status(200, br#"[{"i":2},{"i":3}]"#.to_vec()), + MockOutcome::Status(200, br#"[{"i":4}]"#.to_vec()), + MockOutcome::Status(200, b"[]".to_vec()), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .dataset("some-dataset-id") + .iterate_items::(Default::default()); + let mut seen = Vec::new(); + while let Some(item) = it.next().await.expect("ok") { + seen.push(item["i"].as_i64().expect("i field")); + } + + assert_eq!( + seen, + vec![0, 1, 2, 3, 4], + "all items must be yielded even without a total header" + ); + assert_eq!( + backend.call_count(), + 4, + "three data pages plus the terminating empty page" + ); +} From 0edc5853aaa598e1279298b5058e5386b8098a96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 15:34:06 +0000 Subject: [PATCH 09/20] docs: add iterate() total-cap note and list_iterator! macro to task_collection Completes the previous commit: task_collection was the 12th option-taking collection and was missed by both the doc note (item 1) and the macro extraction (item 4). All 11 uniform collections now use list_iterator!. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/clients/task_collection.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/clients/task_collection.rs b/src/clients/task_collection.rs index b22eef4..796b460 100644 --- a/src/clients/task_collection.rs +++ b/src/clients/task_collection.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{create_resource, list_resource, ResourceContext}; -use crate::clients::pagination::ListIterator; +use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -33,21 +33,13 @@ impl TaskCollectionClient { } /// Lazily iterates over all tasks matching `options`, fetching pages on demand. + /// + /// `options.limit` caps the *total* number of items yielded across all pages, unlike + /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size + /// with [`with_chunk_size`](crate::ListIterator::with_chunk_size); see + /// [`ListIterator`] for details. pub fn iterate(&self, options: ListOptions) -> ListIterator { - let client = self.clone(); - let start = options.offset.unwrap_or(0); - let total_limit = options.limit; - ListIterator::new( - start, - total_limit, - Box::new(move |offset, page_limit| { - let client = client.clone(); - let mut options = options.clone(); - options.offset = Some(offset); - options.limit = page_limit; - Box::pin(async move { client.list(options).await }) - }), - ) + list_iterator!(self, options, list) } /// Creates a new task from the given definition. From df0287524f6f1405c61ad0d697dd7b217f2dc822 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 16:23:11 +0000 Subject: [PATCH 10/20] feat: add KVS iterate_keys, hermetic iterator tests, reconcile iterator docs Adds KeyValueStoreClient::iterate_keys with a cursor-based KeyValueStoreKeysIterator (exclusiveStartKey/nextExclusiveStartKey), matching the JS reference listKeys() async-iterable. Adds an integration test and hermetic MockBackend coverage for the list_iterator! macro wiring, RunCollectionClient::iterate, and the new key iterator. Reconciles the StoreActorIterator/ListIterator docs, adds iterate()/iterate_items()/iterate_keys() to the per-resource tables, documents with_chunk_size on the Store page, adds a large-cap first-page doc note, and clarifies StoreListOptions.limit. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 5 + docs/README.md | 16 +-- docs/actors.md | 1 + docs/builds.md | 1 + docs/misc.md | 24 +++- docs/runs.md | 1 + docs/schedules.md | 1 + docs/storages.md | 31 +++++- docs/tasks.md | 1 + docs/webhooks.md | 4 +- src/clients/key_value_store.rs | 118 +++++++++++++++++++- src/clients/pagination.rs | 7 ++ src/clients/store_collection.rs | 5 +- src/lib.rs | 2 +- tests/key_value_store.rs | 56 ++++++++++ tests/unit_http.rs | 187 ++++++++++++++++++++++++++++++++ 16 files changed, 444 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc9e74f..5071884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ to [Semantic Versioning](https://semver.org/). counterpart to the reference client's async-iterable list results. The options' `limit` caps the total number of items yielded (matching the reference client), and `ListIterator::with_chunk_size` sets the per-request page size. +- `KeyValueStoreClient::iterate_keys()`, returning a cursor-based `KeyValueStoreKeysIterator` + that auto-paginates a store's keys via `exclusiveStartKey`/`nextExclusiveStartKey` (the + auto-paginating counterpart to `list_keys`, matching the reference client's `listKeys()` + async-iterable). +- Re-exported `StoreActorIterator` at the crate root. ### Changed - Bumped `API_SPEC_VERSION` to `v2-2026-07-10T105921Z`. The spec delta (added `401`/`402` diff --git a/docs/README.md b/docs/README.md index c4e8552..b848921 100644 --- a/docs/README.md +++ b/docs/README.md @@ -85,9 +85,10 @@ types is: - Shared: `ListOptions`, `StorageListOptions` - Client configuration: `RequestCompression` -plus the common container `PaginationList`, the query helper `QueryParams`, and `ListIterator` -(the return type of every collection client's `iterate()` method). Import any of them -directly from `apify_client`: +plus the common container `PaginationList`, the query helper `QueryParams`, `ListIterator` +(the return type of every collection client's `iterate()` method), and `StoreActorIterator` +(a type alias for `ListIterator`, the return type of +`StoreCollectionClient::iterate`). Import any of them directly from `apify_client`: ```rust,no_run use apify_client::{ApifyClient, ActorListOptions, StoreListOptions, DownloadItemsFormat}; @@ -188,10 +189,11 @@ across all pages without tracking offsets yourself, call `iterate(...)` instead: lazy `ListIterator` (re-exported at the crate root) that fetches the next page from the API on demand as you consume items. Every collection client provides it (`actors`, `builds`, `runs`, `tasks`, `datasets`, `key_value_stores`, `request_queues`, `schedules`, `webhooks`, -`webhook_dispatches`, `store`, and the nested Actor `versions`/`env_vars`), and `DatasetClient` -exposes `iterate_items()` for dataset items. The options' `limit` caps the total number of items -yielded (unset iterates everything); to control the per-request page size, call -`.with_chunk_size(n)` on the returned iterator. +`webhook_dispatches`, `store`, and the nested Actor `versions`/`env_vars`). `DatasetClient` +exposes `iterate_items()` for dataset items, and `KeyValueStoreClient` exposes `iterate_keys()` +for store keys (cursor-based). The options' `limit` caps the total number of items yielded (unset +iterates everything); to control the per-request page size, call `.with_chunk_size(n)` on the +returned iterator (offset-paginated iterators only). ```rust,no_run use apify_client::{ApifyClient, ActorListOptions}; diff --git a/docs/actors.md b/docs/actors.md index 6c82a17..7e63354 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -8,6 +8,7 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options)` | `ActorListOptions { offset, limit, desc, my, sort_by }` | `PaginationList` | Lists your Actors. | +| `iterate(options)` | `ActorListOptions` | `ListIterator` | Lazily iterates all Actors across pages (auto-pagination). | | `create(actor)` | `&impl Serialize` | `Actor` | Creates an Actor from a definition. | ## `ActorClient` diff --git a/docs/builds.md b/docs/builds.md index e18dacb..e6b1179 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -8,6 +8,7 @@ collections are available via `actor.builds()`. | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options)` | `ListOptions` | `PaginationList` | Lists builds. | +| `iterate(options)` | `ListOptions` | `ListIterator` | Lazily iterates all builds across pages (auto-pagination). | ## `BuildClient` diff --git a/docs/misc.md b/docs/misc.md index cb64423..6ca5ab9 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -10,9 +10,12 @@ | `iterate(options)` | `StoreListOptions` | `StoreActorIterator` | Lazy, page-fetching iterator. | `StoreListOptions`: `offset`, `limit`, `search`, `sort_by`, `category`, `username`, -`pricing_model`. +`pricing_model`. `limit` means a single page's size for `list`, but a cap on the *total* number of +items yielded for `iterate` (see below). -`StoreActorIterator::next()` is `async` and fallible — it returns +`StoreActorIterator` is a type alias for `ListIterator` (the shared iterator +returned by every collection's `iterate`), re-exported at the crate root alongside `ListIterator` +itself. Its `next()` is `async` and fallible — it returns `ApifyClientResult>` (i.e. `Result, ApifyClientError>`), fetching the next page on demand and yielding `Ok(None)` once the listing is exhausted. Drive it with `.await?`: @@ -29,6 +32,23 @@ while let Some(actor) = iter.next().await? { # } ``` +`options.limit` caps the total number of Actors the iterator yields (unset iterates the whole +Store). The per-request page size is separate: call `.with_chunk_size(n)` on the returned +`StoreActorIterator` to fetch `n` Actors per API call (when unset, the API's default page size is +used). If you set a large `limit` cap, also set `with_chunk_size` so the first request does not ask +for the entire cap at once — for example, `client.store().iterate(opts).with_chunk_size(50)`: + +```rust,no_run +# use apify_client::{ApifyClient, StoreListOptions}; +# async fn run(client: ApifyClient) -> Result<(), Box> { +let mut iter = client.store().iterate(StoreListOptions::default()).with_chunk_size(50); +while let Some(actor) = iter.next().await? { + println!("{}", actor.id); +} +# Ok(()) +# } +``` + `ActorStoreListItem` (from `apify_client::models`) is the element type yielded by both `list` and the iterator. Its fields: diff --git a/docs/runs.md b/docs/runs.md index eef0113..27a3895 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -8,6 +8,7 @@ collections are available via `actor.runs()` and `task.runs()`. | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options, filter)` | `ListOptions`, `RunListOptions { status, started_after, started_before }` | `PaginationList` | Lists runs, optionally filtered by status and start time. | +| `iterate(options, filter)` | `ListOptions`, `RunListOptions` | `ListIterator` | Lazily iterates all runs across pages (auto-pagination). | ## `RunClient` diff --git a/docs/schedules.md b/docs/schedules.md index dd18d67..326eda3 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -7,6 +7,7 @@ Obtained via `client.schedules()` (collection) and `client.schedule(id)` (single | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options)` | `ListOptions` | `PaginationList` | Lists schedules. | +| `iterate(options)` | `ListOptions` | `ListIterator` | Lazily iterates all schedules across pages (auto-pagination). | | `create(schedule)` | `&impl Serialize` | `Schedule` | Creates a schedule. | ## `ScheduleClient` diff --git a/docs/storages.md b/docs/storages.md index c54e31e..0acae62 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -33,6 +33,7 @@ let dataset_client = client.dataset(&dataset.id); ## Datasets — `client.datasets()` / `client.dataset(id)` `DatasetCollectionClient`: `list(options: StorageListOptions)`, +`iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), `get_or_create(name: Option<&str>)`. `StorageListOptions`: `offset`, `limit`, `desc`, `unnamed`, `ownership`. @@ -60,6 +61,7 @@ let scratch = client.datasets().get_or_create(None).await?; | `update(fields)` | `&impl Serialize` | `Dataset` | Updates metadata. | | `delete()` | — | `()` | Deletes the dataset. | | `list_items::(options)` | `DatasetListItemsOptions` | `PaginationList` | Reads items (pagination via response headers). | +| `iterate_items::(options)` | `DatasetListItemsOptions` | `ListIterator` | Lazily iterates all items across pages (auto-pagination). | | `push_items(items)` | `&impl Serialize` | `()` | Appends items (object or array). | | `get_statistics()` | — | `Option` | Field statistics. | | `download_items(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Vec` | Export items as JSON/CSV/XLSX/XML/RSS/HTML. | @@ -111,7 +113,9 @@ println!("exported {} bytes of CSV", csv.len()); ## Key-value stores — `client.key_value_stores()` / `client.key_value_store(id)` -`KeyValueStoreCollectionClient`: `list(options: StorageListOptions)`, `get_or_create(name: Option<&str>)`. +`KeyValueStoreCollectionClient`: `list(options: StorageListOptions)`, +`iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), +`get_or_create(name: Option<&str>)`. `KeyValueStoreClient`: @@ -120,7 +124,8 @@ println!("exported {} bytes of CSV", csv.len()); | `get()` | — | `Option` | Store metadata. | | `update(fields)` | `&impl Serialize` | `KeyValueStore` | Updates metadata. | | `delete()` | — | `()` | Deletes the store. | -| `list_keys(options)` | `ListKeysOptions` | `KeyValueStoreKeysPage` | Lists keys (key-based pagination). | +| `list_keys(options)` | `ListKeysOptions` | `KeyValueStoreKeysPage` | Lists one page of keys (key-based pagination). | +| `iterate_keys(options)` | `ListKeysOptions` | `KeyValueStoreKeysIterator` | Lazily iterates all keys across pages (cursor-based auto-pagination). | | `get_records(options)` | `GetRecordsOptions { collection, prefix, signature }` | `Vec` | Downloads all records as a ZIP archive (raw bytes). | | `record_exists(key)` | `&str` | `bool` | Whether a record exists (HEAD). | | `get_record(key)` | `&str` | `Option` | Reads a record's raw value. | @@ -135,9 +140,29 @@ println!("exported {} bytes of CSV", csv.len()); `KeyValueStoreRecord` exposes `value: Vec`, `content_type`, plus `as_text()` and `json::()` helpers. +`iterate_keys(options)` returns a `KeyValueStoreKeysIterator` — the auto-paginating counterpart to +`list_keys` (which returns a single page). Key-value stores use cursor-based pagination, so the +iterator threads the `nextExclusiveStartKey` cursor through for you. Its `next()` is `async` and +fallible, returning `ApifyClientResult>` and yielding `Ok(None)` once the +store is exhausted. `options.limit` caps the total number of keys yielded (unset iterates the whole +store); `prefix`/`collection`/`signature` filter every page: + +```rust,no_run +# use apify_client::{ApifyClient, ListKeysOptions}; +# async fn run(client: ApifyClient, store_id: &str) -> Result<(), Box> { +let mut keys = client.key_value_store(store_id).iterate_keys(ListKeysOptions::default()); +while let Some(key) = keys.next().await? { + println!("{} ({:?} bytes)", key.key, key.size); +} +# Ok(()) +# } +``` + ## Request queues — `client.request_queues()` / `client.request_queue(id)` -`RequestQueueCollectionClient`: `list(options: StorageListOptions)`, `get_or_create(name: Option<&str>)`. +`RequestQueueCollectionClient`: `list(options: StorageListOptions)`, +`iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), +`get_or_create(name: Option<&str>)`. `RequestQueueClient` (chainable `with_client_key(key)` for lock coordination): diff --git a/docs/tasks.md b/docs/tasks.md index 384ed2c..775a155 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -7,6 +7,7 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options)` | `ListOptions` | `PaginationList` | Lists tasks. | +| `iterate(options)` | `ListOptions` | `ListIterator` | Lazily iterates all tasks across pages (auto-pagination). | | `create(task)` | `&impl Serialize` | `Task` | Creates a task. | ## `TaskClient` diff --git a/docs/webhooks.md b/docs/webhooks.md index 0f68cdd..13b8ec3 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -9,6 +9,7 @@ webhook collections are available via `actor.webhooks()` and `task.webhooks()`. | Method | Arguments | Returns | Description | |---|---|---|---| | `list(options)` | `ListOptions` | `PaginationList` | Lists webhooks. | +| `iterate(options)` | `ListOptions` | `ListIterator` | Lazily iterates all webhooks across pages (auto-pagination). | | `create(webhook)` | `&impl Serialize` | `Webhook` | Creates a webhook. | ## `WebhookClient` @@ -23,7 +24,8 @@ webhook collections are available via `actor.webhooks()` and `task.webhooks()`. ## Webhook dispatches -`WebhookDispatchCollectionClient`: `list(options)`. +`WebhookDispatchCollectionClient`: `list(options)`, `iterate(options)` (lazy +`ListIterator` auto-pagination). `WebhookDispatchClient`: `get()`. ## The `Webhook` model diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 061c737..e1e1d03 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -1,5 +1,7 @@ //! Client for a single key-value store (`/v2/key-value-stores/{storeId}` and variants). +use std::collections::VecDeque; + use serde::Serialize; use crate::clients::base::{ @@ -11,7 +13,7 @@ use crate::common::{ }; use crate::error::ApifyClientResult; use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; -use crate::models::{KeyValueStore, KeyValueStoreKeysPage, KeyValueStoreRecord}; +use crate::models::{KeyValueStore, KeyValueStoreKey, KeyValueStoreKeysPage, KeyValueStoreRecord}; /// Options for listing keys in a key-value store. #[derive(Debug, Default, Clone)] @@ -111,6 +113,36 @@ impl KeyValueStoreClient { get_resource_required(&self.ctx, Some("keys"), ¶ms).await } + /// Lazily iterates over every key in the store, fetching pages on demand. + /// + /// Returns a [`KeyValueStoreKeysIterator`]; call its `next()` to get one key at a time, + /// transparently fetching the following page once the local buffer drains, until the store + /// is exhausted. This is the auto-paginating counterpart to the single-page + /// [`list_keys`](Self::list_keys), matching the reference client's `listKeys()` + /// `AsyncIterable`. + /// + /// Key-value stores use cursor-based (not offset) pagination: each page is anchored by the + /// previous page's `nextExclusiveStartKey`, so the iterator threads that cursor through + /// automatically. The `prefix`, `collection` and `signature` filters from `options` are + /// carried into every page. + /// + /// `options.limit` caps the *total* number of keys yielded across all pages (unset/`0` + /// iterates the entire store); it also bounds each page's request size, mirroring the + /// reference client. `options.exclusive_start_key`, when set, resumes iteration after that + /// key. + pub fn iterate_keys(&self, options: ListKeysOptions) -> KeyValueStoreKeysIterator { + let remaining = options.limit.filter(|&l| l > 0); + KeyValueStoreKeysIterator { + client: self.clone(), + options, + remaining, + next_exclusive_start_key: None, + buffer: VecDeque::new(), + first_page: true, + exhausted: false, + } + } + /// Downloads all records from the store as a ZIP archive (raw bytes). /// /// Each record is stored as a separate file in the archive, with the filename equal to the @@ -285,3 +317,87 @@ impl KeyValueStoreClient { Ok(()) } } + +/// A lazy, page-fetching async iterator over the keys in a key-value store. +/// +/// Created by [`KeyValueStoreClient::iterate_keys`]. Each call to [`next`](Self::next) returns +/// the next key, fetching another page from the API when the local buffer is exhausted, until +/// every key has been yielded (or the caller's total-key cap is reached). +/// +/// Unlike the offset/limit-paginated [`ListIterator`](crate::ListIterator), key-value stores use +/// cursor-based pagination: each page is anchored by the previous page's +/// `nextExclusiveStartKey`. Termination mirrors the reference client's `listKeys()` generator — +/// the walk stops once a page comes back empty, the API stops returning a next cursor, or the +/// caller's `limit` is exhausted. +pub struct KeyValueStoreKeysIterator { + client: KeyValueStoreClient, + /// Base listing options. The `prefix`/`collection`/`signature` filters are carried into every + /// page unchanged; `limit` and `exclusive_start_key` are overridden per page after the first. + options: ListKeysOptions, + /// Keys still allowed under the caller's total cap (`options.limit`); `None` = uncapped. + /// Decremented by each page's key count and passed as the next page's request `limit`, + /// matching the reference client. + remaining: Option, + /// Cursor for the next page: the previous page's `next_exclusive_start_key`. + next_exclusive_start_key: Option, + buffer: VecDeque, + /// `true` until the first page has been fetched. The first page honours the caller's + /// `exclusive_start_key`/`limit` verbatim; later pages are driven by the cursor and remaining + /// budget. + first_page: bool, + exhausted: bool, +} + +impl KeyValueStoreKeysIterator { + /// Returns the next key, or `None` when the store is exhausted (or the caller's `limit` is + /// reached). Fetches another page from the API when the local buffer is empty. + pub async fn next(&mut self) -> ApifyClientResult> { + if let Some(item) = self.buffer.pop_front() { + return Ok(Some(item)); + } + if self.exhausted { + return Ok(None); + } + + // Build this page's options. The first page uses the caller's options verbatim; later + // pages advance the cursor and request only the remaining budget (reference parity). + let mut page_options = self.options.clone(); + if !self.first_page { + page_options.exclusive_start_key = self.next_exclusive_start_key.clone(); + page_options.limit = self.remaining; + } + self.first_page = false; + + let page = self.client.list_keys(page_options).await?; + let received = page.items.len() as i64; + + if let Some(rem) = self.remaining.as_mut() { + *rem -= received; + } + self.next_exclusive_start_key = page.next_exclusive_start_key; + + // Stop when the page is empty, the API returns no further cursor, or the caller's cap is + // reached — the same three termination conditions as the reference `listKeys()` loop. + if received == 0 + || self.next_exclusive_start_key.is_none() + || matches!(self.remaining, Some(r) if r <= 0) + { + self.exhausted = true; + } + + self.buffer.extend(page.items); + Ok(self.buffer.pop_front()) + } + + /// Eagerly drains the iterator into a single `Vec`, fetching every remaining page. + /// + /// Convenience for callers that want all keys at once; prefer [`next`](Self::next) to process + /// keys as they stream in without buffering the whole set. + pub async fn collect_all(mut self) -> ApifyClientResult> { + let mut out = Vec::new(); + while let Some(item) = self.next().await? { + out.push(item); + } + Ok(out) + } +} diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 7ee2f73..3a3c1e8 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -85,6 +85,13 @@ fn min_positive_limit(a: Option, b: Option) -> Option { /// 10 items, and `iterate(opts).with_chunk_size(50)` fetches 50 per request while yielding /// everything. /// +/// **Large caps and the first page.** When a total cap is set but no page size is, the first page +/// requests `limit == cap` (the reference client does the same, via +/// `minForLimitParam(options.limit, options.chunkSize)`). If you set a very large cap — larger +/// than the endpoint's maximum `limit` — also call [`with_chunk_size`](Self::with_chunk_size) with +/// a value at or below that maximum, so the first request stays within the endpoint's accepted +/// range rather than asking for the whole cap up front. +/// /// # Example /// ```no_run /// use apify_client::ApifyClient; diff --git a/src/clients/store_collection.rs b/src/clients/store_collection.rs index dd5df29..bc8e12b 100644 --- a/src/clients/store_collection.rs +++ b/src/clients/store_collection.rs @@ -19,7 +19,10 @@ pub type StoreActorIterator = ListIterator; pub struct StoreListOptions { /// Number of items to skip. pub offset: Option, - /// Maximum number of items to return. + /// Item limit. Its meaning depends on the method: for [`StoreCollectionClient::list`] it is a + /// single page's size (the maximum items that one call returns); for + /// [`StoreCollectionClient::iterate`] it is a cap on the *total* number of items yielded across + /// all pages. See each method's docs. pub limit: Option, /// Full-text search query. pub search: Option, diff --git a/src/lib.rs b/src/lib.rs index 1ea0d24..f9f67d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -74,7 +74,7 @@ pub use clients::run::{ LastRunOptions, RunChargeOptions, RunMetamorphOptions, RunResurrectOptions, }; pub use clients::run_collection::RunListOptions; -pub use clients::store_collection::StoreListOptions; +pub use clients::store_collection::{StoreActorIterator, StoreListOptions}; // Compile-test the code snippets in the README and the external `docs/` pages so every // in-documentation code snippet stays valid and runnable. Pulling each Markdown file in as diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index e341001..2b4a54f 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -73,6 +73,62 @@ async fn iterate_key_value_stores() { ); } +/// Iteration: `iterate_keys` yields every key in the store across the cursor-paginated listing. +/// +/// Creates a store, writes several records, then drives the `KeyValueStoreKeysIterator` to +/// completion and asserts every written key is yielded exactly once. This exercises the +/// cursor-based (`exclusiveStartKey`/`nextExclusiveStartKey`) auto-pagination helper end to end. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_keys_yields_all_keys() { + let client = require_client!(); + let name = common::unique_name("kvs-keys-iter"); + let store = client + .key_value_stores() + .get_or_create(Some(&name)) + .await + .expect("create store"); + + let cleanup_client = client.clone(); + let id = store.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.key_value_store(&id).delete().await; + }); + + let store_client = client.key_value_store(&store.id); + let expected: Vec = (0..5).map(|i| format!("iter-key-{i:02}")).collect(); + for key in &expected { + store_client + .set_record_json(key, &json!({ "n": key })) + .await + .expect("set record"); + } + + // Drive the iterator to completion and collect the yielded keys. + let mut iter = store_client.iterate_keys(Default::default()); + let mut seen = Vec::new(); + while let Some(key) = iter.next().await.expect("key iteration should not error") { + seen.push(key.key); + } + + for key in &expected { + assert!( + seen.contains(key), + "iterate_keys should yield every created key; missing {key}, saw {seen:?}" + ); + } + // No key should appear more than once across the paginated walk. + let mut unique = seen.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + unique.len(), + seen.len(), + "iterate_keys must not yield duplicate keys, saw {seen:?}" + ); + + store_client.delete().await.expect("delete store"); +} + /// Record keys containing characters that are valid for the API (`!`, `'`, `(`, `)`) but /// reserved in a URL path must round-trip correctly, proving the path segment is /// percent-encoded rather than interpolated raw. diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 2dc1205..88dbbd7 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -17,6 +17,8 @@ struct MockBackend { responses: Mutex>, calls: AtomicUsize, last_url: Mutex>, + /// Every request URL in call order, so tests can assert per-page pagination wiring. + urls: Mutex>, last_headers: Mutex>, last_body: Mutex>>, } @@ -33,6 +35,7 @@ impl MockBackend { responses: Mutex::new(responses), calls: AtomicUsize::new(0), last_url: Mutex::new(None), + urls: Mutex::new(Vec::new()), last_headers: Mutex::new(std::collections::HashMap::new()), last_body: Mutex::new(None), }) @@ -46,6 +49,11 @@ impl MockBackend { self.last_url.lock().unwrap().clone() } + /// All request URLs in call order. + fn urls(&self) -> Vec { + self.urls.lock().unwrap().clone() + } + /// Case-insensitive lookup of the last request's header value. fn last_header(&self, name: &str) -> Option { let headers = self.last_headers.lock().unwrap(); @@ -65,6 +73,7 @@ impl HttpBackend for MockBackend { async fn send(&self, request: HttpRequest) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); *self.last_url.lock().unwrap() = Some(request.url.clone()); + self.urls.lock().unwrap().push(request.url.clone()); *self.last_headers.lock().unwrap() = request.headers.clone(); *self.last_body.lock().unwrap() = request.body.clone(); let mut queue = self.responses.lock().unwrap(); @@ -557,3 +566,181 @@ async fn iterate_items_without_total_header_walks_all_pages() { "three data pages plus the terminating empty page" ); } + +/// Hermetic coverage of the `list_iterator!` macro wiring used by every offset/limit collection +/// `iterate()`. A total-item cap plus a smaller `with_chunk_size` must (a) yield exactly the cap +/// and (b) drive the per-page request window correctly: the first page requests +/// `min(remaining, chunk_size)` at `offset=0`, and the second advances `offset` by the items +/// received and requests only the remaining budget. Previously this path was exercised only by +/// `APIFY_TOKEN`-gated integration tests, so it was skipped offline. +#[tokio::test] +async fn iterate_macro_caps_and_pages_correctly() { + let backend = MockBackend::new(vec![ + // Endpoint reports a large total (10) so termination is driven by the cap, not by total. + MockOutcome::Status( + 200, + br#"{"data":{"total":10,"items":[{"id":"a0"},{"id":"a1"}]}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"total":10,"items":[{"id":"a2"}]}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .actors() + .iterate(apify_client::ActorListOptions { + limit: Some(3), + ..Default::default() + }) + .with_chunk_size(2); + let mut ids = Vec::new(); + while let Some(actor) = it.next().await.expect("ok") { + ids.push(actor.id); + } + + assert_eq!(ids, vec!["a0", "a1", "a2"], "cap of 3 must yield exactly 3"); + assert_eq!(backend.call_count(), 2, "two pages suffice under the cap"); + let urls = backend.urls(); + assert!( + urls[0].contains("offset=0") && urls[0].contains("limit=2"), + "first page requests min(remaining=3, chunk=2)=2 at offset 0, got {}", + urls[0] + ); + assert!( + urls[1].contains("offset=2") && urls[1].contains("limit=1"), + "second page advances to offset 2 and requests the remaining budget of 1, got {}", + urls[1] + ); +} + +/// Hermetic coverage of `RunCollectionClient::iterate`, which builds its iterator directly +/// (not via the `list_iterator!` macro) because it threads a separate `filter` argument. The +/// iterator must walk every page using the reported total for termination and must forward the +/// `status` filter on every page request. Offline-only; the integration suite gates this on a +/// live token. +#[tokio::test] +async fn run_collection_iterate_walks_pages_and_forwards_filter() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"total":3,"limit":2,"items":[{"id":"r0"},{"id":"r1"}]}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"total":3,"limit":2,"items":[{"id":"r2"}]}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client.runs().iterate( + Default::default(), + apify_client::RunListOptions { + status: vec!["SUCCEEDED".to_owned()], + ..Default::default() + }, + ); + let mut ids = Vec::new(); + while let Some(run) = it.next().await.expect("ok") { + ids.push(run.id); + } + + assert_eq!( + ids, + vec!["r0", "r1", "r2"], + "all runs across pages must be yielded" + ); + assert_eq!(backend.call_count(), 2, "two pages then total-driven stop"); + for url in backend.urls() { + assert!( + url.contains("status=SUCCEEDED"), + "the status filter must be forwarded on every page, got {url}" + ); + } +} + +/// Hermetic coverage of the cursor-based `KeyValueStoreClient::iterate_keys`. Key-value stores +/// paginate by `exclusiveStartKey`/`nextExclusiveStartKey` (not offset), so this walks two pages +/// and asserts: every key is yielded, the second request carries the previous page's +/// `nextExclusiveStartKey` as `exclusiveStartKey`, and iteration stops once the API returns a +/// null next cursor. +#[tokio::test] +async fn iterate_keys_walks_cursor_pages() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k0"},{"key":"k1"}],"isTruncated":true,"nextExclusiveStartKey":"k1"}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k2"}],"isTruncated":false,"nextExclusiveStartKey":null}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .key_value_store("some-store") + .iterate_keys(Default::default()); + let mut keys = Vec::new(); + while let Some(key) = it.next().await.expect("ok") { + keys.push(key.key); + } + + assert_eq!( + keys, + vec!["k0", "k1", "k2"], + "all keys across pages must be yielded" + ); + assert_eq!(backend.call_count(), 2, "two pages then null-cursor stop"); + let urls = backend.urls(); + assert!( + !urls[0].contains("exclusiveStartKey="), + "first page has no start cursor, got {}", + urls[0] + ); + assert!( + urls[1].contains("exclusiveStartKey=k1"), + "second page must carry the previous nextExclusiveStartKey as the cursor, got {}", + urls[1] + ); +} + +/// The `iterate_keys` `limit` is a total cap that also bounds the first page's request size +/// (reference parity): with `limit=2` the first request asks for `limit=2` and, once two keys +/// are consumed, the walk stops without a second fetch even though the API advertised more keys. +#[tokio::test] +async fn iterate_keys_limit_caps_the_walk() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k0"},{"key":"k1"}],"isTruncated":true,"nextExclusiveStartKey":"k1"}}"#.to_vec(), + )]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .key_value_store("some-store") + .iterate_keys(apify_client::ListKeysOptions { + limit: Some(2), + ..Default::default() + }); + let mut keys = Vec::new(); + while let Some(key) = it.next().await.expect("ok") { + keys.push(key.key); + } + + assert_eq!( + keys, + vec!["k0", "k1"], + "yields exactly the first (capped) page" + ); + assert_eq!( + backend.call_count(), + 1, + "the cap is reached on the first page, so no second fetch despite isTruncated" + ); + assert!( + backend.urls()[0].contains("limit=2"), + "the cap bounds the first page's request size, got {}", + backend.urls()[0] + ); +} From d6809f4932dad6baa9594222b3917e5c44728f3c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 16:44:43 +0000 Subject: [PATCH 11/20] fix: normalize/clamp iterate_keys page limit, re-export cursor iterators, doc polish Derives every iterate_keys page request limit from the remaining budget clamped to a documented KEY_LIST_MAX_LIMIT (1000), so limit=0 means uncapped and a large finite cap paginates instead of sending an out-of-range limit. Adds a defensive cap-truncation guard, drops the unused collect_all, and re-exports KeyValueStoreKeysIterator and RequestQueueRequestsIterator at the crate root. Adds hermetic zero-limit and large-cap tests, a KeyValueStoreKey docs field table, ListKeysOptions.limit dual-meaning callouts, and reworded CHANGELOG to disclose the store().iterate() limit semantic change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 8 +++- docs/README.md | 6 ++- docs/storages.md | 18 +++++++- src/clients/key_value_store.rs | 67 ++++++++++++++++------------ src/lib.rs | 6 ++- tests/key_value_store.rs | 10 +++-- tests/unit_http.rs | 81 ++++++++++++++++++++++++++++++++++ 7 files changed, 157 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5071884..ad97845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,12 @@ to [Semantic Versioning](https://semver.org/). - Bumped `API_SPEC_VERSION` to `v2-2026-07-10T105921Z`. The spec delta (added `401`/`402` error responses and relaxed field nullability/optionality) needs no code change: error responses are handled generically and response models are forward-compatible. -- `StoreCollectionClient::iterate` now uses the shared `ListIterator`; `StoreActorIterator` is - a type alias for `ListIterator` (existing usage is unaffected). +- `StoreCollectionClient::iterate` now uses the shared `ListIterator`, and `StoreActorIterator` + is a type alias for `ListIterator`. As part of this, `store().iterate()`'s + `options.limit` changed from a per-page size (0.5.0) to a cap on the total number of items + yielded, for consistency with the reference client and the other `iterate()` methods; set the + per-page size with `ListIterator::with_chunk_size` instead. The `StoreActorIterator` type alias + itself is unchanged. - Corrected the `src/models.rs` module doc to describe forward-compatibility accurately. - Bumped crate version to `0.6.0`. diff --git a/docs/README.md b/docs/README.md index b848921..0e0e8c6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -86,9 +86,11 @@ types is: - Client configuration: `RequestCompression` plus the common container `PaginationList`, the query helper `QueryParams`, `ListIterator` -(the return type of every collection client's `iterate()` method), and `StoreActorIterator` +(the return type of every collection client's `iterate()` method), `StoreActorIterator` (a type alias for `ListIterator`, the return type of -`StoreCollectionClient::iterate`). Import any of them directly from `apify_client`: +`StoreCollectionClient::iterate`), and the two cursor-based iterators `KeyValueStoreKeysIterator` +(from `KeyValueStoreClient::iterate_keys`) and `RequestQueueRequestsIterator` (from +`RequestQueueClient::paginate_requests`). Import any of them directly from `apify_client`: ```rust,no_run use apify_client::{ApifyClient, ActorListOptions, StoreListOptions, DownloadItemsFormat}; diff --git a/docs/storages.md b/docs/storages.md index 0acae62..3f5a52a 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -136,7 +136,11 @@ println!("exported {} bytes of CSV", csv.len()); | `get_record_public_url(key)` | `&str` | `String` | Shareable (HMAC-signed for private) record URL. | | `create_keys_public_url(expires)` | `Option` | `String` | Shareable keys-list URL. | -`ListKeysOptions`: `limit`, `exclusive_start_key`, `prefix`, `collection`, `signature`. +`ListKeysOptions`: `limit`, `exclusive_start_key`, `prefix`, `collection`, `signature`. Like +`StoreListOptions.limit`, the meaning of `limit` depends on the method: for `list_keys` it is a +single page's size (max keys returned by one call, capped at 1000 by the API); for `iterate_keys` +it is a cap on the *total* number of keys yielded across all pages (unset iterates the whole +store). `KeyValueStoreRecord` exposes `value: Vec`, `content_type`, plus `as_text()` and `json::()` helpers. @@ -145,7 +149,8 @@ println!("exported {} bytes of CSV", csv.len()); iterator threads the `nextExclusiveStartKey` cursor through for you. Its `next()` is `async` and fallible, returning `ApifyClientResult>` and yielding `Ok(None)` once the store is exhausted. `options.limit` caps the total number of keys yielded (unset iterates the whole -store); `prefix`/`collection`/`signature` filter every page: +store); each individual request is bounded to the endpoint's maximum page size (1000), so a larger +cap still paginates. `prefix`/`collection`/`signature` filter every page: ```rust,no_run # use apify_client::{ApifyClient, ListKeysOptions}; @@ -158,6 +163,15 @@ while let Some(key) = keys.next().await? { # } ``` +`KeyValueStoreKey` (from `apify_client::models`) is the element type yielded by the iterator and +listed in `KeyValueStoreKeysPage::items`. Its fields: + +| Field | Type | Description | +|---|---|---| +| `key` | `String` | The record key (always present). | +| `size` | `Option` | Size of the record value in bytes, if reported by the API. | +| `extra` | `Extra` | Any other fields returned by the API. | + ## Request queues — `client.request_queues()` / `client.request_queue(id)` `RequestQueueCollectionClient`: `list(options: StorageListOptions)`, diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index e1e1d03..c1198b7 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -18,7 +18,10 @@ use crate::models::{KeyValueStore, KeyValueStoreKey, KeyValueStoreKeysPage, KeyV /// Options for listing keys in a key-value store. #[derive(Debug, Default, Clone)] pub struct ListKeysOptions { - /// Maximum number of keys to return. + /// Key limit. Its meaning depends on the method: for [`KeyValueStoreClient::list_keys`] it is a + /// single page's size (max keys one call returns, capped at 1000 by the API); for + /// [`KeyValueStoreClient::iterate_keys`] it is a cap on the *total* number of keys yielded + /// across all pages (unset/`0` iterates the whole store). pub limit: Option, /// Start listing after this key (exclusive), for pagination. pub exclusive_start_key: Option, @@ -126,10 +129,11 @@ impl KeyValueStoreClient { /// automatically. The `prefix`, `collection` and `signature` filters from `options` are /// carried into every page. /// - /// `options.limit` caps the *total* number of keys yielded across all pages (unset/`0` - /// iterates the entire store); it also bounds each page's request size, mirroring the - /// reference client. `options.exclusive_start_key`, when set, resumes iteration after that - /// key. + /// `options.limit` caps the *total* number of keys yielded across all pages; leaving it unset + /// (or `0`) iterates the entire store, matching the reference client. It is honoured across as + /// many pages as needed — each individual request is bounded to the endpoint's maximum page + /// size ([`KEY_LIST_MAX_LIMIT`]), so a cap larger than one page still works. + /// `options.exclusive_start_key`, when set, resumes iteration after that key. pub fn iterate_keys(&self, options: ListKeysOptions) -> KeyValueStoreKeysIterator { let remaining = options.limit.filter(|&l| l > 0); KeyValueStoreKeysIterator { @@ -318,6 +322,12 @@ impl KeyValueStoreClient { } } +/// The maximum number of keys the `GET /v2/key-value-stores/{storeId}/keys` endpoint accepts in +/// its `limit` query parameter (per the OpenAPI spec: `minimum: 1, maximum: 1000`). Each page the +/// key iterator requests is bounded to this value so a large total cap still paginates correctly +/// instead of asking the API for an out-of-range `limit`. +pub const KEY_LIST_MAX_LIMIT: i64 = 1000; + /// A lazy, page-fetching async iterator over the keys in a key-value store. /// /// Created by [`KeyValueStoreClient::iterate_keys`]. Each call to [`next`](Self::next) returns @@ -335,15 +345,17 @@ pub struct KeyValueStoreKeysIterator { /// page unchanged; `limit` and `exclusive_start_key` are overridden per page after the first. options: ListKeysOptions, /// Keys still allowed under the caller's total cap (`options.limit`); `None` = uncapped. - /// Decremented by each page's key count and passed as the next page's request `limit`, - /// matching the reference client. + /// Decremented by each page's key count. Each request asks for `min(remaining, + /// KEY_LIST_MAX_LIMIT)` so the cap is honoured across pages without exceeding the endpoint's + /// maximum `limit`. remaining: Option, /// Cursor for the next page: the previous page's `next_exclusive_start_key`. next_exclusive_start_key: Option, buffer: VecDeque, - /// `true` until the first page has been fetched. The first page honours the caller's - /// `exclusive_start_key`/`limit` verbatim; later pages are driven by the cursor and remaining - /// budget. + /// `true` until the first page has been fetched. Only the first page honours the caller's + /// `exclusive_start_key`; later pages are driven by the cursor. The request `limit` is derived + /// from `remaining` on every page (never sent verbatim), so a `limit` of `0`/unset is treated + /// as "no limit" rather than sending an out-of-range `limit=0`. first_page: bool, exhausted: bool, } @@ -359,18 +371,29 @@ impl KeyValueStoreKeysIterator { return Ok(None); } - // Build this page's options. The first page uses the caller's options verbatim; later - // pages advance the cursor and request only the remaining budget (reference parity). + // Build this page's options. The request `limit` is always derived from the remaining + // budget (never the caller's raw `limit`), clamped to the endpoint maximum: this normalizes + // an unset/`0` cap to "no limit" and keeps a large finite cap within the accepted range so + // it paginates instead of 400-ing. Only the first page uses the caller's + // `exclusive_start_key`; later pages advance the cursor. let mut page_options = self.options.clone(); + page_options.limit = self.remaining.map(|rem| rem.min(KEY_LIST_MAX_LIMIT)); if !self.first_page { page_options.exclusive_start_key = self.next_exclusive_start_key.clone(); - page_options.limit = self.remaining; } self.first_page = false; let page = self.client.list_keys(page_options).await?; - let received = page.items.len() as i64; - + let mut items = page.items; + let received = items.len() as i64; + + // Enforce the caller's total cap exactly, even if the API returns more than requested + // (defensive parity with `ListIterator`). + if let Some(rem) = self.remaining { + if received > rem { + items.truncate(rem.max(0) as usize); + } + } if let Some(rem) = self.remaining.as_mut() { *rem -= received; } @@ -385,19 +408,7 @@ impl KeyValueStoreKeysIterator { self.exhausted = true; } - self.buffer.extend(page.items); + self.buffer.extend(items); Ok(self.buffer.pop_front()) } - - /// Eagerly drains the iterator into a single `Vec`, fetching every remaining page. - /// - /// Convenience for callers that want all keys at once; prefer [`next`](Self::next) to process - /// keys as they stream in without buffering the whole set. - pub async fn collect_all(mut self) -> ApifyClientResult> { - let mut out = Vec::new(); - while let Some(item) = self.next().await? { - out.push(item); - } - Ok(out) - } } diff --git a/src/lib.rs b/src/lib.rs index f9f67d2..a696f33 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,10 +66,12 @@ pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; pub use clients::actor::{ActorBuildOptions, ActorStartOptions}; pub use clients::actor_collection::ActorListOptions; pub use clients::dataset::{DatasetDownloadOptions, DatasetListItemsOptions, DownloadItemsFormat}; -pub use clients::key_value_store::{GetRecordOptions, GetRecordsOptions, ListKeysOptions}; +pub use clients::key_value_store::{ + GetRecordOptions, GetRecordsOptions, KeyValueStoreKeysIterator, ListKeysOptions, +}; pub use clients::log::LogOptions; pub use clients::pagination::ListIterator; -pub use clients::request_queue::ListRequestsOptions; +pub use clients::request_queue::{ListRequestsOptions, RequestQueueRequestsIterator}; pub use clients::run::{ LastRunOptions, RunChargeOptions, RunMetamorphOptions, RunResurrectOptions, }; diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 2b4a54f..1e229f0 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -73,11 +73,15 @@ async fn iterate_key_value_stores() { ); } -/// Iteration: `iterate_keys` yields every key in the store across the cursor-paginated listing. +/// Iteration: `iterate_keys` yields every key in the store. /// /// Creates a store, writes several records, then drives the `KeyValueStoreKeysIterator` to -/// completion and asserts every written key is yielded exactly once. This exercises the -/// cursor-based (`exclusiveStartKey`/`nextExclusiveStartKey`) auto-pagination helper end to end. +/// completion and asserts every written key is yielded exactly once. With only a handful of keys +/// this fits in the endpoint's single default page, so it validates the helper end to end but +/// does not cross a page boundary — `iterate_keys` has no per-page-size knob (its `limit` is a +/// total cap, matching the JS reference), so forcing >1 page would need >1000 keys. The +/// multi-page cursor-threading path (`exclusiveStartKey`/`nextExclusiveStartKey`) is covered +/// hermetically by `iterate_keys_walks_cursor_pages` in `tests/unit_http.rs`. #[tokio::test(flavor = "multi_thread")] async fn iterate_keys_yields_all_keys() { let client = require_client!(); diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 88dbbd7..b5cc8b5 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -744,3 +744,84 @@ async fn iterate_keys_limit_caps_the_walk() { backend.urls()[0] ); } + +/// `iterate_keys` with `limit = Some(0)` means "iterate everything" — the first request must NOT +/// send `limit=0` (out of the endpoint's `minimum: 1` range); the `0` is normalized to no limit. +#[tokio::test] +async fn iterate_keys_zero_limit_sends_no_limit_and_walks_all() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k0"}],"isTruncated":true,"nextExclusiveStartKey":"k0"}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k1"}],"isTruncated":false,"nextExclusiveStartKey":null}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .key_value_store("some-store") + .iterate_keys(apify_client::ListKeysOptions { + limit: Some(0), + ..Default::default() + }); + let mut keys = Vec::new(); + while let Some(key) = it.next().await.expect("ok") { + keys.push(key.key); + } + + assert_eq!(keys, vec!["k0", "k1"], "0 must iterate the whole store"); + assert!( + !backend.urls()[0].contains("limit="), + "limit=0 must be normalized to no limit param, got {}", + backend.urls()[0] + ); +} + +/// A finite `iterate_keys` cap larger than one page must be clamped to the endpoint maximum per +/// request (so it does not 400) while still yielding across pages until the cap or the store is +/// exhausted. +#[tokio::test] +async fn iterate_keys_large_cap_clamps_page_size() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k0"},{"key":"k1"}],"isTruncated":true,"nextExclusiveStartKey":"k1"}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k2"}],"isTruncated":false,"nextExclusiveStartKey":null}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + // Cap of 1500 > the endpoint max (1000): each request must ask for at most 1000. + let mut it = client + .key_value_store("some-store") + .iterate_keys(apify_client::ListKeysOptions { + limit: Some(1500), + ..Default::default() + }); + let mut keys = Vec::new(); + while let Some(key) = it.next().await.expect("ok") { + keys.push(key.key); + } + + assert_eq!( + keys, + vec!["k0", "k1", "k2"], + "walk continues under the large cap" + ); + let urls = backend.urls(); + assert!( + urls[0].contains("limit=1000") && urls[1].contains("limit=1000"), + "each request must be clamped to the endpoint max of 1000, got {urls:?}" + ); + assert!( + urls[1].contains("exclusiveStartKey=k1"), + "the cursor must still advance across the clamped pages, got {}", + urls[1] + ); +} From fd515141f7ea46f735789c8d526f80105a9b049e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 16:54:53 +0000 Subject: [PATCH 12/20] docs: make iterate_keys example self-contained and avoid Option debug output Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/storages.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/storages.md b/docs/storages.md index 3f5a52a..13237e1 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -154,10 +154,13 @@ cap still paginates. `prefix`/`collection`/`signature` filter every page: ```rust,no_run # use apify_client::{ApifyClient, ListKeysOptions}; -# async fn run(client: ApifyClient, store_id: &str) -> Result<(), Box> { -let mut keys = client.key_value_store(store_id).iterate_keys(ListKeysOptions::default()); +# async fn run(client: ApifyClient) -> Result<(), Box> { +// Obtain a store id from a metadata model (e.g. get_or_create), then iterate its keys. +let store = client.key_value_stores().get_or_create(None).await?; +let mut keys = client.key_value_store(&store.id).iterate_keys(ListKeysOptions::default()); while let Some(key) = keys.next().await? { - println!("{} ({:?} bytes)", key.key, key.size); + // `size` is optional; default to 0 bytes when the API does not report it. + println!("{} ({} bytes)", key.key, key.size.unwrap_or(0)); } # Ok(()) # } From bc4e40abc4cc4ee95d0aacccdb9e1cdf722f4c60 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 17:13:02 +0000 Subject: [PATCH 13/20] fix: iterate_keys terminates on isTruncated; drop QueryParams from docs import list - iterate_keys now leads termination with the model's is_truncated flag (cursor presence retained as fallback), avoiding a wasted final fetch and matching model semantics. Adds hermetic test iterate_keys_stops_on_is_truncated_even_with_trailing_cursor. - Drop QueryParams from the docs crate-root import prose (no public API consumes a caller-built QueryParams); re-export left intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/README.md | 2 +- src/clients/key_value_store.rs | 15 ++++++++--- tests/unit_http.rs | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0e0e8c6..a42e0e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -85,7 +85,7 @@ types is: - Shared: `ListOptions`, `StorageListOptions` - Client configuration: `RequestCompression` -plus the common container `PaginationList`, the query helper `QueryParams`, `ListIterator` +plus the common container `PaginationList`, `ListIterator` (the return type of every collection client's `iterate()` method), `StoreActorIterator` (a type alias for `ListIterator`, the return type of `StoreCollectionClient::iterate`), and the two cursor-based iterators `KeyValueStoreKeysIterator` diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index c1198b7..1767058 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -337,8 +337,9 @@ pub const KEY_LIST_MAX_LIMIT: i64 = 1000; /// Unlike the offset/limit-paginated [`ListIterator`](crate::ListIterator), key-value stores use /// cursor-based pagination: each page is anchored by the previous page's /// `nextExclusiveStartKey`. Termination mirrors the reference client's `listKeys()` generator — -/// the walk stops once a page comes back empty, the API stops returning a next cursor, or the -/// caller's `limit` is exhausted. +/// the walk stops once the page reports `isTruncated == false` (the authoritative end-of-data +/// signal), a page comes back empty, the API stops returning a next cursor, or the caller's +/// `limit` is exhausted. pub struct KeyValueStoreKeysIterator { client: KeyValueStoreClient, /// Base listing options. The `prefix`/`collection`/`signature` filters are carried into every @@ -384,6 +385,7 @@ impl KeyValueStoreKeysIterator { self.first_page = false; let page = self.client.list_keys(page_options).await?; + let is_truncated = page.is_truncated; let mut items = page.items; let received = items.len() as i64; @@ -399,9 +401,14 @@ impl KeyValueStoreKeysIterator { } self.next_exclusive_start_key = page.next_exclusive_start_key; - // Stop when the page is empty, the API returns no further cursor, or the caller's cap is - // reached — the same three termination conditions as the reference `listKeys()` loop. + // Stop when the page is empty; the model reports no more keys (`is_truncated == false`, + // the authoritative "more data?" signal); the API returns no next cursor to advance on + // (a robustness fallback in case the flag and cursor disagree — e.g. `isTruncated:true` + // with a null cursor — which also prevents re-fetching the same first page); or the + // caller's cap is reached. Leading with `is_truncated` avoids one wasted empty fetch when + // a final page still carries a cursor, matching the model semantics. if received == 0 + || !is_truncated || self.next_exclusive_start_key.is_none() || matches!(self.remaining, Some(r) if r <= 0) { diff --git a/tests/unit_http.rs b/tests/unit_http.rs index b5cc8b5..141d195 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -706,6 +706,52 @@ async fn iterate_keys_walks_cursor_pages() { ); } +/// `iterate_keys` terminates on the model's `isTruncated == false` flag (the authoritative +/// end-of-data signal), not merely on a null cursor. If the API sends the final page with +/// `isTruncated:false` but still carries a non-null `nextExclusiveStartKey`, the walk must stop +/// and not perform a wasted extra fetch. (The mock repeats its last scripted page, so a regression +/// that followed the stale cursor would loop; the iteration guard below catches that.) +#[tokio::test] +async fn iterate_keys_stops_on_is_truncated_even_with_trailing_cursor() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k0"},{"key":"k1"}],"isTruncated":true,"nextExclusiveStartKey":"k1"}}"#.to_vec(), + ), + // Final page: no more data (isTruncated:false) yet a non-null trailing cursor. + MockOutcome::Status( + 200, + br#"{"data":{"items":[{"key":"k2"}],"isTruncated":false,"nextExclusiveStartKey":"k2"}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let mut it = client + .key_value_store("some-store") + .iterate_keys(Default::default()); + let mut keys = Vec::new(); + let mut guard = 0; + while let Some(key) = it.next().await.expect("ok") { + keys.push(key.key); + guard += 1; + assert!( + guard < 100, + "iterate_keys did not terminate on isTruncated=false" + ); + } + + assert_eq!( + keys, + vec!["k0", "k1", "k2"], + "all keys across pages must be yielded" + ); + assert_eq!( + backend.call_count(), + 2, + "isTruncated:false ends the walk on the second page; no wasted fetch despite the trailing cursor" + ); +} + /// The `iterate_keys` `limit` is a total cap that also bounds the first page's request size /// (reference parity): with `limit=2` the first request asks for `limit=2` and, once two keys /// are consumed, the walk stops without a second fetch even though the API advertised more keys. From 4996e067ca48e04f72b181166a483c7d8859d005 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 17:23:39 +0000 Subject: [PATCH 14/20] fix: remove out-of-scope KVS get_records endpoint GET /v2/key-value-stores/{storeId}/records is not exposed by the reference JS client, so per client_requirements it is out of scope. Remove get_records and GetRecordsOptions (method, re-export, docs rows, integration test) and record the removal in the changelog. Also correct the iterate_keys doc wording to not claim it mirrors the reference generator's cursor loop. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 5 ++++ docs/README.md | 2 +- docs/storages.md | 1 - src/clients/key_value_store.rs | 53 ++++------------------------------ src/lib.rs | 4 +-- tests/key_value_store.rs | 42 --------------------------- 6 files changed, 12 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad97845..0675edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ to [Semantic Versioning](https://semver.org/). - Corrected the `src/models.rs` module doc to describe forward-compatibility accurately. - Bumped crate version to `0.6.0`. +### Removed +- `KeyValueStoreClient::get_records` and `GetRecordsOptions`. The `GET /v2/key-value-stores/{storeId}/records` + endpoint is not implemented by the reference JS client, so it is out of scope; its removal + corrects an earlier scope violation. + ## [0.5.0] - 2026-07-10 ### Added diff --git a/docs/README.md b/docs/README.md index a42e0e3..a57046e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -78,7 +78,7 @@ types is: - Actors: `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions` - Runs: `RunListOptions`, `RunResurrectOptions`, `RunMetamorphOptions`, `RunChargeOptions`, `LastRunOptions` - Datasets: `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat` -- Key-value stores: `ListKeysOptions`, `GetRecordsOptions`, `GetRecordOptions` +- Key-value stores: `ListKeysOptions`, `GetRecordOptions` - Request queues: `ListRequestsOptions` - Store: `StoreListOptions` - Logs: `LogOptions` diff --git a/docs/storages.md b/docs/storages.md index 13237e1..78e7316 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -126,7 +126,6 @@ println!("exported {} bytes of CSV", csv.len()); | `delete()` | — | `()` | Deletes the store. | | `list_keys(options)` | `ListKeysOptions` | `KeyValueStoreKeysPage` | Lists one page of keys (key-based pagination). | | `iterate_keys(options)` | `ListKeysOptions` | `KeyValueStoreKeysIterator` | Lazily iterates all keys across pages (cursor-based auto-pagination). | -| `get_records(options)` | `GetRecordsOptions { collection, prefix, signature }` | `Vec` | Downloads all records as a ZIP archive (raw bytes). | | `record_exists(key)` | `&str` | `bool` | Whether a record exists (HEAD). | | `get_record(key)` | `&str` | `Option` | Reads a record's raw value. | | `set_record_raw(key, bytes, content_type)` | `&str`, `Vec`, `&str` | `()` | Stores a raw record. | diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 1767058..45818e8 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -33,20 +33,6 @@ pub struct ListKeysOptions { pub signature: Option, } -/// Options for downloading all records as a ZIP archive via -/// [`KeyValueStoreClient::get_records`]. -/// -/// Covers the spec query parameters of `GET /v2/key-value-stores/{storeId}/records`. -#[derive(Debug, Default, Clone)] -pub struct GetRecordsOptions { - /// Only include records belonging to this collection from the store schema. - pub collection: Option, - /// Only include records whose key starts with this prefix. - pub prefix: Option, - /// URL-signing signature granting access to a private store's records. - pub signature: Option, -} - /// Options for reading a single record via [`KeyValueStoreClient::get_record_with_options`]. /// /// Covers the spec query parameters of @@ -147,36 +133,6 @@ impl KeyValueStoreClient { } } - /// Downloads all records from the store as a ZIP archive (raw bytes). - /// - /// Each record is stored as a separate file in the archive, with the filename equal to the - /// record key. Use [`GetRecordsOptions`] to filter by `collection` or `prefix`, or to pass a - /// URL-signing `signature` for a private store. Wraps - /// `GET /v2/key-value-stores/{storeId}/records`. - pub async fn get_records(&self, options: GetRecordsOptions) -> ApifyClientResult> { - let mut params = QueryParams::new(); - params - .add_str("collection", options.collection) - .add_str("prefix", options.prefix) - .add_str("signature", options.signature); - let url = self - .ctx - .merged_params(¶ms) - .apply_to_url(&self.ctx.url(Some("records"))); - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(response.body) - } - /// Returns `true` if a record with the given key exists. pub async fn record_exists(&self, key: &str) -> ApifyClientResult { head_exists( @@ -336,10 +292,11 @@ pub const KEY_LIST_MAX_LIMIT: i64 = 1000; /// /// Unlike the offset/limit-paginated [`ListIterator`](crate::ListIterator), key-value stores use /// cursor-based pagination: each page is anchored by the previous page's -/// `nextExclusiveStartKey`. Termination mirrors the reference client's `listKeys()` generator — -/// the walk stops once the page reports `isTruncated == false` (the authoritative end-of-data -/// signal), a page comes back empty, the API stops returning a next cursor, or the caller's -/// `limit` is exhausted. +/// `nextExclusiveStartKey`. The walk stops once the page reports `isTruncated == false` (the +/// authoritative end-of-data signal), a page comes back empty, the API stops returning a next +/// cursor, or the caller's `limit` is exhausted. This yields the same result as the reference +/// client's `listKeys()` async-iterable (which loops on the cursor); leading with `isTruncated` +/// additionally avoids a wasted empty fetch when a final page still carries a cursor. pub struct KeyValueStoreKeysIterator { client: KeyValueStoreClient, /// Base listing options. The `prefix`/`collection`/`signature` filters are carried into every diff --git a/src/lib.rs b/src/lib.rs index a696f33..c53ff11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,9 +66,7 @@ pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; pub use clients::actor::{ActorBuildOptions, ActorStartOptions}; pub use clients::actor_collection::ActorListOptions; pub use clients::dataset::{DatasetDownloadOptions, DatasetListItemsOptions, DownloadItemsFormat}; -pub use clients::key_value_store::{ - GetRecordOptions, GetRecordsOptions, KeyValueStoreKeysIterator, ListKeysOptions, -}; +pub use clients::key_value_store::{GetRecordOptions, KeyValueStoreKeysIterator, ListKeysOptions}; pub use clients::log::LogOptions; pub use clients::pagination::ListIterator; pub use clients::request_queue::{ListRequestsOptions, RequestQueueRequestsIterator}; diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 1e229f0..151dec7 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -183,48 +183,6 @@ async fn record_key_with_special_chars_round_trips() { assert!(!store_client.record_exists(key).await.expect("exists after")); } -/// Simple GET: download all records as a ZIP archive. -/// -/// Stores a record, then downloads the whole store via `get_records` and asserts the response -/// is a non-empty ZIP archive (the spec response is `application/zip`; ZIP files start with the -/// `PK\x03\x04` local-file-header magic). -#[tokio::test(flavor = "multi_thread")] -async fn get_records_returns_zip_archive() { - let client = require_client!(); - let name = common::unique_name("kvs-zip"); - let store = client - .key_value_stores() - .get_or_create(Some(&name)) - .await - .expect("create store"); - - let cleanup_client = client.clone(); - let id = store.id.clone(); - let _guard = common::Cleanup::new(move || async move { - let _ = cleanup_client.key_value_store(&id).delete().await; - }); - - let store_client = client.key_value_store(&store.id); - store_client - .set_record_json("OUTPUT", &json!({ "zip": true })) - .await - .expect("set record"); - - let archive = store_client - .get_records(Default::default()) - .await - .expect("download records as zip"); - assert!(!archive.is_empty(), "ZIP archive should not be empty"); - assert_eq!( - &archive[..4], - b"PK\x03\x04", - "response should be a ZIP archive (PK magic bytes)" - ); - - // Happy-path cleanup in the body (the guard above remains a panic-safety net). - store_client.delete().await.expect("delete store"); -} - /// Complex flow: create -> get -> set record -> read record -> list keys -> update -> delete. #[tokio::test(flavor = "multi_thread")] async fn key_value_store_crud_flow() { From d4d12213acefeed3547ada280a8b730cacdbbdeb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 17:45:55 +0000 Subject: [PATCH 15/20] docs: document set_status_message and get_openapi_definition; simplify iterator cap Address 3 open items from the independent full client review: - Document ApifyClient::set_status_message in docs/README.md + root README. - Add BuildClient::get_openapi_definition to the docs/builds.md method table. - Drop dead .max(0) clamp in the offset and KVS iterators (rem is provably > 0). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 4 ++++ README.md | 3 ++- docs/README.md | 26 ++++++++++++++++++++++++++ docs/builds.md | 1 + src/clients/key_value_store.rs | 2 +- src/clients/pagination.rs | 2 +- 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0675edf..b53f002 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ to [Semantic Versioning](https://semver.org/). per-page size with `ListIterator::with_chunk_size` instead. The `StoreActorIterator` type alias itself is unchanged. - Corrected the `src/models.rs` module doc to describe forward-compatibility accurately. +- Documented `ApifyClient::set_status_message` and `BuildClient::get_openapi_definition` in the + `docs/` pages and README. +- Simplified the total-cap truncation in the offset and key-value-store iterators (removed a + dead `.max(0)` clamp on an already-positive remaining count). - Bumped crate version to `0.6.0`. ### Removed diff --git a/README.md b/README.md index 7e7858d..1c38650 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ It provides a resource-oriented, async interface that mirrors the official - Resource clients for Actors, runs, builds, tasks, datasets, key-value stores, request queues, schedules, webhooks, the Apify Store, users and logs. - Convenience helpers: run/wait, log streaming (redirection; needs the `futures-util` crate — - see [Installation](#installation)), lazy Store iteration. + see [Installation](#installation)), lazy Store iteration, and `set_status_message` for + updating the current Actor run's status (see [`docs/README.md`](docs/README.md#convenience-methods)). - A replaceable HTTP transport for testing or custom runtimes. ## Installation diff --git a/docs/README.md b/docs/README.md index a57046e..8d432da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -171,6 +171,32 @@ reach those values the client maps Rust's native `std::env::consts::OS` spelling (`macos` → `darwin`, `windows` → `win32`, `solaris`/`illumos` → `sunos`); all other tokens (`linux`, `android`, `freebsd`, …) are already identical and pass through unchanged. +### Convenience methods + +Beyond the resource accessors, `ApifyClient` exposes one convenience method: + +| Method | Arguments | Returns | Description | +|---|---|---|---| +| `set_status_message(message, is_terminal)` | `message: &str`, `is_terminal: bool` | `ActorRun` | Sets the status message of the *current* Actor run. | + +`set_status_message` updates the run identified by the `ACTOR_RUN_ID` environment variable, so it +only works when called from inside an Actor run. `message` is the human-readable status text; when +`is_terminal` is `true` the message becomes final and is not overwritten by later updates. It +returns the updated [`ActorRun`](runs.md), or +[`ApifyClientError::InvalidArgument`](#error-handling) if `ACTOR_RUN_ID` is not set. + +```rust,no_run +use apify_client::ApifyClient; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let client = ApifyClient::new("my-api-token"); + // Called from inside an Actor run (reads ACTOR_RUN_ID from the environment). + client.set_status_message("Processing input…", false).await?; + Ok(()) +} +``` + ## Resource clients Accessor methods on `ApifyClient` return resource clients — the collection accessor (plural) diff --git a/docs/builds.md b/docs/builds.md index e6b1179..f6b19d4 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -18,6 +18,7 @@ collections are available via `actor.builds()`. | `abort()` | — | `Build` | Aborts the build. | | `delete()` | — | `()` | Deletes the build. | | `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. | +| `get_openapi_definition()` | — | `Option` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). | | `log()` | — | `LogClient` | Access the build's log. | The returned `Build` model's fields (`id`, `status`, `build_number`, …) are documented in diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 45818e8..bd68d74 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -350,7 +350,7 @@ impl KeyValueStoreKeysIterator { // (defensive parity with `ListIterator`). if let Some(rem) = self.remaining { if received > rem { - items.truncate(rem.max(0) as usize); + items.truncate(rem as usize); } } if let Some(rem) = self.remaining.as_mut() { diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index 3a3c1e8..d024244 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -182,7 +182,7 @@ impl ListIterator { let mut items = page.items; if let Some(rem) = self.remaining { if received > rem { - items.truncate(rem.max(0) as usize); + items.truncate(rem as usize); } } From c1e3757000514030e7acdfa65f1395514c6846fc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 18:11:17 +0000 Subject: [PATCH 16/20] docs: clarify iterate_items filter edge cases, trim comment, sync iterate() docs - Document the item-drop edge case (fully-filtered window -> empty page -> walk terminates) in iterate_items, alongside the existing duplicate case - Trim the verbose pagination-total unwrap_or(0) rationale comment to 3 lines - Add iterate() to the ActorVersion/ActorEnvVar collection tables in docs/actors.md to agree with docs/README.md and the code Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/actors.md | 4 ++-- src/clients/dataset.rs | 28 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/actors.md b/docs/actors.md index 7e63354..16be588 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -157,6 +157,6 @@ println!("build {} status {:?}", build.id, build.status); ## Actor versions and environment variables `ActorVersionClient`: `get`, `update`, `delete`, `env_var(name)`, `env_vars()`. -`ActorVersionCollectionClient`: `list(options)`, `create(version)`. +`ActorVersionCollectionClient`: `list(options)`, `iterate(options)`, `create(version)`. `ActorEnvVarClient`: `get`, `update`, `delete`. -`ActorEnvVarCollectionClient`: `list()`, `create(env_var)`. +`ActorEnvVarCollectionClient`: `list()`, `iterate()`, `create(env_var)`. diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 14b984f..d59a815 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -210,12 +210,9 @@ impl DatasetClient { let items: Vec = serde_json::from_slice(&response.body)?; let count = items.len() as i64; - // When the endpoint omits the total header, report `0` ("total unknown"), not the current - // page's own item count. A fallback of `count` would look like a genuine total equal to - // the number of items already returned, which [`ListIterator`] reads as "listing complete" - // and would stop after the first page, silently dropping later items. `0` instead routes - // iteration to the short-page / empty-page backstop, which walks every page. The live - // dataset-items endpoint always sends this header, so this only guards a degenerate case. + // Fall back to `0` ("total unknown"), never `count`: a total equal to the items already + // returned would look complete and stop iteration after page one, dropping later items. + // `0` routes iteration to the short-page/empty-page backstop, which walks every page. let total = response .header("x-apify-pagination-total") .and_then(|v| v.parse().ok()) @@ -248,11 +245,20 @@ impl DatasetClient { /// /// Server-side filters (`skip_empty`/`clean`/`skip_hidden`) are forwarded on every page /// request. Paging advances the offset by the number of items each page returns, exactly - /// like the reference JavaScript client (`currentOffset += items.length`). When a filter - /// drops items from a page, that post-filter count is smaller than the raw window, so the - /// next page starts at an offset that overlaps the previous window and some items may be - /// yielded more than once. If you need every filtered item exactly once, apply the filter - /// client-side over an unfiltered iteration instead. + /// like the reference JavaScript client (`currentOffset += items.length`). Because the offset + /// advances by the post-filter count rather than the raw window size, filtered iteration is + /// not exact — this matches the reference client, and it can distort results two ways: + /// + /// - Duplicates: when a page's filtered count is smaller than its raw window, the next page + /// starts at an offset that overlaps the previous window, so some items are yielded more + /// than once. + /// - Dropped items: when a filter removes *every* item in a raw window, the page comes back + /// with no items; iteration treats that empty page as the end of the dataset (the empty-page + /// backstop in [`ListIterator`]) and stops, even though unfiltered items still exist at + /// higher offsets. + /// + /// If you need every filtered item exactly once, apply the filter client-side over an + /// unfiltered iteration instead. pub fn iterate_items( &self, options: DatasetListItemsOptions, From 255235eba026bc5aef23b7308f5750878c124180 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 18:37:23 +0000 Subject: [PATCH 17/20] docs: complete ActorStartOptions fields, JSONL format, robust store example, clean Option output Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 11 +++++++++++ docs/README.md | 2 +- docs/actors.md | 20 ++++++++++++++++---- docs/storages.md | 5 ++++- examples/get_account.rs | 2 +- examples/iterate_store.rs | 6 +++++- examples/run_store_actor.rs | 19 ++++++++++--------- 7 files changed, 48 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b53f002..7d9b8a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,17 @@ to [Semantic Versioning](https://semver.org/). endpoint is not implemented by the reference JS client, so it is out of scope; its removal corrects an earlier scope violation. +### Documentation +- Documented all `ActorStartOptions` fields in `docs/actors.md` (added the previously undocumented + `restart_on_error`, `force_permission_level`, and `webhooks`). +- Listed `JSONL` in the `download_items` format summary in `docs/storages.md` for consistency with + the `DownloadItemsFormat` variant list. +- Noted why the request-queue iterator is named `paginate_requests` (mirrors the reference JS + `paginateRequests`) rather than an `iterate_*` verb. +- Made the `run_store_actor` example resilient to Store ranking shifts by falling back to the + well-known `apify/hello-world` identifier, and cleaned up `Option` output in the `get_account` + and `iterate_store` examples and the `docs/README.md` quick-start snippet. + ## [0.5.0] - 2026-07-10 ### Added diff --git a/docs/README.md b/docs/README.md index 8d432da..ff992b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,7 +56,7 @@ use apify_client::ApifyClient; async fn main() -> Result<(), Box> { let client = ApifyClient::new("my-api-token"); let user = client.me().get().await?.expect("account"); - println!("Logged in as {:?}", user.username); + println!("Logged in as {}", user.username.as_deref().unwrap_or("(none)")); Ok(()) } ``` diff --git a/docs/actors.md b/docs/actors.md index 16be588..5998f92 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -33,10 +33,22 @@ be an Actor ID or a `username~name` (or `username/name`) reference. ### `ActorStartOptions` -`build`, `memory_mbytes`, `timeout_secs`, `wait_for_finish`, `max_items`, -`max_total_charge_usd`, `content_type` — all optional. Used by both `start` and `call` -(for `call`, `wait_for_finish` is server-side; the `wait_secs` argument controls -client-side polling). +All fields are optional. Used by both `start` and `call` here, and by the identical `start` / +`call` methods on [tasks](tasks.md) (for `call`, `wait_for_finish` is server-side; the +`wait_secs` argument controls client-side polling). + +| Field | Type | Description | +|---|---|---| +| `build` | `Option` | Tag or number of the build to run (e.g. `latest`, `0.1.2`). | +| `memory_mbytes` | `Option` | Memory in megabytes allocated for the run. | +| `timeout_secs` | `Option` | Timeout for the run in seconds (`0` means no timeout). | +| `wait_for_finish` | `Option` | Maximum seconds to wait server-side for the run to finish (max 60). | +| `max_items` | `Option` | Maximum number of dataset items to charge (pay-per-result Actors). | +| `max_total_charge_usd` | `Option` | Maximum total charge in USD (pay-per-event Actors). | +| `content_type` | `Option` | Content type of the input body. Defaults to `application/json`. | +| `restart_on_error` | `Option` | Whether to restart the run if it fails. | +| `force_permission_level` | `Option` | Override the Actor's permission level for this run. | +| `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. Encoded as base64 JSON in the `webhooks` query parameter, matching the reference clients. | The `wait_secs` argument of `call` (and of `wait_for_finish` on runs/builds) controls the client-side polling budget: diff --git a/docs/storages.md b/docs/storages.md index 78e7316..ee3f141 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -64,7 +64,7 @@ let scratch = client.datasets().get_or_create(None).await?; | `iterate_items::(options)` | `DatasetListItemsOptions` | `ListIterator` | Lazily iterates all items across pages (auto-pagination). | | `push_items(items)` | `&impl Serialize` | `()` | Appends items (object or array). | | `get_statistics()` | — | `Option` | Field statistics. | -| `download_items(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Vec` | Export items as JSON/CSV/XLSX/XML/RSS/HTML. | +| `download_items(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Vec` | Export items as JSON/JSONL/CSV/XLSX/XML/RSS/HTML. | | `create_items_public_url(options, expires)` | `DatasetListItemsOptions`, `Option` | `String` | Shareable (HMAC-signed for private) items URL. | `DatasetListItemsOptions` (all optional): @@ -203,6 +203,9 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: `paginate_requests(page_limit)` returns a `RequestQueueRequestsIterator` — a lazy, page-fetching iterator (parity with the Store iterator in [Store, users and logs](misc.md#apify-store--clientstore)). +It is named `paginate_requests` (rather than an `iterate_*` verb like the dataset/key-value-store +iterators) to mirror the reference JavaScript client's `paginateRequests` method, keeping the +public interface consistent across the two clients. Its `next()` is `async` and fallible, returning `ApifyClientResult>`, fetching the next page on demand and yielding `Ok(None)` once the queue is exhausted. `page_limit` bounds the requests fetched per page (`None` diff --git a/examples/get_account.rs b/examples/get_account.rs index 0bb3249..4e1756a 100644 --- a/examples/get_account.rs +++ b/examples/get_account.rs @@ -12,7 +12,7 @@ async fn main() -> Result<(), Box> { let user = client.me().get().await?.expect("current user"); println!("Account id: {}", user.id); - println!("Username: {:?}", user.username); + println!("Username: {}", user.username.as_deref().unwrap_or("(none)")); // Monthly usage for the current billing cycle (`None` == current cycle). let usage = client.me().monthly_usage().await?; diff --git a/examples/iterate_store.rs b/examples/iterate_store.rs index c2b9be5..23b2fea 100644 --- a/examples/iterate_store.rs +++ b/examples/iterate_store.rs @@ -19,7 +19,11 @@ async fn main() -> Result<(), Box> { let mut count = 0; while let Some(actor) = iter.next().await? { - println!("{}: {:?}", actor.id, actor.title.or(actor.name)); + let label = actor + .title + .or(actor.name) + .unwrap_or_else(|| "(untitled)".to_string()); + println!("{}: {label}", actor.id); count += 1; if count >= 10 { break; diff --git a/examples/run_store_actor.rs b/examples/run_store_actor.rs index d545f96..f0fdcf7 100644 --- a/examples/run_store_actor.rs +++ b/examples/run_store_actor.rs @@ -2,7 +2,8 @@ //! its default dataset. //! //! This example uses the Store API (`client.store()`) to find the actor first, so it really -//! exercises "run an Actor discovered in the Store" rather than hard-coding an Actor ID. +//! exercises "run an Actor discovered in the Store"; it only falls back to the well-known +//! `apify/hello-world` identifier if the search does not surface it. //! //! Run with: `APIFY_TOKEN=... cargo run --example run_store_actor` @@ -24,22 +25,22 @@ async fn main() -> Result<(), Box> { }) .await?; - let actor = store_page + // Prefer the ID discovered via search, but fall back to the well-known `apify/hello-world` + // identifier if Store ranking pushes it out of the first page — that keeps the example + // (and its CI smoke test) from failing when search results shift. + let actor_id = store_page .items .into_iter() .find(|a| { a.username.as_deref() == Some("apify") && a.name.as_deref() == Some("hello-world") }) - .expect("apify/hello-world should be discoverable in the Apify Store"); - println!( - "Found Store actor {} (\"{}\")", - actor.id, - actor.title.clone().unwrap_or_default() - ); + .map(|a| a.id) + .unwrap_or_else(|| "apify~hello-world".to_string()); + println!("Using Store actor {actor_id}"); // Run the discovered Actor and wait up to 2 minutes for it to finish. let run = client - .actor(&actor.id) + .actor(&actor_id) .call::(None, Default::default(), Some(120)) .await?; println!("Run {} finished with status {:?}", run.id, run.status); From 44fb51d5ae2c0346b87540fcd7d60aff55ed81a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 18:42:15 +0000 Subject: [PATCH 18/20] docs: link task start/call rows to ActorStartOptions field table Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 3 ++- docs/tasks.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9b8a8..7616a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,8 @@ to [Semantic Versioning](https://semver.org/). ### Documentation - Documented all `ActorStartOptions` fields in `docs/actors.md` (added the previously undocumented - `restart_on_error`, `force_permission_level`, and `webhooks`). + `restart_on_error`, `force_permission_level`, and `webhooks`), and linked the task `start`/`call` + rows in `docs/tasks.md` to that field table. - Listed `JSONL` in the `download_items` format summary in `docs/storages.md` for consistency with the `DownloadItemsFormat` variant list. - Noted why the request-queue iterator is named `paginate_requests` (mirrors the reference JS diff --git a/docs/tasks.md b/docs/tasks.md index 775a155..8c450f0 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -17,8 +17,8 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). | `get()` | — | `Option` | Fetches the task. | | `update(fields)` | `&impl Serialize` | `Task` | Updates the task. | | `delete()` | — | `()` | Deletes the task. | -| `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run. | -| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits. | +| `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run. See [`ActorStartOptions`](actors.md#actorstartoptions) for the full field list. | +| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits. Same [`ActorStartOptions`](actors.md#actorstartoptions) as `start`. | | `get_input()` / `update_input(input)` | — / `&impl Serialize` | `Option` / `Value` | The task's saved input. | | `last_run(status)` | `Option<&str>` | `RunClient` | The task's last run, optionally filtered by status. See [Actor runs](runs.md) for the accepted `status` values. | | `last_run_with_options(options)` | `LastRunOptions { status, origin }` | `RunClient` | The task's last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | From d0bebf2084e25761a88d448413730d8726f3e0b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 19:07:36 +0000 Subject: [PATCH 19/20] fix: retry create-then-iterate tests for collection list eventual consistency The iterate_key_value_stores integration test flaked on CI: a just-created key-value store was not yet reflected in the collection LIST endpoint when the test scanned the iterator once. This is eventual consistency, not a pagination or client bug. Add a bounded, shared test helper `iter_contains_eventually` that rebuilds the iterator and re-scans up to 5 times with a 500ms backoff (~2s budget), matching on the first attempt when the entity is already present (no-op in the common case). Use it for all 12 create-then-iterate collection assertions that share this race. Test-only change; no client runtime behaviour is affected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- Cargo.toml | 2 +- tests/actor.rs | 43 +++++++++++++++++++++++++--------------- tests/actor_run.rs | 27 +++++++++++++++---------- tests/build.rs | 21 ++++++++++++-------- tests/common/mod.rs | 38 +++++++++++++++++++++++++++++++++++ tests/dataset.rs | 21 ++++++++++++-------- tests/key_value_store.rs | 21 ++++++++++++-------- tests/request_queue.rs | 21 ++++++++++++-------- tests/schedule.rs | 21 ++++++++++++-------- tests/task.rs | 21 ++++++++++++-------- tests/webhook.rs | 42 ++++++++++++++++++++++++--------------- 11 files changed, 186 insertions(+), 92 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f4fa1fb..019d5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ brotli = "7" flate2 = "1" [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } uuid = { version = "1", features = ["v4"] } [features] diff --git a/tests/actor.rs b/tests/actor.rs index 83928a4..609c8b6 100644 --- a/tests/actor.rs +++ b/tests/actor.rs @@ -92,17 +92,22 @@ async fn iterate_actors() { }); // Restrict to the caller's own Actors, newest-first, with a small page size. - let iter = client - .actors() - .iterate(apify_client::ActorListOptions { - my: Some(true), - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = actor.id.clone(); assert!( - common::iter_contains(iter, move |a| a.id == target).await, + common::iter_contains_eventually( + || { + client + .actors() + .iterate(apify_client::ActorListOptions { + my: Some(true), + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |a| a.id == target, + ) + .await, "actor iteration should yield the created actor" ); } @@ -124,12 +129,15 @@ async fn iterate_actor_versions() { let _ = cleanup_client.actor(&id).delete().await; }); - let iter = client - .actor(&actor.id) - .versions() - .iterate(Default::default()); assert!( - common::iter_contains(iter, |v| v.version_number == "0.0").await, + common::iter_contains_eventually( + || client + .actor(&actor.id) + .versions() + .iterate(Default::default()), + |v| v.version_number == "0.0", + ) + .await, "version iteration should yield the initial 0.0 version" ); } @@ -165,9 +173,12 @@ async fn iterate_actor_env_vars() { .await .expect("create env var"); - let iter = version_client.env_vars().iterate(); assert!( - common::iter_contains(iter, |e| e.name == "ITER_VAR").await, + common::iter_contains_eventually( + || version_client.env_vars().iterate(), + |e| e.name == "ITER_VAR", + ) + .await, "env-var iteration should yield the created variable" ); } diff --git a/tests/actor_run.rs b/tests/actor_run.rs index e708b46..03271a7 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -96,19 +96,24 @@ async fn iterate_runs() { // Newest-first with a small page size so the just-finished run is near the front. `limit` // is a total-item cap, so it is left unset here; page size is set via `with_chunk_size`. - let iter = client - .runs() - .iterate( - apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }, - Default::default(), - ) - .with_chunk_size(5); let target = run.id.clone(); assert!( - common::iter_contains(iter, move |r| r.id == target).await, + common::iter_contains_eventually( + || { + client + .runs() + .iterate( + apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }, + Default::default(), + ) + .with_chunk_size(5) + }, + move |r| r.id == target, + ) + .await, "run iteration should yield the started run" ); } diff --git a/tests/build.rs b/tests/build.rs index bfc96e3..dab9193 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -62,16 +62,21 @@ async fn iterate_builds() { .await .expect("start build"); - let iter = actor_client - .builds() - .iterate(apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = build.id.clone(); assert!( - common::iter_contains(iter, move |b| b.id == target).await, + common::iter_contains_eventually( + || { + actor_client + .builds() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |b| b.id == target, + ) + .await, "build iteration should yield the started build" ); } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 25e45e8..cb523ca 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -142,6 +142,44 @@ where false } +/// Number of times [`iter_contains_eventually`] rebuilds the iterator and re-scans while waiting +/// for a just-created resource to become visible in its collection LIST endpoint. +pub const ITER_RETRY_ATTEMPTS: usize = 5; + +/// Delay between the attempts made by [`iter_contains_eventually`]. With [`ITER_RETRY_ATTEMPTS`] +/// attempts the helper sleeps at most `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` (a couple +/// of seconds) before giving up. +pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500); + +/// Like [`iter_contains`], but tolerant of eventual consistency in collection LIST endpoints. +/// +/// A resource created through a write endpoint is not always immediately reflected in its +/// collection's LIST response — the write and the list index converge asynchronously on the +/// server. A create-then-iterate test that scans the collection exactly once therefore races that +/// convergence and flakes when the just-created entity has not yet propagated (observed on CI for +/// `iterate_key_value_stores`, and structurally possible for every sibling collection test). +/// +/// This helper rebuilds a fresh iterator via `make_iter` and re-scans it with [`iter_contains`] up +/// to [`ITER_RETRY_ATTEMPTS`] times, sleeping [`ITER_RETRY_BACKOFF`] between attempts, returning +/// `true` as soon as `pred` matches. When the entity is already visible it matches on the first +/// attempt and returns immediately with no sleeping — so it is a no-op in the common +/// already-consistent case and only pays the backoff on the rare lagging run. +pub async fn iter_contains_eventually(mut make_iter: Mk, mut pred: F) -> bool +where + Mk: FnMut() -> apify_client::ListIterator, + F: FnMut(&T) -> bool, +{ + for attempt in 0..ITER_RETRY_ATTEMPTS { + if iter_contains(make_iter(), &mut pred).await { + return true; + } + if attempt + 1 < ITER_RETRY_ATTEMPTS { + tokio::time::sleep(ITER_RETRY_BACKOFF).await; + } + } + false +} + /// Generates a unique, collision-resistant resource name for test isolation. /// /// The name embeds the test-specific `prefix`, a random UUID fragment, and is kept short diff --git a/tests/dataset.rs b/tests/dataset.rs index 118005e..7c81caf 100644 --- a/tests/dataset.rs +++ b/tests/dataset.rs @@ -62,16 +62,21 @@ async fn iterate_datasets() { }); // Newest-first with a small page size so the iterator must fetch at least one page. - let iter = client - .datasets() - .iterate(apify_client::StorageListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = dataset.id.clone(); assert!( - common::iter_contains(iter, move |d| d.id == target).await, + common::iter_contains_eventually( + || { + client + .datasets() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |d| d.id == target, + ) + .await, "dataset iteration should yield the created dataset" ); } diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 151dec7..e8586b8 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -59,16 +59,21 @@ async fn iterate_key_value_stores() { let _ = cleanup_client.key_value_store(&id).delete().await; }); - let iter = client - .key_value_stores() - .iterate(apify_client::StorageListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = store.id.clone(); assert!( - common::iter_contains(iter, move |s| s.id == target).await, + common::iter_contains_eventually( + || { + client + .key_value_stores() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |s| s.id == target, + ) + .await, "key-value store iteration should yield the created store" ); } diff --git a/tests/request_queue.rs b/tests/request_queue.rs index 219e670..6072d6b 100644 --- a/tests/request_queue.rs +++ b/tests/request_queue.rs @@ -60,16 +60,21 @@ async fn iterate_request_queues() { let _ = cleanup_client.request_queue(&id).delete().await; }); - let iter = client - .request_queues() - .iterate(apify_client::StorageListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = queue.id.clone(); assert!( - common::iter_contains(iter, move |q| q.id == target).await, + common::iter_contains_eventually( + || { + client + .request_queues() + .iterate(apify_client::StorageListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |q| q.id == target, + ) + .await, "request queue iteration should yield the created queue" ); } diff --git a/tests/schedule.rs b/tests/schedule.rs index b9b1170..5873785 100644 --- a/tests/schedule.rs +++ b/tests/schedule.rs @@ -69,16 +69,21 @@ async fn iterate_schedules() { let _ = cleanup_client.schedule(&id).delete().await; }); - let iter = client - .schedules() - .iterate(apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = schedule.id.clone(); assert!( - common::iter_contains(iter, move |s| s.id == target).await, + common::iter_contains_eventually( + || { + client + .schedules() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |s| s.id == target, + ) + .await, "schedule iteration should yield the created schedule" ); } diff --git a/tests/task.rs b/tests/task.rs index 9155567..a56971c 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -68,16 +68,21 @@ async fn iterate_tasks() { let _ = cleanup_client.task(&id).delete().await; }); - let iter = client - .tasks() - .iterate(apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = task.id.clone(); assert!( - common::iter_contains(iter, move |t| t.id == target).await, + common::iter_contains_eventually( + || { + client + .tasks() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |t| t.id == target, + ) + .await, "task iteration should yield the created task" ); } diff --git a/tests/webhook.rs b/tests/webhook.rs index ea27073..3dc8082 100644 --- a/tests/webhook.rs +++ b/tests/webhook.rs @@ -114,16 +114,21 @@ async fn iterate_webhooks() { let _ = cleanup_client.webhook(&id).delete().await; }); - let iter = client - .webhooks() - .iterate(apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = webhook.id.clone(); assert!( - common::iter_contains(iter, move |w| w.id == target).await, + common::iter_contains_eventually( + || { + client + .webhooks() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |w| w.id == target, + ) + .await, "webhook iteration should yield the created webhook" ); } @@ -152,16 +157,21 @@ async fn iterate_webhook_dispatches() { .expect("test webhook"); assert!(!dispatch.id.is_empty()); - let iter = client - .webhook_dispatches() - .iterate(apify_client::ListOptions { - desc: Some(true), - ..Default::default() - }) - .with_chunk_size(5); let target = dispatch.id.clone(); assert!( - common::iter_contains(iter, move |d| d.id == target).await, + common::iter_contains_eventually( + || { + client + .webhook_dispatches() + .iterate(apify_client::ListOptions { + desc: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + }, + move |d| d.id == target, + ) + .await, "webhook-dispatch iteration should yield the triggered dispatch" ); } From 96e60328bf496de11e3a44a7e2a594dd5cfcbf15 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 19:17:11 +0000 Subject: [PATCH 20/20] test: widen eventual-consistency retry budget and trim helper docstring Address flake-fix re-review: raise iter_contains_eventually retry budget from ~2s (5 x 500ms) to ~15s (16 x 1s) so it has real headroom above the ~10s propagation lag previously observed in this suite, instead of an unverified 2s guess that could let the flake recur at lower frequency. Consistent accounts still return on the first attempt with no sleeping. Also drop the concrete incident-test name from the docstring and de-duplicate the budget prose. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- tests/common/mod.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index cb523ca..6cf6724 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -144,26 +144,31 @@ where /// Number of times [`iter_contains_eventually`] rebuilds the iterator and re-scans while waiting /// for a just-created resource to become visible in its collection LIST endpoint. -pub const ITER_RETRY_ATTEMPTS: usize = 5; +pub const ITER_RETRY_ATTEMPTS: usize = 16; -/// Delay between the attempts made by [`iter_contains_eventually`]. With [`ITER_RETRY_ATTEMPTS`] -/// attempts the helper sleeps at most `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` (a couple -/// of seconds) before giving up. -pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500); +/// Delay between the attempts made by [`iter_contains_eventually`]. +pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(1000); /// Like [`iter_contains`], but tolerant of eventual consistency in collection LIST endpoints. /// /// A resource created through a write endpoint is not always immediately reflected in its /// collection's LIST response — the write and the list index converge asynchronously on the /// server. A create-then-iterate test that scans the collection exactly once therefore races that -/// convergence and flakes when the just-created entity has not yet propagated (observed on CI for -/// `iterate_key_value_stores`, and structurally possible for every sibling collection test). +/// convergence and flakes when the just-created entity has not yet propagated. /// /// This helper rebuilds a fresh iterator via `make_iter` and re-scans it with [`iter_contains`] up /// to [`ITER_RETRY_ATTEMPTS`] times, sleeping [`ITER_RETRY_BACKOFF`] between attempts, returning /// `true` as soon as `pred` matches. When the entity is already visible it matches on the first /// attempt and returns immediately with no sleeping — so it is a no-op in the common /// already-consistent case and only pays the backoff on the rare lagging run. +/// +/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = ~15s of retrying before giving up. +/// This is deliberately larger than a "couple of seconds": the only Apify propagation lag actually +/// measured in this suite is the dataset-items count settling at ~10s, and the collection LIST +/// index convergence time is not independently measured, so a ~2s budget could let the flake recur +/// at a lower (harder-to-diagnose) frequency. ~15s gives real headroom above the ~10s observation +/// while still failing fast enough on a genuinely-missing entity (a true bug). The cost lands only +/// on lagging or genuinely-failing runs; a consistent account never sleeps. pub async fn iter_contains_eventually(mut make_iter: Mk, mut pred: F) -> bool where Mk: FnMut() -> apify_client::ListIterator,