diff --git a/CHANGELOG.md b/CHANGELOG.md index fe270dc..3e29d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,47 @@ All notable changes to the Apify Go client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.6.0] - 2026-07-10 + +### Added + +- Lazy `Iterate` helpers on every list collection (`Actors`, `Runs`, `Builds`, `Tasks`, + `Datasets`, `KeyValueStores`, `RequestQueues`, `Schedules`, `Webhooks`, `WebhookDispatches`, + actor versions and env vars) plus dataset-item iteration (`DatasetClient.IterateItems` and the + generic `IterateDatasetItems[T]`), backed by a new exported generic iterator type + `ListIterator[T]`. As in the reference client's iterable `list()`, the options' `Limit` caps the + total number of items yielded across all pages (unset means all), and the per-page size is a + separate `chunkSize` argument (nil for the server default). +- Cursor-based key iteration on the key-value store: `KeyValueStoreClient.IterateKeys` returns a + `KeyValueStoreKeysIterator` that lazily walks all keys via `nextExclusiveStartKey`, matching the + reference client's iterable `listKeys()`. `Limit` caps the total keys yielded and `chunkSize` is + the page size. + +### Fixed + +- The `Iterate` helpers now honor a caller-set `Offset` on the list options as the starting point + (iteration begins there and the cap counts from that offset), instead of silently discarding it. + Matches the reference client's `options.offset` handling. + +### Changed + +- Bumped `APISpecVersion` to `v2-2026-07-10T105921Z`. +- Bumped `ClientVersion` to `0.6.0`. +- **Breaking:** `StoreCollectionClient.Iterate` now takes a second `chunkSize *int64` argument and + treats the options' `Limit` as a total-item cap rather than the per-page size, to match the + reference client's iterator semantics. `StoreActorIterator` is now an alias of + `ListIterator[ActorStoreListItem]`. +- Synced the `APISpecVersion` reference in the `README.md` "Versioning" section to match `version.go`. + +### Documentation + +- Documented how the client-side `Call`/`WaitForFinish` polling relates to `WithTimeout` (each + poll asks the server to wait ≤60s, so the per-request timeout never cuts off a `nil` wait). +- Added a pointer to where API tokens come from (Apify Console → Settings → Integrations). +- Added the `IterateDatasetItems[T]` signature and a usage example to the storages guide. +- Added `WithPublicBaseURL` to the `NewClientWithOptions` sample and a note distinguishing + `client.Build(id)` from `Actor.Build(...)`. + ## [0.5.0] - 2026-07-09 ### Changed diff --git a/README.md b/README.md index aa63452..71abaec 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,13 @@ import ( "context" "fmt" "log" + "os" apify "github.com/apify/apify-client-go" ) func main() { - client := apify.NewClient("my-api-token") + client := apify.NewClient(os.Getenv("APIFY_TOKEN")) ctx := context.Background() // Start an Actor and wait for it to finish. @@ -63,6 +64,11 @@ func main() { } ``` +Get your API token from the [Apify Console](https://console.apify.com/) under **Settings → +Integrations** (the **Personal API tokens** section). The client never reads it from the +environment itself: pass the token to `NewClient`/`WithToken` explicitly (the examples above +read `APIFY_TOKEN` from the environment only as a convenience in `main`). + ## Configuration Use `NewClient(token)` for a token-only setup, or `NewClientWithOptions` with functional @@ -71,8 +77,9 @@ options for full control: ```go client := apify.NewClientWithOptions( apify.WithToken("my-api-token"), - apify.WithBaseURL("https://api.apify.com"), // /v2 is appended automatically - apify.WithMaxRetries(8), // default 8 + apify.WithBaseURL("https://api.apify.com"), // /v2 is appended automatically + apify.WithPublicBaseURL("https://api.apify.com"), // base for signed, shareable URLs + apify.WithMaxRetries(8), // default 8 apify.WithMinDelayBetweenRetries(500*time.Millisecond), apify.WithTimeout(360*time.Second), // default 6 minutes apify.WithUserAgentSuffix("MyTool/1.0"), @@ -179,7 +186,7 @@ func main() { - `apify.ClientVersion` — the semantic version of this library. - `apify.APISpecVersion` — the Apify OpenAPI spec version this client was built against - (`v2-2026-07-08T143931Z`). + (`v2-2026-07-10T105921Z`). ### Releasing diff --git a/actor_collection.go b/actor_collection.go index e816145..dbdcab8 100644 --- a/actor_collection.go +++ b/actor_collection.go @@ -40,6 +40,19 @@ func (c *ActorCollectionClient) List(ctx context.Context, options ActorListOptio return listResource[Actor](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the Actors matching the options, fetching pages on +// demand. The options' Limit caps the total number of Actors yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *ActorCollectionClient) Iterate(options ActorListOptions, chunkSize *int64) *ListIterator[Actor] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Actor], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // Create creates a new Actor. actor is any JSON-serializable Actor definition. func (c *ActorCollectionClient) Create(ctx context.Context, actor any) (Actor, error) { return createResource[Actor](ctx, c.ctx, NewQueryParams(), actor) diff --git a/actor_env_var.go b/actor_env_var.go index c7c1263..67a6a4b 100644 --- a/actor_env_var.go +++ b/actor_env_var.go @@ -17,6 +17,16 @@ func (c *ActorEnvVarCollectionClient) List(ctx context.Context) (PaginationList[ return listResource[ActorEnvVar](ctx, c.ctx, "", NewQueryParams()) } +// Iterate returns a lazy iterator over the version's environment variables. Mirrors the +// reference client's iterable list(). The env-vars endpoint is not offset-paginated (it +// returns the full set in a single page), so there is no Limit/chunkSize control and the +// closure ignores the offset/limit arguments; the iterator drains that one page. +func (c *ActorEnvVarCollectionClient) Iterate() *ListIterator[ActorEnvVar] { + return newListIterator(nil, nil, 0, func(ctx context.Context, _, _ int64) (PaginationList[ActorEnvVar], error) { + return c.List(ctx) + }) +} + // Create creates a new environment variable. func (c *ActorEnvVarCollectionClient) Create(ctx context.Context, envVar ActorEnvVar) (ActorEnvVar, error) { return createResource[ActorEnvVar](ctx, c.ctx, NewQueryParams(), envVar) diff --git a/actor_version.go b/actor_version.go index 7058ecf..ad1fae6 100644 --- a/actor_version.go +++ b/actor_version.go @@ -19,6 +19,19 @@ func (c *ActorVersionCollectionClient) List(ctx context.Context, options ListOpt return listResource[ActorVersion](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the Actor's versions matching the options, fetching +// pages on demand. The options' Limit caps the total number of versions yielded (unset means +// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference +// client's iterable list(). +func (c *ActorVersionCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[ActorVersion] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorVersion], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // Create creates a new Actor version. version is any JSON-serializable version definition. func (c *ActorVersionCollectionClient) Create(ctx context.Context, version any) (ActorVersion, error) { return createResource[ActorVersion](ctx, c.ctx, NewQueryParams(), version) diff --git a/build.go b/build.go index b9297b0..5cc8d22 100644 --- a/build.go +++ b/build.go @@ -28,6 +28,19 @@ func (c *BuildCollectionClient) List(ctx context.Context, options ListOptions) ( return listResource[Build](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the builds matching the options, fetching pages on +// demand. The options' Limit caps the total number of builds yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *BuildCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Build] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Build], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // BuildClient is a client for a specific Actor build (/v2/actor-builds/{buildId}). type BuildClient struct { ctx *resourceContext diff --git a/dataset.go b/dataset.go index d47a0ef..5a09945 100644 --- a/dataset.go +++ b/dataset.go @@ -152,11 +152,12 @@ func (c *DatasetClient) Delete(ctx context.Context) error { return deleteResource(ctx, c.ctx, "") } -// ListItems lists items from the dataset. +// ListDatasetItems lists a single page of items from the dataset, decoding each into T +// (e.g. json.RawMessage or a struct). // // The dataset items endpoint returns a bare JSON array (not a data envelope) and reports // pagination via X-Apify-Pagination-* headers, which are surfaced in the returned -// [PaginationList]. T is the item type to decode into (e.g. json.RawMessage or a struct). +// [PaginationList]. func ListDatasetItems[T any](ctx context.Context, c *DatasetClient, options DatasetListItemsOptions) (PaginationList[T], error) { var result PaginationList[T] params := NewQueryParams() @@ -188,6 +189,34 @@ func (c *DatasetClient) ListItems(ctx context.Context, options DatasetListItemsO return ListDatasetItems[json.RawMessage](ctx, c, options) } +// IterateDatasetItems returns a lazy iterator over the dataset's items, decoding each into T +// and fetching pages on demand. The options' Limit caps the total number of items yielded +// (unset means all); the per-page size is chunkSize (nil for the server default). Mirrors the +// reference client's iterable listItems(). +// +// Caveat: offset-based iteration paginates using the item total reported in the +// X-Apify-Pagination-Total header, and that header can lag right after items are pushed (the +// count is updated asynchronously). Iterating immediately after a push may therefore stop early +// (after one page) until the total settles. This matches the reference client's behaviour; wait +// for the total to converge before iterating a just-written dataset if completeness matters. +func IterateDatasetItems[T any](c *DatasetClient, options DatasetListItemsOptions, chunkSize *int64) *ListIterator[T] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[T], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return ListDatasetItems[T](ctx, c, opts) + }) +} + +// IterateItems returns a lazy iterator over the dataset's items, decoding each into a generic +// json.RawMessage. For typed decoding use [IterateDatasetItems]. See IterateDatasetItems for +// how the options' Limit (total cap) and chunkSize (page size) are interpreted, including the +// caveat that the pagination-total header can lag right after a push and cause an immediate +// iteration to stop after one page. +func (c *DatasetClient) IterateItems(options DatasetListItemsOptions, chunkSize *int64) *ListIterator[json.RawMessage] { + return IterateDatasetItems[json.RawMessage](c, options, chunkSize) +} + // DownloadItems downloads dataset items serialized in the given format, returning the raw // bytes. Unlike ListItems (parsed items), this returns the items already serialized to JSON, // CSV, XLSX, XML, RSS or HTML — useful for exporting. diff --git a/dataset_collection.go b/dataset_collection.go index a04d8a7..9cadc6a 100644 --- a/dataset_collection.go +++ b/dataset_collection.go @@ -18,6 +18,19 @@ func (c *DatasetCollectionClient) List(ctx context.Context, options StorageListO return listResource[Dataset](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the datasets matching the options, fetching pages on +// demand. The options' Limit caps the total number of datasets yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *DatasetCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[Dataset] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Dataset], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // GetOrCreate gets the dataset with the given name, creating it if it does not exist. An // empty name creates a new unnamed dataset. func (c *DatasetCollectionClient) GetOrCreate(ctx context.Context, name string) (Dataset, error) { diff --git a/docs/actors.md b/docs/actors.md index a133b47..4438ac7 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -9,6 +9,7 @@ Actors are the programs that run on the Apify platform. Access the Actor collect | Method | Description | | --- | --- | | `List(ctx, ActorListOptions) (PaginationList[Actor], error)` | List the account's Actors. | +| `Iterate(ActorListOptions, chunkSize *int64) *ListIterator[Actor]` | Lazy iterator over matching Actors. `Limit` caps the total yielded; `chunkSize` is the page size. | | `Create(ctx, definition any) (Actor, error)` | Create a new Actor. | `Create` takes a free-form definition (`any`) serialized to JSON, so the Actor's fields are @@ -92,6 +93,11 @@ The `Actor` value returned by `Get`/`Create`/`Update` and listed by `List`: | `Version(n) *ActorVersionClient` / `Versions() *ActorVersionCollectionClient` | Versions. | | `Webhooks() *WebhookCollectionClient` | This Actor's webhooks. | +> **Note — two different `Build`s.** `Actor.Build(ctx, versionNumber, ActorBuildOptions)` here +> *starts* a build of a version and returns the resulting `Build`. It is unrelated to the +> top-level accessor `client.Build(id)`, which returns a `*BuildClient` for inspecting an +> existing build by ID (see [builds.md](builds.md)). Same name, different jobs. + `ValidateInput` is equivalent to `ValidateInputForBuild(ctx, input, "")`: an empty `build` omits the parameter, so the API validates against the build tagged `latest` (per the API specification). Both return the raw JSON validation result from the API — a JSON object @@ -167,15 +173,27 @@ run, err := client.Actor("apify/hello-world").Call(ctx, ) ``` +> **How `Call`/`WaitForFinish` relate to `WithTimeout`.** The wait is done client-side by +> *polling*: the client repeatedly re-fetches the run, each poll being a separate HTTP request +> that asks the server to block for at most 60 seconds (the API's per-request wait cap). Because +> every poll returns within that 60-second server cap — comfortably inside the client's +> `WithTimeout` budget (default 360s, which bounds each individual request, not the total wait) — +> the overall wait is **not** cut off at 360s. It continues across as many polls as needed until +> the run reaches a terminal state. Passing `waitSecs == nil` therefore genuinely waits until the +> run finishes (bounded only by a very large internal cap of ~11.5 days, or by cancelling the +> `ctx` you pass in); a non-nil `waitSecs` bounds the total client-side wait instead. The distinct +> `ActorStartOptions.WaitForFinish` field is unrelated: it only controls the single server-side +> wait on the initial `Start` request (max 60s), not the client-side polling loop. + ## Versions and environment variables `client.Actor(id).Versions()` and `.Version(n)`: | Method | Description | | --- | --- | -| `Versions().List(ctx, ListOptions)` / `Versions().Create(ctx, def)` | List/create versions. | +| `Versions().List(ctx, ListOptions)` / `Versions().Iterate(ListOptions, chunkSize *int64)` / `Versions().Create(ctx, def)` | List/iterate/create versions. | | `Version(n).Get/Update/Delete(ctx)` | Manage a single version. | -| `Version(n).EnvVars().List(ctx)` / `.Create(ctx, ActorEnvVar)` | List/create env vars. | +| `Version(n).EnvVars().List(ctx)` / `.Iterate()` / `.Create(ctx, ActorEnvVar)` | List/iterate/create env vars. | | `Version(n).EnvVar(name).Get/Update/Delete(ctx)` | Manage a single env var. | `ActorEnvVar` fields: diff --git a/docs/builds.md b/docs/builds.md index 82b2304..6817fd6 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -8,6 +8,7 @@ single build with `client.Build(id)`, and an Actor's builds with `client.Actor(i | Method | Description | | --- | --- | | `List(ctx, ListOptions) (PaginationList[Build], error)` | List builds. | +| `Iterate(ListOptions, chunkSize *int64) *ListIterator[Build]` | Lazy iterator over matching builds. `Limit` caps the total yielded; `chunkSize` is the page size. | ## Single build diff --git a/docs/misc.md b/docs/misc.md index c22c2f6..b982de5 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -7,7 +7,7 @@ Browse public Actors with `client.Store()`: | Method | Description | | --- | --- | | `List(ctx, StoreListOptions) (PaginationList[ActorStoreListItem], error)` | One page of Store Actors. | -| `Iterate(StoreListOptions) *StoreActorIterator` | Lazy iterator over all matching Actors. | +| `Iterate(StoreListOptions, chunkSize *int64) *StoreActorIterator` | Lazy iterator over matching Actors. `Limit` caps the total yielded; `chunkSize` is the page size. | `StoreListOptions` (all fields optional): @@ -35,7 +35,8 @@ Each item is an `ActorStoreListItem`: | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | ```go -it := client.Store().Iterate(apify.StoreListOptions{Limit: apify.Ptr(int64(20)), Search: apify.Ptr("scraper")}) +// Limit caps the total number of Actors yielded; the second argument is the per-page size. +it := client.Store().Iterate(apify.StoreListOptions{Limit: apify.Ptr(int64(20)), Search: apify.Ptr("scraper")}, apify.Ptr(int64(10))) for { actor, err := it.Next(ctx) if err != nil { @@ -81,7 +82,10 @@ user, ok, err := client.Me().Get(ctx) if err != nil { log.Fatal(err) } -fmt.Println(user.Username, ok) +if !ok { + log.Fatal("account not found") +} +fmt.Println(user.Username) usage, err := client.Me().MonthlyUsage(ctx) if err != nil { diff --git a/docs/runs.md b/docs/runs.md index c01ca1f..9bce3aa 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -9,6 +9,7 @@ with `client.Run(id)`, and an Actor's or task's runs with `client.Actor(id).Runs | Method | Description | | --- | --- | | `List(ctx, ListOptions, RunListOptions) (PaginationList[ActorRun], error)` | List runs. | +| `Iterate(ListOptions, RunListOptions, chunkSize *int64) *ListIterator[ActorRun]` | Lazy iterator over matching runs. `Limit` caps the total yielded; `chunkSize` is the page size. | `RunListOptions`: @@ -72,7 +73,9 @@ The time filters apply only to Actor- and task-scoped collections. > (empty string) as "unset"; only `RunChargeOptions.Count` is a pointer. ```go -run, err := client.Run(runID).WaitForFinish(ctx, nil) // nil waits indefinitely +// nil polls until the run is terminal and is not cut off by WithTimeout (which bounds each +// poll request, not the total wait) — see docs/actors.md for the full explanation. +run, err := client.Run(runID).WaitForFinish(ctx, nil) if err != nil { log.Fatal(err) } diff --git a/docs/schedules.md b/docs/schedules.md index aee2800..859146d 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -8,6 +8,7 @@ Schedules start Actor or task runs at specified times. Access the schedule colle | Method | Description | | --- | --- | | `List(ctx, ListOptions) (PaginationList[Schedule], error)` | List the account's schedules. | +| `Iterate(ListOptions, chunkSize *int64) *ListIterator[Schedule]` | Lazy iterator over matching schedules. `Limit` caps the total yielded; `chunkSize` is the page size. | | `Create(ctx, definition any) (Schedule, error)` | Create a new schedule. | ### `Schedule` fields diff --git a/docs/storages.md b/docs/storages.md index 4b02727..e8ef9bf 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -6,20 +6,36 @@ Each is reachable both as a top-level resource and as a run's default storage ## Datasets -Collection: `client.Datasets()` — `List(ctx, StorageListOptions)`, -`GetOrCreate(ctx, name string)` (empty name → unnamed dataset). +Collection: `client.Datasets()` — `List(ctx, StorageListOptions) (PaginationList[Dataset], error)`, +`Iterate(StorageListOptions, chunkSize *int64) *ListIterator[Dataset]` (lazy iterator; `Limit` caps total, `chunkSize` is page size), +`GetOrCreate(ctx, name string) (Dataset, error)` (empty name → unnamed dataset). Single dataset: `client.Dataset(id)`: | Method | Description | | --- | --- | -| `Get / Update / Delete(ctx)` | CRUD. | +| `Get(ctx) (Dataset, bool, error)` | Fetch the dataset (`false` if it does not exist). | +| `Update(ctx, newFields any) (Dataset, error)` | Update the dataset. | +| `Delete(ctx) error` | Delete the dataset. | | `ListItems(ctx, DatasetListItemsOptions) (PaginationList[json.RawMessage], error)` | Read items. | +| `IterateItems(DatasetListItemsOptions, chunkSize *int64) *ListIterator[json.RawMessage]` | Lazy iterator over items (`Limit` caps total, `chunkSize` is page size); `IterateDatasetItems[T]` decodes into your type. | | `PushItems(ctx, items any) error` | Append one item or a slice of items. | | `DownloadItems(ctx, DownloadItemsFormat, DatasetDownloadOptions) ([]byte, error)` | Export items (JSON, JSONL, CSV, XLSX, XML, RSS, HTML — see the format constants below). | | `GetStatistics(ctx) (json.RawMessage, bool, error)` | Dataset statistics. | | `CreateItemsPublicURL(ctx, DatasetListItemsOptions, expiresInSecs *int64) (string, error)` | Signed public items URL. | +The `Dataset` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List`/`Iterate`: + +| Field | Type | Meaning | +|---|---|---| +| `ID` | `string` | Unique dataset ID. | +| `Name` | `string` | Dataset name (empty for unnamed datasets). | +| `UserID` | `string` | ID of the user who owns the dataset. | +| `CreatedAt` | `*time.Time` | When the dataset was created. | +| `ModifiedAt` | `*time.Time` | When the dataset was last modified. | +| `ItemCount` | `int64` | Number of items currently stored. | +| `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | + `DatasetListItemsOptions` controls filtering, projection, and pagination of the items (`ListItems`, `ListDatasetItems`, and the `Items` field of `DatasetDownloadOptions`). All fields are optional; the slice fields default to empty and the pointer fields to unset. @@ -65,6 +81,36 @@ for _, item := range page.Items { } ``` +For lazy, typed iteration over items across pages use the generic helper +`apify.IterateDatasetItems[T](dataset *DatasetClient, opts DatasetListItemsOptions, chunkSize *int64) *ListIterator[T]`. +It is the typed counterpart of the `IterateItems` method: `Limit` caps the total number of +items yielded and `chunkSize` sets the per-page size, but each item is decoded into your type +`T` instead of `json.RawMessage`. Advance it with `Next(ctx)`, which returns `nil` when the +iteration is exhausted: + +```go +// A struct matching the shape of your dataset items. +type Result struct { + Title string `json:"title"` +} + +// Limit caps the total items yielded; the last argument is the per-page size. +it := apify.IterateDatasetItems[Result](client.Dataset("DATASET_ID"), + apify.DatasetListItemsOptions{Limit: apify.Ptr(int64(1000))}, + apify.Ptr(int64(100)), +) +for { + item, err := it.Next(ctx) + if err != nil { + log.Fatal(err) + } + if item == nil { + break + } + fmt.Println(item.Title) +} +``` + `DownloadItems` takes a `DownloadItemsFormat`. The exported constants are: | Constant | Value | @@ -103,14 +149,17 @@ csv, _ := client.Dataset(ds.ID).DownloadItems(ctx, apify.FormatCSV, apify.Datase ## Key-value stores -Collection: `client.KeyValueStores()` — `List`, `GetOrCreate`. +Collection: `client.KeyValueStores()` — `List(ctx, StorageListOptions) (PaginationList[KeyValueStore], error)`, `Iterate(StorageListOptions, chunkSize *int64) *ListIterator[KeyValueStore]` (`Limit` caps total, `chunkSize` is page size), `GetOrCreate(ctx, name string) (KeyValueStore, error)`. Single store: `client.KeyValueStore(id)`: | Method | Description | | --- | --- | -| `Get / Update / Delete(ctx)` | CRUD. | -| `ListKeys(ctx, ListKeysOptions) (KeyValueStoreKeysPage, error)` | List keys. | +| `Get(ctx) (KeyValueStore, bool, error)` | Fetch the store (`false` if it does not exist). | +| `Update(ctx, newFields any) (KeyValueStore, error)` | Update the store. | +| `Delete(ctx) error` | Delete the store. | +| `ListKeys(ctx, ListKeysOptions) (KeyValueStoreKeysPage, error)` | List one page of keys. | +| `IterateKeys(ListKeysOptions, chunkSize *int64) *KeyValueStoreKeysIterator` | Lazy iterator over keys (cursor-based). `Limit` caps the total yielded; `chunkSize` is the page size; `Prefix`/`Collection`/`Signature` filter every page; `ExclusiveStartKey` sets where to start. | | `GetRecord(ctx, key) (*KeyValueStoreRecord, bool, error)` | Read a record. | | `GetRecordWithOptions(ctx, key, GetRecordOptions)` | Read with options. | | `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes. | @@ -121,6 +170,17 @@ Single store: `client.KeyValueStore(id)`: | `GetRecordPublicURL(ctx, key) (string, error)` | Signed public record URL. | | `CreateKeysPublicURL(ctx, expiresInSecs *int64) (string, error)` | Signed public key-list URL. | +The `KeyValueStore` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List`/`Iterate`: + +| Field | Type | Meaning | +|---|---|---| +| `ID` | `string` | Unique store ID. | +| `Name` | `string` | Store name (empty for unnamed stores). | +| `UserID` | `string` | ID of the user who owns the store. | +| `CreatedAt` | `*time.Time` | When the store was created. | +| `ModifiedAt` | `*time.Time` | When the store was last modified. | +| `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | + Option structs (all fields optional): | Struct | Field | Type | Meaning | @@ -168,17 +228,32 @@ rec, ok, _ := client.KeyValueStore(store.ID).GetRecord(ctx, "OUTPUT") if ok { fmt.Println(string(rec.Value)) } + +// Lazily iterate over every key in the store (cursor-based paging is handled internally). +keys := client.KeyValueStore(store.ID).IterateKeys(apify.ListKeysOptions{}, nil) +for { + key, err := keys.Next(ctx) + if err != nil { + log.Fatal(err) + } + if key == nil { + break + } + fmt.Println(key.Key, key.Size) +} ``` ## Request queues -Collection: `client.RequestQueues()` — `List`, `GetOrCreate`. +Collection: `client.RequestQueues()` — `List(ctx, StorageListOptions) (PaginationList[RequestQueue], error)`, `Iterate(StorageListOptions, chunkSize *int64) *ListIterator[RequestQueue]` (`Limit` caps total, `chunkSize` is page size), `GetOrCreate(ctx, name string) (RequestQueue, error)`. Single queue: `client.RequestQueue(id)`: | Method | Description | | --- | --- | -| `Get / Update / Delete(ctx)` | CRUD. | +| `Get(ctx) (RequestQueue, bool, error)` | Fetch the queue (`false` if it does not exist). | +| `Update(ctx, newFields any) (RequestQueue, error)` | Update the queue. | +| `Delete(ctx) error` | Delete the queue. | | `ListHead(ctx, limit *int64) (RequestQueueHead, error)` | Requests at the front. | | `AddRequest(ctx, RequestQueueRequest, forefront bool) (RequestQueueOperationInfo, error)` | Add a request. | | `GetRequest(ctx, id) (*RequestQueueRequest, bool, error)` | Read a request. | @@ -194,6 +269,18 @@ Single queue: `client.RequestQueue(id)`: | `UnlockRequests(ctx) (json.RawMessage, error)` | Release all locks held by this client (see `WithClientKey`). | | `WithClientKey(key string) *RequestQueueClient` | Pin a stable client key (required to unlock own locks). | +The `RequestQueue` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List`/`Iterate`: + +| Field | Type | Meaning | +|---|---|---| +| `ID` | `string` | Unique queue ID. | +| `Name` | `string` | Queue name (empty for unnamed queues). | +| `UserID` | `string` | ID of the user who owns the queue. | +| `CreatedAt` | `*time.Time` | When the queue was created. | +| `ModifiedAt` | `*time.Time` | When the queue was last modified. | +| `TotalRequestCount` | `int64` | Total number of requests ever added. | +| `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | + `RequestQueueRequest` is the request payload/record. `URL` is required; `ID` is assigned by the API (omit it on create): diff --git a/docs/tasks.md b/docs/tasks.md index 7aaabf9..69cfbd4 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -8,6 +8,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | Method | Description | | --- | --- | | `List(ctx, ListOptions) (PaginationList[Task], error)` | List the account's tasks. | +| `Iterate(ListOptions, chunkSize *int64) *ListIterator[Task]` | Lazy iterator over matching tasks. `Limit` caps the total yielded; `chunkSize` is the page size. | | `Create(ctx, definition any) (Task, error)` | Create a new task. | ### `Task` fields diff --git a/docs/webhooks.md b/docs/webhooks.md index 7043dd1..20beb81 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -9,6 +9,7 @@ Webhooks notify external services when events occur. Access the webhook collecti | Method | Description | | --- | --- | | `List(ctx, ListOptions) (PaginationList[Webhook], error)` | List webhooks. | +| `Iterate(ListOptions, chunkSize *int64) *ListIterator[Webhook]` | Lazy iterator over matching webhooks. `Limit` caps the total yielded; `chunkSize` is the page size. | | `Create(ctx, definition any) (Webhook, error)` | Create a webhook. | An Actor's or task's webhooks are also listable via `client.Actor(id).Webhooks()` / @@ -76,7 +77,7 @@ The same values apply to the ad-hoc `ActorStartOptions.Webhooks` element (see | Method | Description | | --- | --- | -| `WebhookDispatches().List(ctx, ListOptions)` | List dispatches. | +| `WebhookDispatches().List(ctx, ListOptions)` / `WebhookDispatches().Iterate(ListOptions, chunkSize *int64)` | List/iterate dispatches. | | `WebhookDispatch(id).Get(ctx) (WebhookDispatch, bool, error)` | Fetch a dispatch. | ```go diff --git a/examples/iterate_store/main.go b/examples/iterate_store/main.go index 7f1aa7e..7c11440 100644 --- a/examples/iterate_store/main.go +++ b/examples/iterate_store/main.go @@ -19,9 +19,11 @@ func main() { client := apify.NewClient(os.Getenv("APIFY_TOKEN")) ctx := context.Background() - // Iterate the store lazily, fetching pages of 5 on demand. - limit := int64(5) - it := client.Store().Iterate(apify.StoreListOptions{Limit: &limit}) + // Iterate the store lazily, fetching pages of 5 on demand (chunkSize). The options' Limit + // would cap the total number of Actors yielded; here it is left unset so iteration would + // cover the whole store, and the loop below stops after the first few. + chunkSize := int64(5) + it := client.Store().Iterate(apify.StoreListOptions{}, &chunkSize) const want = 10 for i := 0; i < want; i++ { diff --git a/http_client.go b/http_client.go index 2f30805..261839e 100644 --- a/http_client.go +++ b/http_client.go @@ -150,9 +150,16 @@ func (c *httpClient) callWithHeaders(ctx context.Context, method, url string, bo return nil, lastErr } - // Sleep with randomized exponential backoff before the next attempt. The backoff - // doubles each retry (matching the reference client's async-retry factor of 2) and - // is capped at the overall request timeout. + // Sleep with exponential backoff before the next attempt. The base delay doubles each + // retry (matching the reference client's async-retry factor of 2). Two intentional and + // documented divergences from the reference JS client: + // - Jitter: the actual sleep is drawn uniformly from [delay, 2*delay). The reference + // does not jitter, but the Apify API docs explicitly prescribe it ("wait for a period + // of time chosen randomly from the interval DELAY to 2*DELAY milliseconds") to avoid + // synchronized retries (thundering herd), so we follow the API guidance here. + // - Cap: the base delay is capped at the overall request timeout as a safety net (the + // reference leaves it uncapped). With the default settings this cap is never reached + // within maxRetries, so it does not change observable behaviour. if !sleepWithContext(ctx, randomizedDelay(delay)) { return nil, ctx.Err() } diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000..975ab79 --- /dev/null +++ b/iterator.go @@ -0,0 +1,149 @@ +package apify + +import "context" + +// ListIterator lazily iterates over an offset/limit-paginated collection, fetching one page +// at a time on demand. Obtain one from a collection client's Iterate method and drain it by +// calling Next until it returns (nil, nil). +// +// Its end-user semantics match the reference JS client's iterable list(): the list options' +// Limit is a cap on the total number of items yielded across all pages (unset means "all +// matching items"), the page size is the separate chunkSize argument passed to Iterate (unset +// means the server default), and a caller-set Offset on the options is honored as the starting +// point (iteration begins there and yields at most Limit items from that offset onward). This +// keeps the two clients consistent for callers reasoning about offset/limit/chunk behaviour. +type ListIterator[T any] struct { + // fetch fetches one page starting at offset. limit is the per-page limit to request + // (0 means "unset", i.e. let the server choose). The collection's Iterate method bakes the + // filters into this closure and overrides offset/limit per page. + fetch func(ctx context.Context, offset, limit int64) (PaginationList[T], error) + // limit caps the total number of items yielded (nil or <=0 means no cap). + limit *int64 + // chunkSize is the per-page size (nil or <=0 means the server default). + chunkSize *int64 + + buffer []T + pos int + startOffset int64 // offset the caller asked iteration to start from (0 when unset) + offset int64 + remaining int64 // items still allowed to be yielded after the current buffer; valid once started + started bool + exhausted bool +} + +// newListIterator builds a ListIterator from a page-fetch closure, the total-item cap (limit), +// the per-page size (chunkSize) and the starting offset (startOffset, the caller's Offset, 0 +// when unset). It is the single constructor behind every collection's Iterate helper, keeping +// the paging logic in one place (DRY). +func newListIterator[T any](limit, chunkSize *int64, startOffset int64, fetch func(ctx context.Context, offset, limit int64) (PaginationList[T], error)) *ListIterator[T] { + return &ListIterator[T]{fetch: fetch, limit: limit, chunkSize: chunkSize, startOffset: startOffset, offset: startOffset} +} + +// Next returns the next item, or (nil, nil) once the collection (or the total-item cap) is +// exhausted. It calls the API for another page only when the current in-memory page is used up. +func (it *ListIterator[T]) Next(ctx context.Context) (*T, error) { + for it.pos >= len(it.buffer) { + if it.exhausted { + return nil, nil + } + if err := it.loadPage(ctx); err != nil { + return nil, err + } + } + item := it.buffer[it.pos] + it.pos++ + return &item, nil +} + +// loadPage loads the next page into the buffer, following the same offset/limit/chunkSize +// arithmetic as the reference client's _listPaginated: the first page requests +// min(limit, chunkSize) items; the total cap is then bounded by the reported total, and each +// subsequent page requests min(remaining, chunkSize) items. Iteration ends when a page comes +// back empty or the remaining cap reaches zero. +func (it *ListIterator[T]) loadPage(ctx context.Context) error { + var limitParam int64 + if !it.started { + limitParam = minForLimitParam(it.limitVal(), it.chunkVal()) + } else { + limitParam = minForLimitParam(it.remaining, it.chunkVal()) + } + + page, err := it.fetch(ctx, it.offset, limitParam) + if err != nil { + return err + } + n := int64(len(page.Items)) + it.buffer = page.Items + it.pos = 0 + it.offset += n + + if !it.started { + it.started = true + // Cap the number of items to yield: from the start offset onward at most + // (Total - startOffset) items remain, and no more than the caller's Limit. This mirrors + // the reference client's remainingItems = min(total - offset, limit) - firstPageCount, + // so a caller-set Offset is honored as the starting point and a Limit larger than the + // collection still yields everything from the offset onward. + capItems := page.Total - it.startOffset + if capItems < 0 { + capItems = 0 + } + if l := it.limitVal(); l > 0 && l < capItems { + capItems = l + } + it.remaining = capItems - n + } else { + it.remaining -= n + } + + if n == 0 || it.remaining <= 0 { + it.exhausted = true + } + return nil +} + +// limitVal returns the total-item cap as a plain int64 (0 when unset). +func (it *ListIterator[T]) limitVal() int64 { + if it.limit == nil { + return 0 + } + return *it.limit +} + +// chunkVal returns the per-page size as a plain int64 (0 when unset). +func (it *ListIterator[T]) chunkVal() int64 { + if it.chunkSize == nil { + return 0 + } + return *it.chunkSize +} + +// minForLimitParam mirrors the reference client's minForLimitParam: it treats 0 as "unset" and +// returns the smaller of the two defined values, or 0 when both are unset. +func minForLimitParam(a, b int64) int64 { + if a <= 0 { + return maxInt64(b, 0) + } + if b <= 0 { + return a + } + return minInt64(a, b) +} + +// pageLimitPtr converts a per-page limit into the pointer the list options expect: a positive +// value is sent as the page's Limit; 0 leaves it unset so the server default applies. +func pageLimitPtr(limit int64) *int64 { + if limit > 0 { + return &limit + } + return nil +} + +// offsetVal reads a caller-set starting offset (from a list options struct) into a plain int64, +// treating nil or a negative value as 0 ("start from the beginning"). +func offsetVal(offset *int64) int64 { + if offset == nil || *offset < 0 { + return 0 + } + return *offset +} diff --git a/iterator_test.go b/iterator_test.go new file mode 100644 index 0000000..8eea138 --- /dev/null +++ b/iterator_test.go @@ -0,0 +1,228 @@ +package apify + +import ( + "context" + "testing" +) + +// stubFetcher returns a page-fetch closure over a synthetic collection of `total` sequential +// ints. It honors offset and the requested page limit (limit <= 0 means "server default", +// modeled by defaultPage), and reports the true Total. Requested (offset, limit) pairs are +// appended to calls so tests can assert the paging arithmetic. This keeps the ListIterator +// unit tests hermetic — no network or APIFY_TOKEN required. +func stubFetcher(total int, defaultPage int64, calls *[][2]int64) func(context.Context, int64, int64) (PaginationList[int], error) { + return func(_ context.Context, offset, limit int64) (PaginationList[int], error) { + if calls != nil { + *calls = append(*calls, [2]int64{offset, limit}) + } + pageLen := limit + if pageLen <= 0 { + pageLen = defaultPage + } + var items []int + for i := offset; i < offset+pageLen && i < int64(total); i++ { + items = append(items, int(i)) + } + return PaginationList[int]{ + Total: int64(total), + Offset: offset, + Limit: pageLen, + Count: int64(len(items)), + Items: items, + }, nil + } +} + +// boundedFetcher models two server behaviors the plain stubFetcher cannot: (a) a server-side +// page cap (serverPage), so a page can come back non-empty but shorter than requested, and +// (b) an over-reported / lagging Total (reportedTotal), which may exceed the real backing size. +// It backs a collection of `backing` sequential ints and appends each requested (offset,limit) +// to calls. +func boundedFetcher(backing int, reportedTotal, serverPage int64, calls *[][2]int64) func(context.Context, int64, int64) (PaginationList[int], error) { + return func(_ context.Context, offset, limit int64) (PaginationList[int], error) { + if calls != nil { + *calls = append(*calls, [2]int64{offset, limit}) + } + pageLen := limit + if pageLen <= 0 { + pageLen = serverPage + } + if serverPage > 0 && pageLen > serverPage { + pageLen = serverPage + } + var items []int + for i := offset; i < offset+pageLen && i < int64(backing); i++ { + items = append(items, int(i)) + } + return PaginationList[int]{ + Total: reportedTotal, + Offset: offset, + Limit: pageLen, + Count: int64(len(items)), + Items: items, + }, nil + } +} + +// drainInts fully drains an int iterator, with a hard safety bound so a broken termination +// condition fails fast instead of hanging the test. +func drainInts(t *testing.T, it *ListIterator[int]) []int { + t.Helper() + var out []int + for len(out) <= 100000 { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatalf("Next: %v", err) + } + if item == nil { + return out + } + out = append(out, *item) + } + t.Fatal("iterator did not terminate") + return nil +} + +func TestListIteratorTotalCap(t *testing.T) { + // Limit=3 (total cap), page size 2, backing collection of 10 → exactly 3 items across 2 pages. + var calls [][2]int64 + it := newListIterator(ptrInt64(3), ptrInt64(2), 0, stubFetcher(10, 1000, &calls)) + got := drainInts(t, it) + if len(got) != 3 { + t.Fatalf("expected 3 items (Limit cap), got %d: %v", len(got), got) + } + // First page requests min(limit,chunk)=2 at offset 0; second requests min(remaining=1,chunk)=1 at offset 2. + want := [][2]int64{{0, 2}, {2, 1}} + if len(calls) != len(want) { + t.Fatalf("expected %d page fetches, got %d: %v", len(want), len(calls), calls) + } + for i, w := range want { + if calls[i] != w { + t.Fatalf("page %d: requested (offset,limit)=%v, want %v (all calls: %v)", i, calls[i], w, calls) + } + } +} + +func TestListIteratorNoCapPagesAll(t *testing.T) { + // No total cap, page size 2, backing 5 → all 5 items across 3 pages (2+2+1). + var calls [][2]int64 + it := newListIterator(nil, ptrInt64(2), 0, stubFetcher(5, 1000, &calls)) + got := drainInts(t, it) + if len(got) != 5 { + t.Fatalf("expected all 5 items, got %d: %v", len(got), got) + } + for i, v := range got { + if v != i { + t.Fatalf("item %d = %d, want %d (items: %v)", i, v, i, got) + } + } + if len(calls) != 3 { + t.Fatalf("expected 3 page fetches (2+2+1), got %d: %v", len(calls), calls) + } +} + +func TestListIteratorServerDefaultPage(t *testing.T) { + // No cap and no chunk size → the page limit is left unset (0) so the server default applies. + var calls [][2]int64 + it := newListIterator(nil, nil, 0, stubFetcher(3, 1000, &calls)) + got := drainInts(t, it) + if len(got) != 3 { + t.Fatalf("expected 3 items, got %d", len(got)) + } + if calls[0][1] != 0 { + t.Fatalf("first page should request limit 0 (server default), got %d", calls[0][1]) + } +} + +func TestListIteratorLimitLargerThanTotal(t *testing.T) { + // A Limit larger than the collection yields the whole collection (cap bounded by Total). + it := newListIterator(ptrInt64(100), nil, 0, stubFetcher(4, 1000, nil)) + if got := drainInts(t, it); len(got) != 4 { + t.Fatalf("expected 4 items (whole collection), got %d", len(got)) + } +} + +func TestListIteratorEmptyCollection(t *testing.T) { + it := newListIterator(nil, ptrInt64(2), 0, stubFetcher(0, 1000, nil)) + if got := drainInts(t, it); len(got) != 0 { + t.Fatalf("expected 0 items, got %d", len(got)) + } +} + +func TestListIteratorHonorsStartOffset(t *testing.T) { + // Caller-set Offset=4 on a backing collection of 10, page size 3: iteration must start at + // item 4 and yield 4..9 (6 items), mirroring the reference's options.offset start point. + var calls [][2]int64 + it := newListIterator(nil, ptrInt64(3), 4, stubFetcher(10, 1000, &calls)) + got := drainInts(t, it) + want := []int{4, 5, 6, 7, 8, 9} + if len(got) != len(want) { + t.Fatalf("expected %v (from offset 4), got %v", want, got) + } + for i, v := range want { + if got[i] != v { + t.Fatalf("item %d = %d, want %d (items: %v)", i, got[i], v, got) + } + } + // The first page must be requested at the caller's offset, not 0. + if calls[0][0] != 4 { + t.Fatalf("first page requested at offset %d, want 4 (calls: %v)", calls[0][0], calls) + } +} + +func TestListIteratorStartOffsetWithLimit(t *testing.T) { + // Offset=4 with a total cap of 3 must yield exactly items 4,5,6 (cap counts from the offset). + it := newListIterator(ptrInt64(3), ptrInt64(2), 4, stubFetcher(10, 1000, nil)) + got := drainInts(t, it) + want := []int{4, 5, 6} + if len(got) != len(want) { + t.Fatalf("expected %v (offset 4, cap 3), got %v", want, got) + } + for i, v := range want { + if got[i] != v { + t.Fatalf("item %d = %d, want %d (items: %v)", i, got[i], v, got) + } + } +} + +func TestListIteratorNonFinalShortPage(t *testing.T) { + // The server caps every page at 2 items even when the iterator requests a larger chunk (10), + // so pages 1 and 2 are non-final yet shorter than requested. Iteration must keep paging (via + // the reported Total) and still yield the whole collection rather than stopping early on the + // first short page. + var calls [][2]int64 + it := newListIterator(nil, ptrInt64(10), 0, boundedFetcher(6, 6, 2, &calls)) + got := drainInts(t, it) + if len(got) != 6 { + t.Fatalf("expected all 6 items despite short pages, got %d: %v", len(got), got) + } + for i, v := range got { + if v != i { + t.Fatalf("item %d = %d, want %d (items: %v)", i, v, i, got) + } + } + // Pages at offsets 0,2,4 return 2 items each (remaining reaches 0 after the third), so the + // iterator must not issue a needless fourth (empty) fetch. + if len(calls) != 3 { + t.Fatalf("expected 3 page fetches (2+2+2), got %d: %v", len(calls), calls) + } +} + +func TestListIteratorOverReportedTotalTerminates(t *testing.T) { + // The endpoint over-reports Total=100 but the collection really holds 3 items (e.g. a lagging + // pagination-total header). Once a page comes back empty the iterator must stop instead of + // looping toward the phantom cap. + var calls [][2]int64 + it := newListIterator(nil, ptrInt64(2), 0, boundedFetcher(3, 100, 10, &calls)) + got := drainInts(t, it) + if len(got) != 3 { + t.Fatalf("expected exactly 3 items (real backing size), got %d: %v", len(got), got) + } + // offset 0 -> 2 items, offset 2 -> 1 item, offset 3 -> empty (terminates). + if len(calls) != 3 { + t.Fatalf("expected 3 page fetches, got %d: %v", len(calls), calls) + } +} + +// ptrInt64 is a local helper for the pointer-typed limit/chunkSize arguments. +func ptrInt64(v int64) *int64 { return &v } diff --git a/key_value_store.go b/key_value_store.go index 455e220..0f025d4 100644 --- a/key_value_store.go +++ b/key_value_store.go @@ -98,6 +98,114 @@ func (c *KeyValueStoreClient) ListKeys(ctx context.Context, options ListKeysOpti return getResourceRequired[KeyValueStoreKeysPage](ctx, c.ctx, "keys", params) } +// IterateKeys returns a lazy iterator over the store's keys, fetching one page at a time on +// demand via the cursor-based keys endpoint (exclusiveStartKey / nextExclusiveStartKey). It +// mirrors the reference client's async-iterable listKeys() and follows this client's iteration +// convention (like the collection Iterate helpers): the options' Limit caps the total number of +// keys yielded across all pages (unset means all keys), the per-page size is the separate +// chunkSize argument (nil for the server default), Prefix/Collection/Signature filter every +// page, and ExclusiveStartKey sets the key to start listing after. +// +// Because keys are cursor-paginated (not offset/limit paginated) it uses its own +// KeyValueStoreKeysIterator rather than the generic ListIterator, sharing the cursor mechanics +// of RequestQueueClient.PaginateRequests. +func (c *KeyValueStoreClient) IterateKeys(options ListKeysOptions, chunkSize *int64) *KeyValueStoreKeysIterator { + return &KeyValueStoreKeysIterator{client: c, options: options, chunkSize: chunkSize, nextStartKey: options.ExclusiveStartKey} +} + +// KeyValueStoreKeysIterator lazily iterates over a key-value store's keys, fetching one page at +// a time via the cursor-based listing endpoint. Obtain one from KeyValueStoreClient.IterateKeys +// and drain it by calling Next until it returns (nil, nil). +type KeyValueStoreKeysIterator struct { + client *KeyValueStoreClient + options ListKeysOptions + chunkSize *int64 // per-page size (nil or <=0 means the server default) + + buffer []KeyValueStoreKey + pos int + nextStartKey *string // cursor for the next page (nil once the API reports no more keys) + remaining int64 // total-item cap countdown; <0 means "no cap". Valid once started. + started bool + exhausted bool +} + +// Next returns the next key, or (nil, nil) when the iterator is exhausted (no more keys or the +// total-item cap is reached). It calls the API for another page only when the current in-memory +// page is used up. +func (it *KeyValueStoreKeysIterator) Next(ctx context.Context) (*KeyValueStoreKey, error) { + for it.pos >= len(it.buffer) { + if it.exhausted { + return nil, nil + } + if err := it.fetchPage(ctx); err != nil { + return nil, err + } + } + key := it.buffer[it.pos] + it.pos++ + return &key, nil +} + +// fetchPage loads the next page of keys into the buffer, advancing the cursor and the remaining +// cap. The per-page limit is the smaller of the remaining total cap and the requested chunk +// size, so the final page is never over-fetched. +func (it *KeyValueStoreKeysIterator) fetchPage(ctx context.Context) error { + opts := it.options + opts.ExclusiveStartKey = it.nextStartKey + + // capLeft is how many items the total cap still allows (0 = "no cap so far", treated as + // unset when combined with the chunk size). + var capLeft int64 + if !it.started { + if l := it.options.Limit; l != nil && *l > 0 { + capLeft = *l + } + } else if it.remaining > 0 { + capLeft = it.remaining + } + opts.Limit = pageLimitPtr(minForLimitParam(capLeft, it.chunkVal())) + + page, err := it.client.ListKeys(ctx, opts) + if err != nil { + return err + } + n := int64(len(page.Items)) + it.buffer = page.Items + it.pos = 0 + + if page.NextExclusiveStartKey != "" { + it.nextStartKey = &page.NextExclusiveStartKey + } else { + it.nextStartKey = nil + } + + if !it.started { + it.started = true + if l := it.options.Limit; l != nil && *l > 0 { + it.remaining = *l - n + } else { + it.remaining = -1 // no cap + } + } else if it.remaining >= 0 { + it.remaining -= n + } + + // Stop when the API returns an empty page, reports no more keys (not truncated / no next + // cursor), or the total-item cap has been reached. + if n == 0 || !page.IsTruncated || it.nextStartKey == nil || it.remaining == 0 { + it.exhausted = true + } + return nil +} + +// chunkVal returns the per-page size as a plain int64 (0 when unset, i.e. server default). +func (it *KeyValueStoreKeysIterator) chunkVal() int64 { + if it.chunkSize == nil || *it.chunkSize < 0 { + return 0 + } + return *it.chunkSize +} + // GetRecords downloads all records of the store (optionally filtered) as a ZIP archive, // returning the raw bytes. func (c *KeyValueStoreClient) GetRecords(ctx context.Context, options GetRecordsOptions) ([]byte, error) { diff --git a/key_value_store_collection.go b/key_value_store_collection.go index 5136bd1..7fc01d4 100644 --- a/key_value_store_collection.go +++ b/key_value_store_collection.go @@ -19,6 +19,19 @@ func (c *KeyValueStoreCollectionClient) List(ctx context.Context, options Storag return listResource[KeyValueStore](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the key-value stores matching the options, fetching +// pages on demand. The options' Limit caps the total number of stores yielded (unset means +// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference +// client's iterable list(). +func (c *KeyValueStoreCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[KeyValueStore] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[KeyValueStore], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // GetOrCreate gets the store with the given name, creating it if it does not exist. An // empty name creates a new unnamed store. func (c *KeyValueStoreCollectionClient) GetOrCreate(ctx context.Context, name string) (KeyValueStore, error) { diff --git a/models.go b/models.go index 81bbe55..3569d75 100644 --- a/models.go +++ b/models.go @@ -5,8 +5,10 @@ import ( "time" ) -// Extra is the catch-all map of unmodelled JSON fields. Every resource model carries one -// so that additive changes to the API never break deserialization (forward compatibility). +// Extra is the catch-all map of unmodelled JSON fields. Most resource models carry one so +// that unknown fields are preserved rather than dropped. Forward compatibility with additive +// API fields holds for every model regardless: the client never sets DisallowUnknownFields, +// so encoding/json silently ignores fields a model does not declare. type Extra = map[string]json.RawMessage // unmarshalWithExtra unmarshals data into the typed value v (which must be a pointer to a diff --git a/request_queue_collection.go b/request_queue_collection.go index 8d58dbe..0a88cfb 100644 --- a/request_queue_collection.go +++ b/request_queue_collection.go @@ -19,6 +19,19 @@ func (c *RequestQueueCollectionClient) List(ctx context.Context, options Storage return listResource[RequestQueue](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the request queues matching the options, fetching +// pages on demand. The options' Limit caps the total number of queues yielded (unset means +// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference +// client's iterable list(). +func (c *RequestQueueCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[RequestQueue] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[RequestQueue], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // GetOrCreate gets the queue with the given name, creating it if it does not exist. An // empty name creates a new unnamed queue. func (c *RequestQueueCollectionClient) GetOrCreate(ctx context.Context, name string) (RequestQueue, error) { diff --git a/run_collection.go b/run_collection.go index e4e80e1..a801c9f 100644 --- a/run_collection.go +++ b/run_collection.go @@ -39,3 +39,16 @@ func (c *RunCollectionClient) List(ctx context.Context, options ListOptions, fil filter.apply(params) return listResource[ActorRun](ctx, c.ctx, "", params) } + +// Iterate returns a lazy iterator over the runs matching the options and filter, fetching +// pages on demand. The options' Limit caps the total number of runs yielded (unset means all); +// the per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *RunCollectionClient) Iterate(options ListOptions, filter RunListOptions, chunkSize *int64) *ListIterator[ActorRun] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorRun], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts, filter) + }) +} diff --git a/schedule.go b/schedule.go index 40ae741..cbad8e8 100644 --- a/schedule.go +++ b/schedule.go @@ -19,6 +19,19 @@ func (c *ScheduleCollectionClient) List(ctx context.Context, options ListOptions return listResource[Schedule](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the schedules matching the options, fetching pages on +// demand. The options' Limit caps the total number of schedules yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *ScheduleCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Schedule] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Schedule], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // Create creates a new schedule. schedule is any JSON-serializable schedule definition. func (c *ScheduleCollectionClient) Create(ctx context.Context, schedule any) (Schedule, error) { return createResource[Schedule](ctx, c.ctx, NewQueryParams(), schedule) diff --git a/store_collection.go b/store_collection.go index 8feb228..3ba2f90 100644 --- a/store_collection.go +++ b/store_collection.go @@ -55,55 +55,19 @@ func (c *StoreCollectionClient) List(ctx context.Context, options StoreListOptio return listResource[ActorStoreListItem](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over all Store Actors matching the options, fetching pages -// on demand. The options' Limit (if set) is used as the per-page size. -func (c *StoreCollectionClient) Iterate(options StoreListOptions) *StoreActorIterator { - return &StoreActorIterator{client: c, options: options} -} - // StoreActorIterator lazily iterates over Apify Store Actors, fetching one page at a time. -type StoreActorIterator struct { - client *StoreCollectionClient - options StoreListOptions - - buffer []ActorStoreListItem - pos int - offset int64 - total int64 - exhausted bool -} - -// Next returns the next Store Actor, or (nil, nil) when the iterator is exhausted. -func (it *StoreActorIterator) Next(ctx context.Context) (*ActorStoreListItem, error) { - for it.pos >= len(it.buffer) { - if it.exhausted { - return nil, nil - } - if err := it.fetchPage(ctx); err != nil { - return nil, err - } - } - item := it.buffer[it.pos] - it.pos++ - return &item, nil -} - -// fetchPage loads the next page of Store Actors into the buffer. -func (it *StoreActorIterator) fetchPage(ctx context.Context) error { - opts := it.options - opts.Offset = &it.offset - page, err := it.client.List(ctx, opts) - if err != nil { - return err - } - it.buffer = page.Items - it.pos = 0 - it.total = page.Total - it.offset += int64(len(page.Items)) +// It is the generic [ListIterator] specialized to Store Actors; drain it with Next. +type StoreActorIterator = ListIterator[ActorStoreListItem] - // Exhausted when the page is empty or we have reached the reported total. - if len(page.Items) == 0 || it.offset >= it.total { - it.exhausted = true - } - return nil +// Iterate returns a lazy iterator over the Store Actors matching the options, fetching pages +// on demand. The options' Limit caps the total number of Actors yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *StoreCollectionClient) Iterate(options StoreListOptions, chunkSize *int64) *StoreActorIterator { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorStoreListItem], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) } diff --git a/task_collection.go b/task_collection.go index 2020e3f..4fe4138 100644 --- a/task_collection.go +++ b/task_collection.go @@ -19,6 +19,19 @@ func (c *TaskCollectionClient) List(ctx context.Context, options ListOptions) (P return listResource[Task](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the tasks matching the options, fetching pages on +// demand. The options' Limit caps the total number of tasks yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *TaskCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Task] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Task], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // Create creates a new task. task is any JSON-serializable task definition. func (c *TaskCollectionClient) Create(ctx context.Context, task any) (Task, error) { return createResource[Task](ctx, c.ctx, NewQueryParams(), task) diff --git a/tests/iterate_test.go b/tests/iterate_test.go new file mode 100644 index 0000000..af37140 --- /dev/null +++ b/tests/iterate_test.go @@ -0,0 +1,449 @@ +package apify_test + +import ( + "context" + "testing" + "time" + + apify "github.com/apify/apify-client-go" +) + +// iterFindAttempts / iterFindBackoff bound how long iterateFindIDs waits for a freshly +// created resource to surface in a collection listing. Collection list indexes are eventually +// consistent, so a resource can be missing from the first listing right after creation; the +// helper re-iterates a fresh iterator until it converges. +const ( + iterFindAttempts = 8 + iterFindBackoff = time.Second +) + +// iterateFindIDs repeatedly drains a fresh iterator (from makeIter) and checks that every id +// in want is yielded. Within one pass it stops early once all wanted ids are seen and gives up +// after safetyCap items so a shared account with many pre-existing resources cannot make the +// test iterate unboundedly. Across passes it retries with backoff to absorb list-index +// eventual consistency. A small per-page Limit combined with several wanted ids exercises +// paging across more than one page. +func iterateFindIDs[T any](t *testing.T, ctx context.Context, makeIter func() *apify.ListIterator[T], idOf func(*T) string, want []string, safetyCap int) { + t.Helper() + missing := make(map[string]bool, len(want)) + for _, id := range want { + missing[id] = true + } + for attempt := 0; attempt < iterFindAttempts; attempt++ { + it := makeIter() + seen := 0 + for { + item, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if item == nil { + break + } + seen++ + if id := idOf(item); missing[id] { + delete(missing, id) + if len(missing) == 0 { + return + } + } + if safetyCap > 0 && seen >= safetyCap { + break + } + } + if len(missing) == 0 { + return + } + time.Sleep(iterFindBackoff) + } + t.Fatalf("iteration did not yield all created resources; missing %v", missing) +} + +// smallPageDesc returns list options sorted newest-first (Desc), so just-created resources +// appear near the front of the listing. It only sets the sort order; the page size is controlled +// separately by the chunkSize argument passed to Iterate at each call site (a small chunkSize is +// what forces iteration across multiple pages). +func smallPageDesc() apify.ListOptions { + return apify.ListOptions{Desc: ptr(true)} +} + +func TestIterateActors(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + created, err := client.Actors().Create(ctx, minimalActor(uniqueName("iter-actor"))) + if err != nil { + t.Fatalf("create actor %d: %v", i, err) + } + defer func(id string) { _ = client.Actor(id).Delete(ctx) }(created.ID) + want = append(want, created.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.Actor] { + return client.Actors().Iterate(apify.ActorListOptions{My: ptr(true), Desc: ptr(true)}, ptr(int64(1))) + }, func(a *apify.Actor) string { return a.ID }, want, 500) +} + +func TestIterateTasks(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + task, err := client.Tasks().Create(ctx, taskDef(uniqueName("iter-task"))) + if err != nil { + t.Fatalf("create task %d: %v", i, err) + } + defer func(id string) { _ = client.Task(id).Delete(ctx) }(task.ID) + want = append(want, task.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.Task] { + return client.Tasks().Iterate(smallPageDesc(), ptr(int64(1))) + }, func(x *apify.Task) string { return x.ID }, want, 500) +} + +func TestIterateSchedules(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + sch, err := client.Schedules().Create(ctx, scheduleDef(uniqueName("iter-sch"))) + if err != nil { + t.Fatalf("create schedule %d: %v", i, err) + } + defer func(id string) { _ = client.Schedule(id).Delete(ctx) }(sch.ID) + want = append(want, sch.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.Schedule] { + return client.Schedules().Iterate(smallPageDesc(), ptr(int64(1))) + }, func(s *apify.Schedule) string { return s.ID }, want, 500) +} + +func TestIterateWebhooks(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + wh, err := client.Webhooks().Create(ctx, webhookDef("https://example.com/iter-webhook")) + if err != nil { + t.Fatalf("create webhook %d: %v", i, err) + } + defer func(id string) { _ = client.Webhook(id).Delete(ctx) }(wh.ID) + want = append(want, wh.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.Webhook] { + return client.Webhooks().Iterate(smallPageDesc(), ptr(int64(1))) + }, func(w *apify.Webhook) string { return w.ID }, want, 500) +} + +func TestIterateDatasets(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + ds, err := client.Datasets().GetOrCreate(ctx, uniqueName("iter-ds")) + if err != nil { + t.Fatalf("create dataset %d: %v", i, err) + } + defer func(id string) { _ = client.Dataset(id).Delete(ctx) }(ds.ID) + want = append(want, ds.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.Dataset] { + return client.Datasets().Iterate(apify.StorageListOptions{Desc: ptr(true)}, ptr(int64(1))) + }, func(d *apify.Dataset) string { return d.ID }, want, 500) +} + +func TestIterateKeyValueStores(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + store, err := client.KeyValueStores().GetOrCreate(ctx, uniqueName("iter-kvs")) + if err != nil { + t.Fatalf("create store %d: %v", i, err) + } + defer func(id string) { _ = client.KeyValueStore(id).Delete(ctx) }(store.ID) + want = append(want, store.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.KeyValueStore] { + return client.KeyValueStores().Iterate(apify.StorageListOptions{Desc: ptr(true)}, ptr(int64(1))) + }, func(s *apify.KeyValueStore) string { return s.ID }, want, 500) +} + +func TestIterateRequestQueues(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var want []string + for i := 0; i < 3; i++ { + rq, err := client.RequestQueues().GetOrCreate(ctx, uniqueName("iter-rq")) + if err != nil { + t.Fatalf("create queue %d: %v", i, err) + } + defer func(id string) { _ = client.RequestQueue(id).Delete(ctx) }(rq.ID) + want = append(want, rq.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.RequestQueue] { + return client.RequestQueues().Iterate(apify.StorageListOptions{Desc: ptr(true)}, ptr(int64(1))) + }, func(q *apify.RequestQueue) string { return q.ID }, want, 500) +} + +func TestIterateKeyValueStoreKeys(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + store, err := client.KeyValueStores().GetOrCreate(ctx, uniqueName("iter-keys")) + if err != nil { + t.Fatalf("get-or-create store: %v", err) + } + defer func() { _ = client.KeyValueStore(store.ID).Delete(ctx) }() + kvs := client.KeyValueStore(store.ID) + + want := []string{"alpha", "beta", "gamma", "delta"} + for _, key := range want { + if err := kvs.SetRecordJSON(ctx, key, map[string]any{"k": key}); err != nil { + t.Fatalf("set record %s: %v", key, err) + } + } + + // Iterate all keys with a small per-page chunkSize so cursor paging crosses more than one page. + it := kvs.IterateKeys(apify.ListKeysOptions{}, ptr(int64(2))) + seen := make(map[string]bool) + for { + key, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate keys: %v", err) + } + if key == nil { + break + } + seen[key.Key] = true + } + for _, key := range want { + if !seen[key] { + t.Fatalf("key %q was not yielded by IterateKeys (seen: %v)", key, seen) + } + } +} + +func TestIterateActorVersions(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + created, err := client.Actors().Create(ctx, minimalActor(uniqueName("iter-ver"))) + if err != nil { + t.Fatalf("create actor: %v", err) + } + defer func() { _ = client.Actor(created.ID).Delete(ctx) }() + actor := client.Actor(created.ID) + + // The actor is created with version "0.0"; add a second version so paging spans >1 page. + if _, err := actor.Versions().Create(ctx, map[string]any{ + "versionNumber": "0.1", + "sourceType": "SOURCE_FILES", + "buildTag": "latest", + "sourceFiles": []any{}, + }); err != nil { + t.Fatalf("create version: %v", err) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.ActorVersion] { + return actor.Versions().Iterate(smallPageDesc(), ptr(int64(1))) + }, func(v *apify.ActorVersion) string { return v.VersionNumber }, []string{"0.0", "0.1"}, 100) +} + +func TestIterateActorEnvVars(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + created, err := client.Actors().Create(ctx, minimalActor(uniqueName("iter-env"))) + if err != nil { + t.Fatalf("create actor: %v", err) + } + defer func() { _ = client.Actor(created.ID).Delete(ctx) }() + envVars := client.Actor(created.ID).Version("0.0").EnvVars() + + for _, name := range []string{"VAR_A", "VAR_B", "VAR_C"} { + if _, err := envVars.Create(ctx, apify.ActorEnvVar{Name: name, Value: "v"}); err != nil { + t.Fatalf("create env var %s: %v", name, err) + } + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.ActorEnvVar] { + return envVars.Iterate() + }, func(e *apify.ActorEnvVar) string { return e.Name }, []string{"VAR_A", "VAR_B", "VAR_C"}, 100) +} + +func TestIterateRuns(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + // Start two quick runs of the public hello-world Actor (Start does not wait for finish), + // so the Actor-scoped run collection has more than one item to page through. + actor := client.Actor("apify/hello-world") + var want []string + for i := 0; i < 2; i++ { + run, err := actor.Start(ctx, nil, apify.ActorStartOptions{}) + if err != nil { + t.Fatalf("start run %d: %v", i, err) + } + want = append(want, run.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.ActorRun] { + return actor.Runs().Iterate(smallPageDesc(), apify.RunListOptions{}, ptr(int64(1))) + }, func(r *apify.ActorRun) string { return r.ID }, want, 200) +} + +func TestIterateBuilds(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + // The public hello-world Actor has at least one build; iterate its build collection one + // build per page and confirm the iterator yields real builds with non-empty IDs. + it := client.Actor("apify/hello-world").Builds().Iterate(smallPageDesc(), ptr(int64(1))) + count := 0 + for count < 50 { + build, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if build == nil { + break + } + if build.ID == "" { + t.Fatal("expected a non-empty build ID") + } + count++ + } + if count == 0 { + t.Fatal("expected at least one build for apify/hello-world") + } +} + +func TestIterateWebhookDispatches(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + wh, err := client.Webhooks().Create(ctx, webhookDef("https://example.com/iter-dispatch")) + if err != nil { + t.Fatalf("create webhook: %v", err) + } + defer func() { _ = client.Webhook(wh.ID).Delete(ctx) }() + webhook := client.Webhook(wh.ID) + + // Each Test() dispatches the webhook, creating a dispatch record to page through. + var want []string + for i := 0; i < 2; i++ { + dispatch, err := webhook.Test(ctx) + if err != nil { + t.Fatalf("test webhook %d: %v", i, err) + } + want = append(want, dispatch.ID) + } + + iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.WebhookDispatch] { + return webhook.Dispatches().Iterate(smallPageDesc(), ptr(int64(1))) + }, func(d *apify.WebhookDispatch) string { return d.ID }, want, 200) +} + +func TestIterateDatasetItems(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + ds, err := client.Datasets().GetOrCreate(ctx, uniqueName("iter-items")) + if err != nil { + t.Fatalf("get-or-create: %v", err) + } + defer func() { _ = client.Dataset(ds.ID).Delete(ctx) }() + dataset := client.Dataset(ds.ID) + + const total = 5 + items := make([]map[string]any, total) + for i := 0; i < total; i++ { + items[i] = map[string]any{"n": i} + } + if err := dataset.PushItems(ctx, items); err != nil { + t.Fatalf("push items: %v", err) + } + + // The X-Apify-Pagination-Total header can lag right after a push, and offset-based + // iteration relies on it to page. Wait for the total to converge before iterating. + settled := false + for attempt := 0; attempt < 20; attempt++ { + page, err := dataset.ListItems(ctx, apify.DatasetListItemsOptions{Limit: ptr(int64(1))}) + if err != nil { + t.Fatalf("list items: %v", err) + } + if page.Total >= total { + settled = true + break + } + time.Sleep(500 * time.Millisecond) + } + if !settled { + t.Fatalf("dataset item total did not converge to %d", total) + } + + // Page two items at a time so iteration crosses multiple pages. + it := dataset.IterateItems(apify.DatasetListItemsOptions{}, ptr(int64(2))) + count := 0 + for { + item, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if item == nil { + break + } + count++ + } + if count != total { + t.Fatalf("expected to iterate %d items, got %d", total, count) + } + + // Limit is a cap on the total number of items yielded (not the page size), matching the + // reference client. With Limit=3 and a page size of 2, iteration must stop at exactly 3. + const wantCap = 3 + capped := dataset.IterateItems(apify.DatasetListItemsOptions{Limit: ptr(int64(wantCap))}, ptr(int64(2))) + capCount := 0 + for { + item, err := capped.Next(ctx) + if err != nil { + t.Fatalf("iterate (capped): %v", err) + } + if item == nil { + break + } + capCount++ + } + if capCount != wantCap { + t.Fatalf("expected Limit to cap iteration at %d items, got %d", wantCap, capCount) + } +} diff --git a/tests/store_test.go b/tests/store_test.go index c89d1ae..d58fdf1 100644 --- a/tests/store_test.go +++ b/tests/store_test.go @@ -25,7 +25,7 @@ func TestIterateStore(t *testing.T) { ctx, cancel := testContext(t) defer cancel() - it := client.Store().Iterate(apify.StoreListOptions{Limit: ptr(int64(5))}) + it := client.Store().Iterate(apify.StoreListOptions{}, ptr(int64(5))) count := 0 for count < 12 { item, err := it.Next(ctx) diff --git a/version.go b/version.go index 15649a7..32b1559 100644 --- a/version.go +++ b/version.go @@ -4,10 +4,10 @@ package apify // // It follows Semantic Versioning (https://semver.org/). Changes to the public // interface (other than additive ones) are considered breaking changes. -const ClientVersion = "0.5.0" +const ClientVersion = "0.6.0" // APISpecVersion is the version of the Apify OpenAPI specification that this // client was generated and verified against. // // It corresponds to the `info.version` field of the Apify OpenAPI document. -const APISpecVersion = "v2-2026-07-08T143931Z" +const APISpecVersion = "v2-2026-07-10T105921Z" diff --git a/webhook.go b/webhook.go index ba81302..eabe259 100644 --- a/webhook.go +++ b/webhook.go @@ -25,6 +25,19 @@ func (c *WebhookCollectionClient) List(ctx context.Context, options ListOptions) return listResource[Webhook](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the webhooks matching the options, fetching pages on +// demand. The options' Limit caps the total number of webhooks yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). Mirrors the reference client's +// iterable list(). +func (c *WebhookCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Webhook] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Webhook], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // Create creates a new webhook. webhook is any JSON-serializable webhook definition. func (c *WebhookCollectionClient) Create(ctx context.Context, webhook any) (Webhook, error) { return createResource[Webhook](ctx, c.ctx, NewQueryParams(), webhook) diff --git a/webhook_dispatch.go b/webhook_dispatch.go index 8ed51d7..50f3704 100644 --- a/webhook_dispatch.go +++ b/webhook_dispatch.go @@ -25,6 +25,19 @@ func (c *WebhookDispatchCollectionClient) List(ctx context.Context, options List return listResource[WebhookDispatch](ctx, c.ctx, "", params) } +// Iterate returns a lazy iterator over the webhook dispatches matching the options, fetching +// pages on demand. The options' Limit caps the total number of dispatches yielded (unset means +// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference +// client's iterable list(). +func (c *WebhookDispatchCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[WebhookDispatch] { + return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[WebhookDispatch], error) { + opts := options + opts.Offset = &offset + opts.Limit = pageLimitPtr(limit) + return c.List(ctx, opts) + }) +} + // WebhookDispatchClient is a client for a specific webhook dispatch // (/v2/webhook-dispatches/{dispatchId}). type WebhookDispatchClient struct {