From f4d9aa96500518c470bd9e85991f0eeabc3db2b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 15:43:51 +0000 Subject: [PATCH 1/5] feat: add .NET client for the Apify API (spec v2-2026-07-01T115402Z) Bespoke, idiomatic C#/.NET 8 client (Apify.Client) mirroring the reference JS client: resource clients for actors (+versions, env vars), builds, runs, datasets, key-value stores, request queues, tasks, schedules, webhooks, dispatches, store, users and logs, plus SetStatusMessage and actor ValidateInput. Replaceable HttpClient-based transport, bearer auth, mandated User-Agent, exponential-backoff retries, 404->null, HMAC storage URL signing, binary-safe key-value records and dataset downloads (byte[]), and batch request add with uniqueKey validation and byte-size-bounded, parallelism-limited chunking. Includes xUnit unit tests (mock transport), live integration tests, runnable documentation examples with a CI test step, docs/, CHANGELOG, analyzers + warnings-as-errors + dotnet format gates, and CI workflows for integration tests and NuGet publishing. --- .editorconfig | 24 + .../workflows/dotnet-integration-tests.yml | 77 +++ .github/workflows/dotnet-publish.yml | 114 ++++ .gitignore | 8 + Apify.Client.sln | 36 ++ CHANGELOG.md | 37 ++ Directory.Build.props | 18 + README.md | 58 +- docs/README.md | 146 +++++ docs/actors.md | 75 +++ docs/builds.md | 32 + docs/examples.md | 130 ++++ docs/misc.md | 62 ++ docs/runs.md | 38 ++ docs/schedules.md | 29 + docs/storages.md | 119 ++++ docs/tasks.md | 36 ++ docs/webhooks.md | 36 ++ src/Apify.Client/Apify.Client.csproj | 33 + src/Apify.Client/ApifyClient.cs | 289 +++++++++ src/Apify.Client/ApifyClientOptions.cs | 41 ++ src/Apify.Client/ApifyClientVersion.cs | 24 + .../Exceptions/ApifyApiException.cs | 79 +++ .../Exceptions/ApifyTransportException.cs | 26 + src/Apify.Client/Http/HttpClientTransport.cs | 97 +++ src/Apify.Client/Http/IHttpTransport.cs | 46 ++ src/Apify.Client/Internal/HttpClientCore.cs | 273 ++++++++ src/Apify.Client/Internal/Json.cs | 65 ++ src/Apify.Client/Internal/JsonValues.cs | 64 ++ src/Apify.Client/Internal/QueryParams.cs | 129 ++++ src/Apify.Client/Internal/ResourceContext.cs | 378 +++++++++++ .../Internal/ResponseOwningStream.cs | 72 +++ src/Apify.Client/Internal/RetryConfig.cs | 32 + src/Apify.Client/Internal/Signatures.cs | 113 ++++ src/Apify.Client/Internal/Statuses.cs | 22 + src/Apify.Client/Models/Actor.cs | 41 ++ src/Apify.Client/Models/ActorEnvVar.cs | 86 +++ src/Apify.Client/Models/ActorRun.cs | 60 ++ src/Apify.Client/Models/ActorStoreListItem.cs | 26 + src/Apify.Client/Models/ActorTask.cs | 41 ++ src/Apify.Client/Models/ActorVersion.cs | 20 + src/Apify.Client/Models/ApifyResource.cs | 115 ++++ src/Apify.Client/Models/BatchAddResult.cs | 42 ++ src/Apify.Client/Models/Build.cs | 39 ++ src/Apify.Client/Models/Dataset.cs | 32 + src/Apify.Client/Models/KeyValueStore.cs | 29 + src/Apify.Client/Models/KeyValueStoreKey.cs | 20 + .../Models/KeyValueStoreKeysPage.cs | 57 ++ .../Models/KeyValueStoreRecord.cs | 37 ++ src/Apify.Client/Models/PaginationList.cs | 99 +++ src/Apify.Client/Models/RequestQueue.cs | 32 + src/Apify.Client/Models/RequestQueueHead.cs | 42 ++ .../Models/RequestQueueOperationInfo.cs | 29 + .../Models/RequestQueueRequest.cs | 93 +++ src/Apify.Client/Models/Schedule.cs | 29 + src/Apify.Client/Models/User.cs | 23 + src/Apify.Client/Models/Webhook.cs | 27 + src/Apify.Client/Models/WebhookDispatch.cs | 20 + src/Apify.Client/Options/ActorBuildOptions.cs | 27 + src/Apify.Client/Options/ActorListOptions.cs | 31 + src/Apify.Client/Options/ActorStartOptions.cs | 71 +++ .../Options/BatchAddRequestsOptions.cs | 42 ++ .../Options/DatasetDownloadOptions.cs | 50 ++ .../Options/DatasetListItemsOptions.cs | 75 +++ .../Options/DownloadItemsFormat.cs | 45 ++ src/Apify.Client/Options/GetRecordOptions.cs | 18 + src/Apify.Client/Options/LastRunOptions.cs | 19 + src/Apify.Client/Options/ListKeysOptions.cs | 31 + src/Apify.Client/Options/ListOptions.cs | 25 + .../Options/ListRequestsOptions.cs | 64 ++ src/Apify.Client/Options/LogOptions.cs | 18 + src/Apify.Client/Options/MetamorphOptions.cs | 17 + .../Options/PaginateRequestsOptions.cs | 65 ++ .../Options/RequestQueueClientOptions.cs | 21 + src/Apify.Client/Options/RunChargeOptions.cs | 31 + src/Apify.Client/Options/RunListOptions.cs | 30 + .../Options/RunResurrectOptions.cs | 35 ++ src/Apify.Client/Options/SetRecordOptions.cs | 18 + .../Options/StorageListOptions.cs | 35 ++ src/Apify.Client/Options/StoreListOptions.cs | 69 ++ src/Apify.Client/Options/TaskStartOptions.cs | 49 ++ .../Options/ValidateInputOptions.cs | 22 + .../AbstractWebhookCollectionClient.cs | 33 + src/Apify.Client/Resources/ActorClient.cs | 152 +++++ .../Resources/ActorCollectionClient.cs | 36 ++ .../Resources/ActorEnvVarClient.cs | 42 ++ .../Resources/ActorEnvVarCollectionClient.cs | 36 ++ .../Resources/ActorVersionClient.cs | 52 ++ .../Resources/ActorVersionCollectionClient.cs | 36 ++ src/Apify.Client/Resources/BuildClient.cs | 69 ++ .../Resources/BuildCollectionClient.cs | 31 + src/Apify.Client/Resources/DatasetClient.cs | 185 ++++++ .../Resources/DatasetCollectionClient.cs | 42 ++ .../Resources/KeyValueStoreClient.cs | 186 ++++++ .../KeyValueStoreCollectionClient.cs | 41 ++ src/Apify.Client/Resources/LogClient.cs | 69 ++ .../NestedWebhookCollectionClient.cs | 17 + .../Resources/RequestQueueClient.cs | 589 ++++++++++++++++++ .../Resources/RequestQueueCollectionClient.cs | 39 ++ src/Apify.Client/Resources/RunClient.cs | 218 +++++++ .../Resources/RunCollectionClient.cs | 36 ++ src/Apify.Client/Resources/ScheduleClient.cs | 43 ++ .../Resources/ScheduleCollectionClient.cs | 36 ++ .../Resources/StoreCollectionClient.cs | 59 ++ src/Apify.Client/Resources/TaskClient.cs | 114 ++++ .../Resources/TaskCollectionClient.cs | 36 ++ src/Apify.Client/Resources/UserClient.cs | 93 +++ src/Apify.Client/Resources/WebhookClient.cs | 50 ++ .../Resources/WebhookCollectionClient.cs | 27 + .../Resources/WebhookDispatchClient.cs | 26 + .../WebhookDispatchCollectionClient.cs | 31 + .../Apify.Client.Tests.csproj | 23 + .../Examples/CreateBuildRunActorExample.cs | 50 ++ .../Examples/ExamplesTests.cs | 50 ++ .../Examples/GetAccountExample.cs | 18 + .../Examples/IterateStoreExample.cs | 23 + .../Examples/LogRedirectionExample.cs | 20 + .../Examples/RunAndLastRunStoragesExample.cs | 22 + .../Examples/RunStoreActorExample.cs | 17 + .../Examples/StoragesExample.cs | 58 ++ .../Integration/ActorIntegrationTests.cs | 119 ++++ .../Integration/ActorRunIntegrationTests.cs | 53 ++ .../Integration/BuildIntegrationTests.cs | 40 ++ .../Integration/DatasetIntegrationTests.cs | 82 +++ .../Integration/IntegrationTestBase.cs | 82 +++ .../KeyValueStoreIntegrationTests.cs | 131 ++++ .../RequestQueueIntegrationTests.cs | 146 +++++ .../Integration/ScheduleIntegrationTests.cs | 65 ++ .../Integration/StoreIntegrationTests.cs | 34 + .../Integration/TaskIntegrationTests.cs | 64 ++ .../Integration/UserIntegrationTests.cs | 38 ++ .../Integration/WebhookIntegrationTests.cs | 92 +++ .../Unit/BatchAddRequestsTests.cs | 251 ++++++++ tests/Apify.Client.Tests/Unit/ConfigTests.cs | 53 ++ .../Unit/HttpClientTests.cs | 156 +++++ .../Apify.Client.Tests/Unit/LogClientTests.cs | 64 ++ .../Apify.Client.Tests/Unit/MockTransport.cs | 197 ++++++ .../Unit/ModelSerializationTests.cs | 80 +++ .../Unit/RequestShapeTests.cs | 134 ++++ .../Apify.Client.Tests/Unit/SignatureTests.cs | 59 ++ 140 files changed, 9448 insertions(+), 2 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/dotnet-integration-tests.yml create mode 100644 .github/workflows/dotnet-publish.yml create mode 100644 Apify.Client.sln create mode 100644 CHANGELOG.md create mode 100644 Directory.Build.props create mode 100644 docs/README.md create mode 100644 docs/actors.md create mode 100644 docs/builds.md create mode 100644 docs/examples.md create mode 100644 docs/misc.md create mode 100644 docs/runs.md create mode 100644 docs/schedules.md create mode 100644 docs/storages.md create mode 100644 docs/tasks.md create mode 100644 docs/webhooks.md create mode 100644 src/Apify.Client/Apify.Client.csproj create mode 100644 src/Apify.Client/ApifyClient.cs create mode 100644 src/Apify.Client/ApifyClientOptions.cs create mode 100644 src/Apify.Client/ApifyClientVersion.cs create mode 100644 src/Apify.Client/Exceptions/ApifyApiException.cs create mode 100644 src/Apify.Client/Exceptions/ApifyTransportException.cs create mode 100644 src/Apify.Client/Http/HttpClientTransport.cs create mode 100644 src/Apify.Client/Http/IHttpTransport.cs create mode 100644 src/Apify.Client/Internal/HttpClientCore.cs create mode 100644 src/Apify.Client/Internal/Json.cs create mode 100644 src/Apify.Client/Internal/JsonValues.cs create mode 100644 src/Apify.Client/Internal/QueryParams.cs create mode 100644 src/Apify.Client/Internal/ResourceContext.cs create mode 100644 src/Apify.Client/Internal/ResponseOwningStream.cs create mode 100644 src/Apify.Client/Internal/RetryConfig.cs create mode 100644 src/Apify.Client/Internal/Signatures.cs create mode 100644 src/Apify.Client/Internal/Statuses.cs create mode 100644 src/Apify.Client/Models/Actor.cs create mode 100644 src/Apify.Client/Models/ActorEnvVar.cs create mode 100644 src/Apify.Client/Models/ActorRun.cs create mode 100644 src/Apify.Client/Models/ActorStoreListItem.cs create mode 100644 src/Apify.Client/Models/ActorTask.cs create mode 100644 src/Apify.Client/Models/ActorVersion.cs create mode 100644 src/Apify.Client/Models/ApifyResource.cs create mode 100644 src/Apify.Client/Models/BatchAddResult.cs create mode 100644 src/Apify.Client/Models/Build.cs create mode 100644 src/Apify.Client/Models/Dataset.cs create mode 100644 src/Apify.Client/Models/KeyValueStore.cs create mode 100644 src/Apify.Client/Models/KeyValueStoreKey.cs create mode 100644 src/Apify.Client/Models/KeyValueStoreKeysPage.cs create mode 100644 src/Apify.Client/Models/KeyValueStoreRecord.cs create mode 100644 src/Apify.Client/Models/PaginationList.cs create mode 100644 src/Apify.Client/Models/RequestQueue.cs create mode 100644 src/Apify.Client/Models/RequestQueueHead.cs create mode 100644 src/Apify.Client/Models/RequestQueueOperationInfo.cs create mode 100644 src/Apify.Client/Models/RequestQueueRequest.cs create mode 100644 src/Apify.Client/Models/Schedule.cs create mode 100644 src/Apify.Client/Models/User.cs create mode 100644 src/Apify.Client/Models/Webhook.cs create mode 100644 src/Apify.Client/Models/WebhookDispatch.cs create mode 100644 src/Apify.Client/Options/ActorBuildOptions.cs create mode 100644 src/Apify.Client/Options/ActorListOptions.cs create mode 100644 src/Apify.Client/Options/ActorStartOptions.cs create mode 100644 src/Apify.Client/Options/BatchAddRequestsOptions.cs create mode 100644 src/Apify.Client/Options/DatasetDownloadOptions.cs create mode 100644 src/Apify.Client/Options/DatasetListItemsOptions.cs create mode 100644 src/Apify.Client/Options/DownloadItemsFormat.cs create mode 100644 src/Apify.Client/Options/GetRecordOptions.cs create mode 100644 src/Apify.Client/Options/LastRunOptions.cs create mode 100644 src/Apify.Client/Options/ListKeysOptions.cs create mode 100644 src/Apify.Client/Options/ListOptions.cs create mode 100644 src/Apify.Client/Options/ListRequestsOptions.cs create mode 100644 src/Apify.Client/Options/LogOptions.cs create mode 100644 src/Apify.Client/Options/MetamorphOptions.cs create mode 100644 src/Apify.Client/Options/PaginateRequestsOptions.cs create mode 100644 src/Apify.Client/Options/RequestQueueClientOptions.cs create mode 100644 src/Apify.Client/Options/RunChargeOptions.cs create mode 100644 src/Apify.Client/Options/RunListOptions.cs create mode 100644 src/Apify.Client/Options/RunResurrectOptions.cs create mode 100644 src/Apify.Client/Options/SetRecordOptions.cs create mode 100644 src/Apify.Client/Options/StorageListOptions.cs create mode 100644 src/Apify.Client/Options/StoreListOptions.cs create mode 100644 src/Apify.Client/Options/TaskStartOptions.cs create mode 100644 src/Apify.Client/Options/ValidateInputOptions.cs create mode 100644 src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs create mode 100644 src/Apify.Client/Resources/ActorClient.cs create mode 100644 src/Apify.Client/Resources/ActorCollectionClient.cs create mode 100644 src/Apify.Client/Resources/ActorEnvVarClient.cs create mode 100644 src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs create mode 100644 src/Apify.Client/Resources/ActorVersionClient.cs create mode 100644 src/Apify.Client/Resources/ActorVersionCollectionClient.cs create mode 100644 src/Apify.Client/Resources/BuildClient.cs create mode 100644 src/Apify.Client/Resources/BuildCollectionClient.cs create mode 100644 src/Apify.Client/Resources/DatasetClient.cs create mode 100644 src/Apify.Client/Resources/DatasetCollectionClient.cs create mode 100644 src/Apify.Client/Resources/KeyValueStoreClient.cs create mode 100644 src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs create mode 100644 src/Apify.Client/Resources/LogClient.cs create mode 100644 src/Apify.Client/Resources/NestedWebhookCollectionClient.cs create mode 100644 src/Apify.Client/Resources/RequestQueueClient.cs create mode 100644 src/Apify.Client/Resources/RequestQueueCollectionClient.cs create mode 100644 src/Apify.Client/Resources/RunClient.cs create mode 100644 src/Apify.Client/Resources/RunCollectionClient.cs create mode 100644 src/Apify.Client/Resources/ScheduleClient.cs create mode 100644 src/Apify.Client/Resources/ScheduleCollectionClient.cs create mode 100644 src/Apify.Client/Resources/StoreCollectionClient.cs create mode 100644 src/Apify.Client/Resources/TaskClient.cs create mode 100644 src/Apify.Client/Resources/TaskCollectionClient.cs create mode 100644 src/Apify.Client/Resources/UserClient.cs create mode 100644 src/Apify.Client/Resources/WebhookClient.cs create mode 100644 src/Apify.Client/Resources/WebhookCollectionClient.cs create mode 100644 src/Apify.Client/Resources/WebhookDispatchClient.cs create mode 100644 src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs create mode 100644 tests/Apify.Client.Tests/Apify.Client.Tests.csproj create mode 100644 tests/Apify.Client.Tests/Examples/CreateBuildRunActorExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/ExamplesTests.cs create mode 100644 tests/Apify.Client.Tests/Examples/GetAccountExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/IterateStoreExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/RunAndLastRunStoragesExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs create mode 100644 tests/Apify.Client.Tests/Examples/StoragesExample.cs create mode 100644 tests/Apify.Client.Tests/Integration/ActorIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs create mode 100644 tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/ScheduleIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/TaskIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/ConfigTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/HttpClientTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/LogClientTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/MockTransport.cs create mode 100644 tests/Apify.Client.Tests/Unit/ModelSerializationTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/RequestShapeTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/SignatureTests.cs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..42ce52a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space + +[*.{cs,csx}] +indent_size = 4 +# Use file-scoped namespaces throughout. +csharp_style_namespace_declarations = file_scoped:warning + +# CA1711: the model type `RequestQueue` deliberately mirrors the Apify API resource name (and the +# JS/PHP/Java/Go/Rust sibling clients). Renaming it away from the reserved "Queue" suffix would break +# cross-client naming consistency, which the client requirements mandate. Justified suppression. +dotnet_diagnostic.CA1711.severity = none + +[*.{csproj,props,targets}] +indent_size = 2 + +[*.{yml,yaml,json,md}] +indent_size = 2 diff --git a/.github/workflows/dotnet-integration-tests.yml b/.github/workflows/dotnet-integration-tests.yml new file mode 100644 index 0000000..2e80494 --- /dev/null +++ b/.github/workflows/dotnet-integration-tests.yml @@ -0,0 +1,77 @@ +name: .NET integration tests + +# Language-specific workflow: only runs for the .NET client. Triggers on PRs to master that touch +# .NET client, test, project, example, or documentation code, and can be dispatched manually from any +# branch. +on: + pull_request: + branches: [master] + paths: + - '**/*.cs' + - '**/*.csproj' + - '**/*.sln' + - 'Directory.Build.props' + - '.editorconfig' + # The "Test examples" step validates the in-documentation snippets, so doc changes must re-run + # the workflow even though Markdown is not .NET code. + - 'docs/**' + - 'README.md' + - '.github/workflows/dotnet-integration-tests.yml' + workflow_dispatch: + +# Avoid concurrent runs of the same ref racing on the shared test account. +concurrency: + group: dotnet-integration-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore + run: dotnet restore + + # Formatting gate mandated by the coding rules (dotnet format). + - name: Check formatting (dotnet format) + run: dotnet format --verify-no-changes + + # Build with analyzers and warnings-as-errors (the static-analysis gate). + - name: Build (warnings as errors) + run: dotnet build --configuration Release --no-restore + + # Offline unit tests (mock transport): prove the retry/error/signature logic without the API. + - name: Unit tests + run: dotnet test --configuration Release --no-build --filter Category=Unit + + # Fail fast if the integration-test secret is missing or empty. Without this guard the + # integration tests silently skip (they self-skip when APIFY_TOKEN is unset), so a green run + # would not prove the API logic actually executed. + - name: Require APIFY_TOKEN secret + env: + APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} + run: | + if [ -z "${APIFY_TOKEN}" ]; then + echo "::error::APIFY_TOKEN secret is empty or missing; integration tests would not run against the API." + exit 1 + fi + + - name: Integration tests + env: + # The integration-test token is stored as a repository secret. + APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} + run: dotnet test --configuration Release --no-build --filter Category=Integration + + # Standalone CI step that verifies the documentation examples work end-to-end against the live + # API (the Examples suite runs each example). + - name: Test examples + env: + APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} + run: dotnet test --configuration Release --no-build --filter Category=Examples diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml new file mode 100644 index 0000000..34eeda5 --- /dev/null +++ b/.github/workflows/dotnet-publish.yml @@ -0,0 +1,114 @@ +name: Publish .NET client + +# Language-specific publish workflow for the .NET client. Triggered manually only (workflow_dispatch) +# so a maintainer deliberately decides when a release is cut. The release version is the single source +# of truth in src/Apify.Client/Apify.Client.csproj (). This workflow packs the library, +# pushes it to NuGet.org, tags the release, and creates the GitHub release. +# +# The NuGet API key is read from a repository secret (nothing is stored in the repo). If the account +# has NuGet Trusted Publishing (OIDC) configured, the id-token permission below allows switching to it. +on: + workflow_dispatch: + inputs: + dry_run: + description: 'Run all checks and pack, but do not push, tag, or release.' + type: boolean + default: false + +# Never allow two publish runs to race; publishing two releases concurrently is hard to undo. +concurrency: + group: dotnet-publish + cancel-in-progress: false + +permissions: + contents: write # create the tagged GitHub release + id-token: write # allow NuGet Trusted Publishing (OIDC) if configured + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # A release must only ever be cut from master. + - name: Require master branch + run: | + if [ "${GITHUB_REF}" != "refs/heads/master" ]; then + echo "::error::Publishing is only allowed from master, but this run is on '${GITHUB_REF}'." + exit 1 + fi + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore + run: dotnet restore + + # Gate the release on the same quality bar as CI so a broken build can never be published. + - name: Check formatting (dotnet format) + run: dotnet format --verify-no-changes + + - name: Build (warnings as errors) + run: dotnet build --configuration Release --no-restore + + - name: Unit tests + run: dotnet test --configuration Release --no-build --filter Category=Unit + + - name: Resolve version from csproj + id: version + run: | + version=$(grep -oPm1 '(?<=)[^<]+' src/Apify.Client/Apify.Client.csproj) + if ! echo "${version}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error:: '${version}' is not a bare semver (X.Y.Z)." + exit 1 + fi + echo "tag=v${version}" >> "$GITHUB_OUTPUT" + echo "Resolved release tag: v${version}" + + - name: Ensure tag does not already exist + env: + TAG: ${{ steps.version.outputs.tag }} + run: | + if git ls-remote --exit-code --tags origin "${TAG}" >/dev/null 2>&1; then + echo "::error::Tag ${TAG} already exists on origin; bump first." + exit 1 + fi + + - name: Pack + run: dotnet pack src/Apify.Client/Apify.Client.csproj --configuration Release --no-build --output ./artifacts + + - name: Push to NuGet + if: ${{ github.event.inputs.dry_run != 'true' }} + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + run: | + dotnet nuget push "./artifacts/*.nupkg" \ + --api-key "${NUGET_API_KEY}" \ + --source https://api.nuget.org/v3/index.json \ + --skip-duplicate + + - name: Create and push release tag + if: ${{ github.event.inputs.dry_run != 'true' }} + env: + TAG: ${{ steps.version.outputs.tag }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "${TAG}" -m "Release ${TAG}" + git push origin "${TAG}" + + - name: Create GitHub release + if: ${{ github.event.inputs.dry_run != 'true' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.version.outputs.tag }} + run: | + gh release create "${TAG}" \ + --title "${TAG}" \ + --notes "Apify .NET client ${TAG}. See CHANGELOG.md for details." \ + ./artifacts/*.nupkg diff --git a/.gitignore b/.gitignore index 7282dbf..beab34b 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,11 @@ CodeCoverage/ *.VisualState.xml TestResult.xml nunit-*.xml + +# .NET build output +bin/ +obj/ + +# IDE +.vs/ +*.user diff --git a/Apify.Client.sln b/Apify.Client.sln new file mode 100644 index 0000000..0a6aab4 --- /dev/null +++ b/Apify.Client.sln @@ -0,0 +1,36 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{ED3360CB-7E9D-4B24-8CF1-E80489F99D61}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apify.Client", "src\Apify.Client\Apify.Client.csproj", "{9A0401C4-477A-4111-ABC1-43D95430E810}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{624E02B9-A559-4043-AB5D-2A5CCFC19DA3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apify.Client.Tests", "tests\Apify.Client.Tests\Apify.Client.Tests.csproj", "{6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9A0401C4-477A-4111-ABC1-43D95430E810}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9A0401C4-477A-4111-ABC1-43D95430E810}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9A0401C4-477A-4111-ABC1-43D95430E810}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9A0401C4-477A-4111-ABC1-43D95430E810}.Release|Any CPU.Build.0 = Release|Any CPU + {6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {9A0401C4-477A-4111-ABC1-43D95430E810} = {ED3360CB-7E9D-4B24-8CF1-E80489F99D61} + {6E8E3DF4-F6BA-48A6-8E07-18668FC9A7F6} = {624E02B9-A559-4043-AB5D-2A5CCFC19DA3} + EndGlobalSection +EndGlobal diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..38269ac --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +## 0.1.0 + +- Initial .NET client for the Apify API (spec `v2-2026-07-02T131926Z`). +- Resource clients for Actors, Actor versions and environment variables, builds, runs, datasets, + key-value stores, request queues, tasks, schedules, webhooks, webhook dispatches, the Apify Store, + users, and logs. +- Async-first API (`Task`-returning, `CancellationToken`-aware) with convenience helpers consistent + with the JS reference client: `Actor().CallAsync()`/`StartAsync()`, `ValidateInputAsync()`, + `DefaultBuildAsync()`, `LastRun()`, run `AbortAsync`/`MetamorphAsync`/`RebootAsync`/`ResurrectAsync`/ + `ChargeAsync`/`WaitForFinishAsync`, dataset `ListItemsAsync`/`DownloadItemsAsync`/`PushItemsAsync`/ + public URLs, key-value store records and public URLs, request queue batch add with retries, lazy + request/store iteration (`IAsyncEnumerable`), and log streaming. +- Binary-safe storage payloads: `KeyValueStoreRecord.Value` and `DownloadItemsAsync` return `byte[]` + (raw bytes), and `SetRecordAsync` accepts `byte[]`, so binary records and exports (e.g. XLSX) are + not corrupted; `SetRecordJsonAsync` serializes to JSON bytes. +- `RequestQueueRequest.UserData` and `ActorEnvVar` `Name`/`Value`/`IsSecret` omit the field when set to + `null` (rather than writing a JSON `null`), honoring the documented null-omit contract. +- `BatchAddRequestsAsync` requires a non-empty `UniqueKey` per request, splits batches by both the + 25-request count limit and the ~9 MiB payload-size limit, dispatches chunks with up to + `BatchAddRequestsOptions.MaxParallel` concurrent calls (results merged in input order), and retries + only the requests the API reports unprocessed in a successful response. Consistent with the reference + client, a failed batch call reports that chunk's not-yet-processed requests as unprocessed rather + than throwing. +- `PaginationList.Count` is the number of items in the page (matching the indexer); the total across + all pages is exposed as `Total`. +- `Datasets().GetOrCreateAsync()` and `KeyValueStores().GetOrCreateAsync()` accept an optional schema. +- `RequestQueue(id, RequestQueueClientOptions)` accepts `ClientKey` and `TimeoutSecs`; + `PaginateRequestsAsync()` accepts `PaginateRequestsOptions` (`Limit`, `MaxPageLimit`, + `ExclusiveStartId`, `Cursor`, `Filter`). +- Replaceable HTTP transport (`IHttpTransport`) with a default `HttpClient`-based implementation; + automatic retries with exponential backoff and jitter, growing per-attempt timeouts, and + HMAC-SHA256 storage URL signing. +- Public `ApifyClientVersion.ClientVersion` and `ApifyClientVersion.ApiSpecVersion` constants. +- Integration test suite, documentation with runnable examples, and CI workflows for integration + tests and publishing. diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..6cfc00b --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,18 @@ + + + + net8.0 + latest + enable + disable + true + true + latest + Recommended + true + + diff --git a/README.md b/README.md index ae1aa51..668064c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,56 @@ -# apify-client-dotnet -Apify API client for .NET—Programmatically run Actors, manage and stream data from storages (datasets, key-value stores, request queues), schedule and monitor runs, and access the full Apify platform API. Sync and async interfaces with automatic retries and pagination. +# Apify API client for .NET + +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it in +> production and report issues on the repository. + +A resource-oriented .NET client for the [Apify API](https://docs.apify.com/api/v2), mirroring the +official [JavaScript](https://github.com/apify/apify-client-js) reference client: start from an +`ApifyClient`, then drill down into resources (Actors, runs, datasets, key-value stores, request +queues, tasks, schedules, webhooks, the store, users and logs). + +All calls are asynchronous (`Task`-returning, `CancellationToken`-aware). + +## Requirements + +- .NET 8.0 or newer. + +## Installation + +```bash +dotnet add package Apify.Client +``` + +## Quick start + +```csharp +using Apify.Client; + +var client = new ApifyClient("my-api-token"); + +// Start an Actor and wait for it to finish (null waits indefinitely; pass seconds to bound the wait). +var run = await client.Actor("apify/hello-world").CallAsync(null, null, null); + +// Read items from the run's default dataset. +var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(); +Console.WriteLine("Item count: " + items.Count); +``` + +`new ApifyClient("my-api-token")` takes the token as an explicit argument — it does **not** read +`APIFY_TOKEN` automatically. Get your token from the +[Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). + +## Documentation + +Full documentation lives in [`docs/`](docs/README.md), organized by resource, with runnable +[examples](docs/examples.md). + +## Versioning + +- `Apify.Client.ApifyClientVersion.ClientVersion` — the semantic version of this library. +- `Apify.Client.ApifyClientVersion.ApiSpecVersion` — the Apify OpenAPI spec version this client was + built against. + +## License + +[Apache-2.0](LICENSE). diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..9ddc4ac --- /dev/null +++ b/docs/README.md @@ -0,0 +1,146 @@ +# Apify .NET client documentation + +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it in +> production and report issues on the repository. + +A resource-oriented .NET client for the [Apify API](https://docs.apify.com/api/v2), mirroring the +official [JavaScript](https://github.com/apify/apify-client-js) reference client: start from an +`ApifyClient`, then drill down into resources. + +All API calls are asynchronous and return `Task`/`Task`; every method accepts an optional +`CancellationToken`. Method names mirror the reference client with the .NET `Async` suffix +(`GetAsync`, `ListAsync`, `CallAsync`, …). + +## Contents + +- [Actors](actors.md) — create, run, build, validate input, versions and environment variables. +- [Builds](builds.md) — fetch, wait, abort, logs, OpenAPI definition. +- [Runs](runs.md) — get/wait, abort, metamorph, reboot, resurrect, charge, storages, logs. +- [Storages](storages.md) — datasets, key-value stores, request queues. +- [Tasks](tasks.md) — pre-configured Actor runs. +- [Schedules](schedules.md) +- [Webhooks](webhooks.md) — webhooks and dispatches. +- [Misc](misc.md) — the Apify Store, users, logs. +- [Examples](examples.md) — runnable end-to-end examples. + +## Requirements + +- .NET 8.0 or newer. + +## Installation + +```bash +dotnet add package Apify.Client +``` + +## Quick start + +```csharp +using Apify.Client; + +var client = new ApifyClient("my-api-token"); + +// Start an Actor and wait for it to finish. The last argument is the wait budget in seconds; +// pass a value (e.g. 120) to bound the wait, or null to wait indefinitely (as here). +var run = await client.Actor("apify/hello-world").CallAsync(null, null, null); + +// Read items from the run's default dataset. +var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(); +Console.WriteLine("Item count: " + items.Count); +``` + +`new ApifyClient("my-api-token")` takes the token as an explicit argument — it does **not** read +`APIFY_TOKEN` (or any other environment variable) automatically. Read it yourself if you want that, +e.g. `new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN"))`. + +Get your API token from the +[Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). + +## Configuration + +Pass an `ApifyClientOptions` to configure non-default settings: + +```csharp +using Apify.Client; + +var configured = new ApifyClient(new ApifyClientOptions +{ + Token = "my-api-token", + MaxRetries = 5, + MinDelayBetweenRetriesMillis = 1000, + TimeoutSecs = 120, + UserAgentSuffix = "my-app/1.2.3", +}); +``` + +| Option | Default | Meaning | +|---|---|---| +| `Token` | `null` | API token, sent as a Bearer token. | +| `BaseUrl` | `https://api.apify.com` | API base URL; the `/v2` suffix is appended automatically. | +| `PublicBaseUrl` | `BaseUrl` | Base URL used when building public, shareable resource URLs. | +| `MaxRetries` | `8` | Maximum retries for failed requests. | +| `MinDelayBetweenRetriesMillis` | `500` | Minimum delay between retries (exponential backoff). | +| `MaxDelayBetweenRetriesMillis` | request timeout | Upper bound on the growing inter-retry delay. | +| `TimeoutSecs` | `360` | Overall per-request timeout. | +| `UserAgentSuffix` | `null` | Custom suffix appended to the `User-Agent` header. | +| `HttpTransport` | `HttpClientTransport` | The replaceable transport (`Apify.Client.Http.IHttpTransport`). | + +Requests are retried on network errors, HTTP 429 (rate limit) and 5xx responses, with exponential +backoff and jitter. 4xx responses (other than 429) are thrown immediately as `ApifyApiException`. + +### Replaceable HTTP transport + +The transport is `Apify.Client.Http.IHttpTransport`. The default is `HttpClientTransport`, which wraps +`System.Net.Http.HttpClient`; you can pass a pre-configured `HttpClient` (proxy, TLS, connection pool) +or provide your own `IHttpTransport` (e.g. a mock in tests): + +```csharp +using System.Net.Http; +using Apify.Client; +using Apify.Client.Http; + +var httpClient = new HttpClient(); +var client = new ApifyClient(new ApifyClientOptions +{ + Token = "my-api-token", + HttpTransport = new HttpClientTransport(httpClient), +}); +``` + +## Error handling + +Methods that fetch a single resource return `null` when the resource does not exist (rather than +throwing). Other API failures are thrown as `Apify.Client.Exceptions.ApifyApiException`, which exposes +the HTTP status, API error `Type`, message, attempt count, and request method/path: + +```csharp +using Apify.Client; +using Apify.Client.Exceptions; + +var client = new ApifyClient("my-api-token"); +try +{ + await client.Actor("does/not-exist").UpdateAsync(new { title = "x" }); +} +catch (ApifyApiException e) +{ + Console.WriteLine($"{e.StatusCode} {e.Type}: {e.ApiMessage}"); +} +``` + +## Versioning + +- `Apify.Client.ApifyClientVersion.ClientVersion` — the semantic version of this library. +- `Apify.Client.ApifyClientVersion.ApiSpecVersion` — the Apify OpenAPI spec version this client was + built against. + +```csharp +using Apify.Client; + +Console.WriteLine($"{ApifyClientVersion.ClientVersion} / {ApifyClientVersion.ApiSpecVersion}"); +``` + +## License + +[Apache-2.0](../LICENSE). diff --git a/docs/actors.md b/docs/actors.md new file mode 100644 index 0000000..281ca6b --- /dev/null +++ b/docs/actors.md @@ -0,0 +1,75 @@ +# Actors + +Access the Actor collection with `client.Actors()` and a specific Actor with `client.Actor(id)`, where +`id` is the Actor ID or the `username~name` form. + +## Collection — `client.Actors()` + +- `ListAsync(ActorListOptions? options = null)` — list the account's Actors. Returns + `PaginationList`. Options: `Offset`, `Limit`, `Desc`, `My`, `SortBy`. +- `CreateAsync(object actor)` — create an Actor from any JSON-serializable definition. Returns `Actor`. + +```csharp +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var page = await client.Actors().ListAsync(new ActorListOptions { My = true, Limit = 10 }); +foreach (var actor in page.Items) +{ + Console.WriteLine(actor.Name); +} +``` + +## Single Actor — `client.Actor(id)` + +- `GetAsync()` → `Actor?` (null if not found). +- `UpdateAsync(object newFields)` → `Actor`. +- `DeleteAsync()`. +- `StartAsync(object? input = null, ActorStartOptions? options = null)` → `ActorRun` (returns immediately). +- `CallAsync(object? input = null, ActorStartOptions? options = null, int? waitSecs = null)` → `ActorRun` + (starts then waits; `waitSecs` bounds the wait, `null` waits indefinitely). +- `ValidateInputAsync(object? input = null, ValidateInputOptions? options = null)` → `bool`. +- `BuildAsync(string versionNumber, ActorBuildOptions? options = null)` → `Build`. +- `DefaultBuildAsync(int? waitForFinish = null)` → `BuildClient`. +- `LastRun(LastRunOptions? options = null)` → `RunClient` (filter by `Status`/`Origin`). +- `Builds()` → `BuildCollectionClient`; `Runs()` → `RunCollectionClient`. +- `Version(string versionNumber)` / `Versions()` — Actor versions. +- `Webhooks()` → read-only `NestedWebhookCollectionClient`. + +`ActorStartOptions` fields: `Build`, `MemoryMbytes`, `TimeoutSecs`, `WaitForFinish`, `MaxItems`, +`MaxTotalChargeUsd`, `ContentType`, `RestartOnError`, `ForcePermissionLevel`, `Webhooks`. + +```csharp +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var run = await client.Actor("apify/hello-world").CallAsync( + new { message = "hi" }, + new ActorStartOptions { MemoryMbytes = 256, Build = "latest" }, + 120); +Console.WriteLine(run.Status); +``` + +## Versions and environment variables + +```csharp +using Apify.Client; +using Apify.Client.Models; + +var client = new ApifyClient("my-api-token"); +var actor = client.Actor("me/my-actor"); + +await actor.Versions().CreateAsync(new +{ + versionNumber = "0.1", + sourceType = "SOURCE_FILES", + buildTag = "latest", + sourceFiles = System.Array.Empty(), +}); + +var envVars = actor.Version("0.1").EnvVars(); +await envVars.CreateAsync(new ActorEnvVar("MY_VAR", "value", isSecret: true)); +await actor.Version("0.1").EnvVar("MY_VAR").DeleteAsync(); +``` diff --git a/docs/builds.md b/docs/builds.md new file mode 100644 index 0000000..f6e8afb --- /dev/null +++ b/docs/builds.md @@ -0,0 +1,32 @@ +# Builds + +Access the account-wide build collection with `client.Builds()`, an Actor's builds with +`client.Actor(id).Builds()`, and a specific build with `client.Build(buildId)`. + +## Collection + +- `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`). + +## Single build — `client.Build(buildId)` + +- `GetAsync(int? waitForFinishSecs = null)` → `Build?` — optionally waits up to `waitForFinishSecs` + (server-side, max 60) for the build to finish. +- `AbortAsync()` → `Build`. +- `DeleteAsync()`. +- `WaitForFinishAsync(int? waitSecs = null)` → `Build` — client-side polling until terminal (`null` + waits indefinitely). +- `GetOpenApiDefinitionAsync()` → `JsonObject?`. +- `Log()` → `LogClient`. + +```csharp +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var build = await client.Actor("me/my-actor").BuildAsync("0.0", new ActorBuildOptions()); +var finished = await client.Build(build.Id!).WaitForFinishAsync(300); +Console.WriteLine(finished.Status); + +var log = await client.Build(build.Id!).Log().GetAsync(); +Console.WriteLine(log); +``` diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..f1a10c6 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,130 @@ +# Examples + +Each example below is a complete, runnable scenario. The canonical, compiled versions live in +[`tests/Apify.Client.Tests/Examples`](../tests/Apify.Client.Tests/Examples) and are executed +end-to-end against the live API by the **Test examples** CI step (they require an `APIFY_TOKEN`), so +the snippets here are guaranteed to stay valid and working. + +Every snippet runs inside an `async` context and assumes the following `using` directives appear at the +top of the file, **before** any top-level statements (a `using` after the first statement is a `CS1529` +compile error). `ImplicitUsings` is disabled in this repository, so even `System` is listed explicitly: + +```csharp +using System; +using System.IO; +using System.Text; +using Apify.Client; +using Apify.Client.Models; +using Apify.Client.Options; + +var client = new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN")); +``` + +## Run a store Actor and read its dataset + +```csharp +var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); +var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); +Console.WriteLine("Item count: " + items.Count); +``` + +## Each storage: create, push, read + +```csharp +// Dataset +var dataset = await client.Datasets().GetOrCreateAsync("example-ds"); +await client.Dataset(dataset.Id!).PushItemsAsync(new[] { new { hello = "world" } }); +var items = await client.Dataset(dataset.Id!).ListItemsAsync(new DatasetListItemsOptions()); +Console.WriteLine("Dataset items: " + items.Count); + +// Key-value store +var store = await client.KeyValueStores().GetOrCreateAsync("example-kvs"); +await client.KeyValueStore(store.Id!).SetRecordJsonAsync("OUTPUT", new { answer = 42 }); +var record = await client.KeyValueStore(store.Id!).GetRecordAsync("OUTPUT"); +// GetRecordAsync returns the raw bytes; decode JSON/text records with UTF-8. +var recordText = record is null ? string.Empty : Encoding.UTF8.GetString(record.Value); +Console.WriteLine("KVS record: " + recordText); + +// Request queue +var queue = await client.RequestQueues().GetOrCreateAsync("example-rq"); +await client.RequestQueue(queue.Id!).AddRequestAsync(new RequestQueueRequest("https://example.com", "example")); +var head = await client.RequestQueue(queue.Id!).ListHeadAsync(10); +Console.WriteLine("Queue head size: " + head.Items.Count); +``` + +## Get own account details + +```csharp +var user = await client.Me().GetAsync(); +if (user is not null) +{ + Console.WriteLine("Account " + user.Id + " / " + user.Username); +} +``` + +## Create an Actor, build it, run it, print the log + +```csharp +var created = await client.Actors().CreateAsync(new +{ + name = "example-actor", + isPublic = false, + versions = new[] + { + new + { + versionNumber = "0.0", + sourceType = "SOURCE_FILES", + buildTag = "latest", + sourceFiles = new object[] + { + new { name = "Dockerfile", format = "TEXT", content = "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" }, + new { name = "main.js", format = "TEXT", content = "console.log('hi');" }, + }, + }, + }, +}); + +var build = await client.Actor(created.Id!).BuildAsync("0.0", new ActorBuildOptions()); +await client.Build(build.Id!).WaitForFinishAsync(300); +var run = await client.Actor(created.Id!).CallAsync(null, null, 120); +var log = await client.Run(run.Id!).Log().GetAsync(); +Console.WriteLine(log); +``` + +## Start a run, then read the last run's storages + +```csharp +await client.Actor("apify/hello-world").CallAsync(null, null, 120); +var last = await client.Actor("apify/hello-world").LastRun(new LastRunOptions { Status = "SUCCEEDED" }).GetAsync(); +if (last is not null) +{ + await client.Dataset(last.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); + await client.KeyValueStore(last.DefaultKeyValueStoreId!).GetRecordAsync("OUTPUT"); + Console.WriteLine("Last run: " + last.Id); +} +``` + +## Lazy iteration of the Apify Store + +```csharp +var shown = 0; +await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Limit = 10 })) +{ + Console.WriteLine(item.Name); + if (++shown >= 5) + { + break; + } +} +``` + +## Run an Actor with log redirection (streaming) + +```csharp +var run = await client.Actor("apify/hello-world").StartAsync(); +await client.Run(run.Id!).WaitForFinishAsync(120); +using var stream = await client.Run(run.Id!).GetStreamedLogAsync(); +using var reader = new StreamReader(stream); +Console.WriteLine(await reader.ReadToEndAsync()); +``` diff --git a/docs/misc.md b/docs/misc.md new file mode 100644 index 0000000..0db370e --- /dev/null +++ b/docs/misc.md @@ -0,0 +1,62 @@ +# Store, users and logs + +> Snippets below run inside an `async` context. `ImplicitUsings` is disabled in this repository, so all +> `using` directives (including `System`) are shown explicitly and must precede any statements. + +## Apify Store — `client.Store()` + +Browse public Actors in the [Apify Store](https://apify.com/store). + +- `ListAsync(StoreListOptions?)` → `PaginationList` (one page). +- `IterateAsync(StoreListOptions?)` → `IAsyncEnumerable` (lazy, all pages; + `Limit` is the page size). + +`StoreListOptions`: `Offset`, `Limit`, `Search`, `SortBy`, `Category`, `Username`, `PricingModel`, +`IncludeUnrunnableActors`, `AllowsAgenticUsers`, `ResponseFormat`. + +```csharp +using System; +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Search = "crawler", Limit = 50 })) +{ + Console.WriteLine(item.Name); +} +``` + +## Users — `client.Me()` / `client.User(id)` + +- `GetAsync()` → `User?`. For `Me()` the raw payload includes private account details + (`ToJsonObject()`). +- `MonthlyUsageAsync(string? date = null)` → `JsonObject` (only for `Me()`). +- `LimitsAsync()` / `UpdateLimitsAsync(object newLimits)` (only for `Me()`). + +```csharp +using System; +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +var me = await client.Me().GetAsync(); +Console.WriteLine(me?.Username); +var usage = await client.Me().MonthlyUsageAsync(); +``` + +## Logs — `client.Log(buildOrRunId)` + +- `GetAsync(LogOptions?)` → `string?` (buffered). +- `StreamAsync(LogOptions?)` → `Stream` (live). Also `client.Run(id).GetStreamedLogAsync()`. + +`LogOptions`: `Raw`, `Download`. + +```csharp +using System; +using System.IO; +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +using var stream = await client.Run("some-run-id").GetStreamedLogAsync(); +using var reader = new StreamReader(stream); +Console.WriteLine(await reader.ReadToEndAsync()); +``` diff --git a/docs/runs.md b/docs/runs.md new file mode 100644 index 0000000..489ee0c --- /dev/null +++ b/docs/runs.md @@ -0,0 +1,38 @@ +# Runs + +Access the account-wide run collection with `client.Runs()`, an Actor's or task's runs with +`client.Actor(id).Runs()` / `client.Task(id).Runs()`, and a specific run with `client.Run(runId)`. + +## Collection + +- `ListAsync(ListOptions? options = null, RunListOptions? filter = null)` → `PaginationList`. + `RunListOptions`: `Status` (list), `StartedAfter`, `StartedBefore`. + +## Single run — `client.Run(runId)` + +- `GetAsync(int? waitForFinishSecs = null)` → `ActorRun?`. +- `UpdateAsync(object newFields)` → `ActorRun`. +- `DeleteAsync()`. +- `AbortAsync(bool? gracefully = null)` → `ActorRun`. +- `MetamorphAsync(string targetActorId, object? input = null, MetamorphOptions? options = null)` → `ActorRun`. +- `RebootAsync()` → `ActorRun`. +- `ResurrectAsync(RunResurrectOptions? options = null)` → `ActorRun`. +- `ChargeAsync(RunChargeOptions options)` — record pay-per-event charges (idempotent). +- `WaitForFinishAsync(int? waitSecs = null)` → `ActorRun`. +- `Dataset()`, `KeyValueStore()`, `RequestQueue()` — the run's default storages. +- `Log()` → `LogClient`; `GetStreamedLogAsync()` → `Stream` (live raw log). + +```csharp +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + +// Read the run's default dataset and key-value store. +var items = await client.Run(run.Id!).Dataset().ListItemsAsync(); +var record = await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); + +// Charge a pay-per-event run. +await client.Run(run.Id!).ChargeAsync(new RunChargeOptions("result", count: 3)); +``` diff --git a/docs/schedules.md b/docs/schedules.md new file mode 100644 index 0000000..58d9bf0 --- /dev/null +++ b/docs/schedules.md @@ -0,0 +1,29 @@ +# Schedules + +Schedules automatically start Actor or task runs at specified times. Access the collection with +`client.Schedules()` and a specific schedule with `client.Schedule(id)`. + +## Collection + +- `ListAsync(ListOptions?)` → `PaginationList`. +- `CreateAsync(object schedule)` → `Schedule`. + +## Single schedule — `client.Schedule(id)` + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `GetLogAsync()` → `string?` (invocation log; `null` if none yet). + +```csharp +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +var schedule = await client.Schedules().CreateAsync(new +{ + name = "nightly", + cronExpression = "0 0 * * *", + isEnabled = true, + actions = new object[] { new { type = "RUN_ACTOR", actorId = "apify/hello-world" } }, +}); + +await client.Schedule(schedule.Id!).UpdateAsync(new { cronExpression = "0 12 * * *" }); +``` diff --git a/docs/storages.md b/docs/storages.md new file mode 100644 index 0000000..db0218d --- /dev/null +++ b/docs/storages.md @@ -0,0 +1,119 @@ +# Storages + +The three storage types — datasets, key-value stores and request queues — share the same collection +shape: `ListAsync(StorageListOptions?)` and `GetOrCreateAsync(name?)`. Storages can also be reached +from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). + +> Snippets below run inside an `async` context. `ImplicitUsings` is disabled in this repository, so all +> `using` directives (including `System`) are shown explicitly and must precede any statements. + +`StorageListOptions`: `Offset`, `Limit`, `Desc`, `Unnamed`, `Ownership`. + +## Datasets + +`client.Datasets()` / `client.Dataset(id)`. + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `ListItemsAsync(DatasetListItemsOptions?)` → `PaginationList` (pagination via response headers). +- `DownloadItemsAsync(DownloadItemsFormat, DatasetDownloadOptions?)` → serialized items as `byte[]` + (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). +- `PushItemsAsync(object items)` — push one object or an array of objects. +- `GetStatisticsAsync()` → `JsonObject?`. +- `CreateItemsPublicUrlAsync(DatasetListItemsOptions?, int? expiresInSecs = null)` → signed public URL. + +```csharp +using System; +using System.Text; +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var dataset = await client.Datasets().GetOrCreateAsync("my-dataset"); +await client.Dataset(dataset.Id!).PushItemsAsync(new[] { new { url = "https://a.com", n = 1 } }); +var page = await client.Dataset(dataset.Id!).ListItemsAsync(new DatasetListItemsOptions { Limit = 100 }); +Console.WriteLine(page.Count); // items in this page; page.Total is the count across all pages +var csvBytes = await client.Dataset(dataset.Id!).DownloadItemsAsync(DownloadItemsFormat.Csv); +Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the raw bytes +``` + +## Key-value stores + +`client.KeyValueStores()` / `client.KeyValueStore(id)`. + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `ListKeysAsync(ListKeysOptions?)` → `KeyValueStoreKeysPage`. +- `RecordExistsAsync(key)` → `bool`. +- `GetRecordAsync(key, GetRecordOptions?)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a + `byte[]` of the record's raw bytes (so binary records survive intact); decode it according to + `KeyValueStoreRecord.ContentType` — e.g. `Encoding.UTF8.GetString(record.Value)` for text, or + `JsonSerializer.Deserialize(record.Value)` for JSON. +- `SetRecordAsync(key, byte[] value, contentType, SetRecordOptions?)` and `SetRecordJsonAsync(key, value)` + (serializes `value` to JSON bytes). +- `DeleteRecordAsync(key)`. +- `GetRecordPublicUrlAsync(key)` and `CreateKeysPublicUrlAsync(ListKeysOptions?, int? expiresInSecs)`. + +```csharp +using System; +using System.Text; +using System.Text.Json; +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +var store = await client.KeyValueStores().GetOrCreateAsync("my-store"); + +// Write JSON, then read it back and decode the raw bytes. +await client.KeyValueStore(store.Id!).SetRecordJsonAsync("OUTPUT", new { answer = 42 }); +var record = await client.KeyValueStore(store.Id!).GetRecordAsync("OUTPUT"); +if (record is not null) +{ + Console.WriteLine("content type: " + record.ContentType); + Console.WriteLine("as text: " + Encoding.UTF8.GetString(record.Value)); + var output = JsonSerializer.Deserialize(record.Value); + Console.WriteLine("answer: " + output.GetProperty("answer").GetInt32()); +} + +// Write raw bytes directly (binary-safe). +await client.KeyValueStore(store.Id!).SetRecordAsync("blob", new byte[] { 0x00, 0xFF }, "application/octet-stream"); +``` + +## Request queues + +`client.RequestQueues()` / `client.RequestQueue(id, RequestQueueClientOptions?)`. The options set a +stable `ClientKey` (required to manage locks the client created) and a per-queue `TimeoutSecs`. + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `AddRequestAsync(RequestQueueRequest, bool forefront = false)` → `RequestQueueOperationInfo`. +- `GetRequestAsync(id)`, `UpdateRequestAsync(request, forefront)`, `DeleteRequestAsync(id)`. +- `ListHeadAsync(int? limit)` → `RequestQueueHead`; `ListAndLockHeadAsync(lockSecs, limit?)`. +- `BatchAddRequestsAsync(IReadOnlyList, forefront, BatchAddRequestsOptions?)` — + auto-chunks by count (25) and payload size (~9 MiB) and retries unprocessed requests. Every request + needs a non-empty `UniqueKey`. +- `ListRequestsAsync(ListRequestsOptions?)` and `PaginateRequestsAsync(PaginateRequestsOptions?)` + (`IAsyncEnumerable`). +- Lock management: `ProlongRequestLockAsync`, `DeleteRequestLockAsync`, `UnlockRequestsAsync`. + +```csharp +using System; +using System.Collections.Generic; +using Apify.Client; +using Apify.Client.Models; + +var client = new ApifyClient("my-api-token"); +var queue = await client.RequestQueues().GetOrCreateAsync("my-queue"); +var rq = client.RequestQueue(queue.Id!); + +await rq.AddRequestAsync(new RequestQueueRequest("https://example.com", "example")); + +var batch = new List(); +for (var i = 0; i < 100; i++) +{ + batch.Add(new RequestQueueRequest($"https://example.com/{i}", $"key-{i}")); +} +var result = await rq.BatchAddRequestsAsync(batch); +Console.WriteLine(result.ProcessedRequests.Count); + +await foreach (var request in rq.PaginateRequestsAsync()) +{ + Console.WriteLine(request.Url); +} +``` diff --git a/docs/tasks.md b/docs/tasks.md new file mode 100644 index 0000000..d5e6e33 --- /dev/null +++ b/docs/tasks.md @@ -0,0 +1,36 @@ +# Tasks + +Tasks are pre-configured Actor runs with stored input. Access the collection with `client.Tasks()` and +a specific task with `client.Task(id)`. + +## Collection + +- `ListAsync(ListOptions?)` → `PaginationList`. +- `CreateAsync(object task)` → `ActorTask`. + +## Single task — `client.Task(id)` + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `StartAsync(object? input = null, TaskStartOptions? options = null)` → `ActorRun`. +- `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null)` → `ActorRun`. +- `GetInputAsync()` / `UpdateInputAsync(object input)`. +- `LastRun(LastRunOptions?)` → `RunClient`; `Runs()` → `RunCollectionClient`. +- `Webhooks()` → read-only `NestedWebhookCollectionClient`. + +The model is named `ActorTask` (not `Task`) to avoid colliding with `System.Threading.Tasks.Task`. + +```csharp +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +var task = await client.Tasks().CreateAsync(new +{ + actId = "apify/hello-world", + name = "my-task", + input = new { message = "hello" }, +}); + +await client.Task(task.Id!).UpdateInputAsync(new { message = "updated" }); +var run = await client.Task(task.Id!).CallAsync(null, null, 120); +Console.WriteLine(run.Status); +``` diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 0000000..6a5a611 --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,36 @@ +# Webhooks + +Webhooks notify an external service when specific events occur. Access the account-wide collection with +`client.Webhooks()` and a specific webhook with `client.Webhook(id)`. Webhook dispatches are read with +`client.WebhookDispatches()` and `client.WebhookDispatch(id)`. + +## Webhook collection — `client.Webhooks()` + +- `ListAsync(ListOptions?)` → `PaginationList`. +- `CreateAsync(object webhook)` → `Webhook`. + +Webhooks nested under an Actor or task (`client.Actor(id).Webhooks()`, `client.Task(id).Webhooks()`) +are **read-only** (list only); create account-wide webhooks that target an Actor/task via the webhook's +`condition`. + +## Single webhook — `client.Webhook(id)` + +- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `TestAsync()` → `WebhookDispatch` (dispatch immediately). +- `Dispatches()` → `WebhookDispatchCollectionClient`. + +```csharp +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var webhook = await client.Webhooks().CreateAsync(new +{ + eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, + condition = new { actorId = "apify/hello-world" }, + requestUrl = "https://example.com/webhook", +}); + +await client.Webhook(webhook.Id!).UpdateAsync(new { requestUrl = "https://example.com/updated" }); +await client.Webhook(webhook.Id!).Dispatches().ListAsync(new ListOptions { Limit = 10 }); +``` diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj new file mode 100644 index 0000000..9c2987c --- /dev/null +++ b/src/Apify.Client/Apify.Client.csproj @@ -0,0 +1,33 @@ + + + + Apify.Client + Apify.Client + + + Apify.Client + 0.1.0 + Apify + Apify + Apify API client for .NET + Official, experimental (AI-generated and AI-maintained) Apify API client for .NET: run Actors, manage storages (datasets, key-value stores, request queues), schedules, webhooks and more. + apify;api;client;actor;scraping;crawler;automation + Apache-2.0 + https://github.com/apify/apify-client-dotnet + https://github.com/apify/apify-client-dotnet + git + README.md + true + snupkg + + + + + + + + + + + + diff --git a/src/Apify.Client/ApifyClient.cs b/src/Apify.Client/ApifyClient.cs new file mode 100644 index 0000000..19ba06e --- /dev/null +++ b/src/Apify.Client/ApifyClient.cs @@ -0,0 +1,289 @@ +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Http; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Resources; + +namespace Apify.Client; + +/// +/// The entry point for interacting with the Apify API. +/// +/// +/// +/// Official, but experimental — AI-generated and AI-maintained. This is an official Apify client, +/// but it is experimental: it is generated and maintained by AI. Review the code before relying on it in +/// production and report issues on the repository. +/// +/// +/// Construct it with an API token (and optional settings via ), then +/// obtain resource clients via the accessor methods, e.g. , , +/// . +/// +/// +/// Architecture. The public interface is this class and the resource clients it returns. The +/// replaceable transport is the (default ); +/// pass a custom one via . Cross-cutting behaviour (auth, +/// User-Agent, retries with exponential backoff, timeouts) lives in the internal HTTP client and is +/// applied to every request. +/// +/// +public sealed class ApifyClient +{ + /// Default base URL of the Apify API (without the /v2 suffix). + public const string DefaultBaseUrl = "https://api.apify.com"; + + /// Default maximum number of retries for failed requests. + public const int DefaultMaxRetries = 8; + + /// Default minimum delay between retries, in milliseconds. + public const int DefaultMinDelayMillis = 500; + + /// Default overall per-request timeout, in seconds. + public const int DefaultTimeoutSecs = 360; + + /// Environment variable that signals the client is running on the Apify platform. + private const string EnvIsAtHome = "APIFY_IS_AT_HOME"; + + /// Environment variable holding the current Actor run's id (set on the platform). + private const string EnvActorRunId = "ACTOR_RUN_ID"; + + /// Addresses the current user (/users/me). + private const string MeUserPlaceholder = "me"; + + private readonly HttpClientCore _http; + private readonly string _baseUrl; + private readonly string _publicBaseUrl; + + /// Creates a client with the given API token and otherwise default settings. + /// API token, sent as a Bearer token. + public ApifyClient(string? token = null) + : this(new ApifyClientOptions { Token = token }) + { + } + + /// Creates a client from the given options. + /// The client configuration. + public ApifyClient(ApifyClientOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var transport = options.HttpTransport ?? new HttpClientTransport(); + var maxDelayMillis = options.MaxDelayBetweenRetriesMillis ?? (options.TimeoutSecs * 1000); + var retry = new RetryConfig( + options.MaxRetries, + options.MinDelayBetweenRetriesMillis, + maxDelayMillis, + options.TimeoutSecs); + + var userAgent = BuildUserAgent(options.UserAgentSuffix, options.IsAtHome ?? DefaultIsAtHome); + _http = new HttpClientCore(transport, options.Token, userAgent, retry); + + _baseUrl = TrimTrailingSlash(options.BaseUrl) + "/v2"; + var publicSource = options.PublicBaseUrl ?? options.BaseUrl; + _publicBaseUrl = TrimTrailingSlash(publicSource) + "/v2"; + } + + /// The User-Agent header value this client sends. + public string UserAgent => _http.UserAgent; + + /// The fully-qualified API base URL this client targets (including the /v2 suffix). + public string ApiBaseUrl => _baseUrl; + + // ----- Actor accessors ----------------------------------------------------- + + /// A client for the Actor collection (list & create Actors). + public ActorCollectionClient Actors() => new(_http, _baseUrl); + + /// A client for a specific Actor, addressed by ID or username~name. + /// The Actor ID or username~name. + public ActorClient Actor(string id) => new(this, _http, _baseUrl, id); + + // ----- Build accessors ----------------------------------------------------- + + /// A client for the Actor build collection (list builds). + public BuildCollectionClient Builds() => new(_http, _baseUrl, "actor-builds"); + + /// A client for a specific Actor build. + /// The build ID. + public BuildClient Build(string id) => new(_http, _baseUrl, id); + + // ----- Run accessors ------------------------------------------------------- + + /// A client for the Actor run collection (list runs). + public RunCollectionClient Runs() => new(_http, _baseUrl, "actor-runs"); + + /// A client for a specific Actor run. + /// The run ID. + public RunClient Run(string id) => new(_http, _baseUrl, "actor-runs", id); + + // ----- Dataset accessors --------------------------------------------------- + + /// A client for the dataset collection (list & get-or-create datasets). + public DatasetCollectionClient Datasets() => new(_http, _baseUrl); + + /// A client for a specific dataset, addressed by ID or name. + /// The dataset ID or name. + public DatasetClient Dataset(string id) => DatasetClient.ForId(_http, _baseUrl, id).WithPublicBase(_publicBaseUrl); + + // ----- Key-value store accessors ------------------------------------------- + + /// A client for the key-value store collection. + public KeyValueStoreCollectionClient KeyValueStores() => new(_http, _baseUrl); + + /// A client for a specific key-value store, addressed by ID or name. + /// The store ID or name. + public KeyValueStoreClient KeyValueStore(string id) => KeyValueStoreClient.ForId(_http, _baseUrl, id).WithPublicBase(_publicBaseUrl); + + // ----- Request queue accessors --------------------------------------------- + + /// A client for the request queue collection. + public RequestQueueCollectionClient RequestQueues() => new(_http, _baseUrl); + + /// + /// A client for a specific request queue, addressed by ID or name. Optionally pass options to set a + /// stable ClientKey and/or a per-request TimeoutSecs for this queue's calls. + /// + /// The queue ID or name. + /// Optional per-queue-client options. + public RequestQueueClient RequestQueue(string id, Options.RequestQueueClientOptions? options = null) + => RequestQueueClient.ForId(_http, _baseUrl, id, options); + + // ----- Task accessors ------------------------------------------------------ + + /// A client for the Actor task collection (list & create tasks). + public TaskCollectionClient Tasks() => new(_http, _baseUrl); + + /// A client for a specific Actor task. + /// The task ID. + public TaskClient Task(string id) => new(this, _http, _baseUrl, id); + + // ----- Schedule accessors -------------------------------------------------- + + /// A client for the schedule collection (list & create schedules). + public ScheduleCollectionClient Schedules() => new(_http, _baseUrl); + + /// A client for a specific schedule. + /// The schedule ID. + public ScheduleClient Schedule(string id) => new(_http, _baseUrl, id); + + // ----- Webhook accessors --------------------------------------------------- + + /// A client for the webhook collection (list & create webhooks). + public WebhookCollectionClient Webhooks() => new(_http, _baseUrl); + + /// A client for a specific webhook. + /// The webhook ID. + public WebhookClient Webhook(string id) => new(_http, _baseUrl, id); + + /// A client for the webhook dispatch collection. + public WebhookDispatchCollectionClient WebhookDispatches() => new(_http, _baseUrl, "webhook-dispatches"); + + /// A client for a specific webhook dispatch. + /// The dispatch ID. + public WebhookDispatchClient WebhookDispatch(string id) => new(_http, _baseUrl, id); + + // ----- Misc accessors ------------------------------------------------------ + + /// A client for browsing the Apify Store. + public StoreCollectionClient Store() => new(_http, _baseUrl); + + /// A client for accessing a build's or run's log. + /// The build or run ID. + public LogClient Log(string buildOrRunId) => LogClient.ForId(_http, _baseUrl, buildOrRunId); + + /// A client for the current user (/users/me). + public UserClient Me() => new(_http, _baseUrl, MeUserPlaceholder); + + /// A client for a specific user by ID or username. + /// The user ID or username. + public UserClient User(string id) => new(_http, _baseUrl, id); + + /// + /// Sets the status message of the current Actor run. + /// + /// + /// This convenience method updates the run identified by the ACTOR_RUN_ID environment variable, + /// so it only works when called from inside an Actor run. If is true, the + /// message becomes final and won't be overwritten. Throws if + /// ACTOR_RUN_ID is not set. + /// + /// The status message to set. + /// Whether the message is final. + /// A token to cancel the request. + public Task SetStatusMessageAsync(string message, bool isTerminal = false, CancellationToken cancellationToken = default) + { + var runId = Environment.GetEnvironmentVariable(EnvActorRunId); + if (string.IsNullOrEmpty(runId)) + { + throw new InvalidOperationException("ACTOR_RUN_ID environment variable is not set"); + } + + var fields = new JsonObject + { + ["statusMessage"] = message, + ["isStatusMessageTerminal"] = isTerminal, + }; + return Run(runId).UpdateAsync(fields, cancellationToken); + } + + private static string TrimTrailingSlash(string value) => value.TrimEnd('/'); + + /// + /// Reports whether the client is running on the Apify platform, by reading the APIFY_IS_AT_HOME + /// environment variable (set to a non-empty value on the platform). + /// + private static bool DefaultIsAtHome() + { + var value = Environment.GetEnvironmentVariable(EnvIsAtHome); + return !string.IsNullOrEmpty(value); + } + + /// + /// Builds the User-Agent header value mandated by the client requirements: + /// ApifyClient/{version} ({os}; .NET/{runtimeVersion}); isAtHome/{true|false}. + /// + private static string BuildUserAgent(string? suffix, Func isAtHomeFn) + { + var os = CurrentOs(); + var atHome = isAtHomeFn() ? "true" : "false"; + var ua = string.Format( + CultureInfo.InvariantCulture, + "ApifyClient/{0} ({1}; .NET/{2}); isAtHome/{3}", + ApifyClientVersion.ClientVersion, + os, + Environment.Version, + atHome); + if (!string.IsNullOrEmpty(suffix)) + { + ua += "; " + suffix; + } + + return ua; + } + + /// The lowercase operating-system family name, matching the reference clients' convention. + private static string CurrentOs() + { + if (OperatingSystem.IsWindows()) + { + return "windows"; + } + + if (OperatingSystem.IsMacOS()) + { + return "darwin"; + } + + if (OperatingSystem.IsLinux()) + { + return "linux"; + } + + return "unknown"; + } +} diff --git a/src/Apify.Client/ApifyClientOptions.cs b/src/Apify.Client/ApifyClientOptions.cs new file mode 100644 index 0000000..ce89c42 --- /dev/null +++ b/src/Apify.Client/ApifyClientOptions.cs @@ -0,0 +1,41 @@ +using System; +using Apify.Client.Http; + +namespace Apify.Client; + +/// +/// Configuration for an . All fields have sensible defaults; set only the ones +/// you need to override. +/// +public sealed class ApifyClientOptions +{ + /// API token, sent as a Bearer token. + public string? Token { get; set; } + + /// API base URL; the /v2 suffix is appended automatically. + public string BaseUrl { get; set; } = ApifyClient.DefaultBaseUrl; + + /// Base URL for building public, shareable resource URLs (defaults to ). + public string? PublicBaseUrl { get; set; } + + /// Maximum retries for failed requests (default 8). + public int MaxRetries { get; set; } = ApifyClient.DefaultMaxRetries; + + /// Minimum delay between retries in ms (default 500). + public int MinDelayBetweenRetriesMillis { get; set; } = ApifyClient.DefaultMinDelayMillis; + + /// Upper bound for the growing inter-retry delay in ms (defaults to the request timeout). + public int? MaxDelayBetweenRetriesMillis { get; set; } + + /// Overall per-request timeout in seconds (default 360). + public int TimeoutSecs { get; set; } = ApifyClient.DefaultTimeoutSecs; + + /// Custom suffix appended to the User-Agent header. + public string? UserAgentSuffix { get; set; } + + /// Replaces the default transport (). + public IHttpTransport? HttpTransport { get; set; } + + /// Test seam overriding the isAtHome flag detection. + public Func? IsAtHome { get; set; } +} diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs new file mode 100644 index 0000000..7d5e43f --- /dev/null +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -0,0 +1,24 @@ +namespace Apify.Client; + +/// +/// Public version constants for the Apify .NET client. +/// +/// +/// is the semantic version of this library and +/// is the info.version of the Apify OpenAPI specification this +/// client was generated and verified against. +/// +public static class ApifyClientVersion +{ + /// + /// The semantic version of this client library (see https://semver.org/). Changes to the public + /// interface other than additive ones are considered breaking changes. + /// + public const string ClientVersion = "0.1.0"; + + /// + /// The version of the Apify OpenAPI specification this client was generated and verified against. + /// Corresponds to the info.version field of the Apify OpenAPI document. + /// + public const string ApiSpecVersion = "v2-2026-07-02T131926Z"; +} diff --git a/src/Apify.Client/Exceptions/ApifyApiException.cs b/src/Apify.Client/Exceptions/ApifyApiException.cs new file mode 100644 index 0000000..36ebaa6 --- /dev/null +++ b/src/Apify.Client/Exceptions/ApifyApiException.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json.Nodes; + +namespace Apify.Client.Exceptions; + +/// +/// Thrown for HTTP requests that reach the Apify API but receive a non-success status code. +/// +/// +/// It mirrors the ApifyApiError of the reference JavaScript client and exposes the parsed error +/// , the human-readable , the HTTP , +/// the number of the final , and the request /. +/// +public class ApifyApiException : Exception +{ + /// Creates an API exception with the parsed error details. + /// The HTTP status code of the error response. + /// The machine-readable error type returned by the API, if any. + /// The raw error message returned by the API. + /// The 1-based number of the attempt that produced this error. + /// The HTTP method of the API call. + /// The path of the API endpoint (URL excluding origin). + /// Additional structured error data provided by the API, if any. + public ApifyApiException( + int statusCode, + string? type, + string message, + int attempt, + string httpMethod, + string path, + JsonObject? data = null) + : base(FormatMessage(statusCode, type, message)) + { + StatusCode = statusCode; + Type = type; + ApiMessage = message; + Attempt = attempt; + HttpMethod = httpMethod; + Path = path; + ErrorData = data; + } + + /// The HTTP status code of the error response. + public int StatusCode { get; } + + /// The machine-readable error type returned by the API (e.g. record-not-found). + public string? Type { get; } + + /// The raw error message returned by the API, without the status/type prefix. + public string ApiMessage { get; } + + /// The number of the API call attempt that produced this error (1-based). + public int Attempt { get; } + + /// The HTTP method of the API call (e.g. GET, POST). + public string HttpMethod { get; } + + /// The path of the API endpoint (URL excluding origin). + public string Path { get; } + + /// + /// Additional structured data provided by the API about the error, if any. Named ErrorData + /// (not Data) to avoid hiding . + /// + public JsonObject? ErrorData { get; } + + private static string FormatMessage(int statusCode, string? type, string message) + { + var errType = string.IsNullOrEmpty(type) ? "unknown" : type; + return string.Format( + CultureInfo.InvariantCulture, + "apify API error (status {0}, type {1}): {2}", + statusCode, + errType, + message); + } +} diff --git a/src/Apify.Client/Exceptions/ApifyTransportException.cs b/src/Apify.Client/Exceptions/ApifyTransportException.cs new file mode 100644 index 0000000..8190e57 --- /dev/null +++ b/src/Apify.Client/Exceptions/ApifyTransportException.cs @@ -0,0 +1,26 @@ +using System; + +namespace Apify.Client.Exceptions; + +/// +/// Marks a transport-level (network/timeout) failure, which is retryable by the client. +/// +/// +/// A custom should throw this for connection, DNS and +/// timeout failures; a non-2xx HTTP status must be returned as a normal response instead. +/// +public sealed class ApifyTransportException : Exception +{ + /// Creates a transport exception. + /// A description of the failure. + /// The underlying exception, if any. + /// Whether the failure was caused by a request timeout. + public ApifyTransportException(string message, Exception? innerException = null, bool isTimeout = false) + : base(message, innerException) + { + IsTimeout = isTimeout; + } + + /// Whether the failure was caused by a request timeout. + public bool IsTimeout { get; } +} diff --git a/src/Apify.Client/Http/HttpClientTransport.cs b/src/Apify.Client/Http/HttpClientTransport.cs new file mode 100644 index 0000000..89ec4be --- /dev/null +++ b/src/Apify.Client/Http/HttpClientTransport.cs @@ -0,0 +1,97 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; + +namespace Apify.Client.Http; + +/// +/// The default , backed by . +/// +/// +/// The per-attempt timeout is applied to each request by the orchestrating client via a linked +/// cancellation token, so the shared is left infinite. Non-2xx statuses +/// are returned as normal responses; only connection/timeout failures are thrown as +/// . +/// +public sealed class HttpClientTransport : IHttpTransport, IDisposable +{ + /// Connection-establishment timeout (distinct from the per-request timeout the client applies). + private static readonly TimeSpan ConnectTimeout = TimeSpan.FromSeconds(30); + + private readonly HttpClient _httpClient; + private readonly bool _ownsClient; + + /// + /// Creates a transport, optionally wrapping a caller-supplied (e.g. one + /// configured with a proxy or custom TLS). When none is given, an internal client is created and + /// disposed with this instance. + /// + /// An optional pre-configured HTTP client to use. + public HttpClientTransport(HttpClient? httpClient = null) + { + if (httpClient is not null) + { + _httpClient = httpClient; + _ownsClient = false; + } + else + { + var handler = new SocketsHttpHandler + { + ConnectTimeout = ConnectTimeout, + AllowAutoRedirect = true, + }; + // The client-side retry orchestrator owns the per-request timeout, so disable HttpClient's own. + _httpClient = new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan }; + _ownsClient = true; + } + } + + /// + public async Task SendAsync( + HttpRequestMessage request, + TimeSpan timeout, + bool streaming, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (timeout > TimeSpan.Zero) + { + timeoutCts.CancelAfter(timeout); + } + + var completion = streaming + ? HttpCompletionOption.ResponseHeadersRead + : HttpCompletionOption.ResponseContentRead; + + try + { + return await _httpClient.SendAsync(request, completion, timeoutCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller cancelled: propagate rather than treating it as a retryable timeout. + throw; + } + catch (OperationCanceledException ex) + { + // The per-attempt deadline elapsed (only timeoutCts fired). + throw new ApifyTransportException("the request timed out", ex, isTimeout: true); + } + catch (HttpRequestException ex) + { + throw new ApifyTransportException(ex.Message, ex, isTimeout: false); + } + } + + /// Disposes the internally-created (a supplied one is left alone). + public void Dispose() + { + if (_ownsClient) + { + _httpClient.Dispose(); + } + } +} diff --git a/src/Apify.Client/Http/IHttpTransport.cs b/src/Apify.Client/Http/IHttpTransport.cs new file mode 100644 index 0000000..fac5d16 --- /dev/null +++ b/src/Apify.Client/Http/IHttpTransport.cs @@ -0,0 +1,46 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace Apify.Client.Http; + +/// +/// The replaceable transport contract of the client. +/// +/// +/// +/// Implementations are responsible only for sending a single, fully-prepared +/// and returning the raw . Authentication, the User-Agent header, +/// retries and (de)serialization are handled by the client, so a backend only needs to perform one +/// network round-trip. +/// +/// +/// A non-2xx HTTP status is not an error at this layer — return it as a normal response. Only +/// transport-level failures (connection refused, DNS, timeout) should be thrown, as an +/// . +/// +/// +/// Swap the default implementation () via +/// to share a connection pool, customize +/// TLS/proxy settings, or inject a mock in tests. +/// +/// +public interface IHttpTransport +{ + /// + /// Sends a single request with a per-attempt timeout and returns the response. + /// + /// The fully-prepared request (headers and body already set). + /// The per-attempt timeout budget. + /// + /// When true, return as soon as the response headers arrive so the body can be consumed as a + /// live stream (used by log streaming); otherwise the whole response may be buffered. + /// + /// A token to cancel the request. + Task SendAsync( + HttpRequestMessage request, + TimeSpan timeout, + bool streaming, + CancellationToken cancellationToken); +} diff --git a/src/Apify.Client/Internal/HttpClientCore.cs b/src/Apify.Client/Internal/HttpClientCore.cs new file mode 100644 index 0000000..8334ca8 --- /dev/null +++ b/src/Apify.Client/Internal/HttpClientCore.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Http; + +namespace Apify.Client.Internal; + +/// +/// The orchestrating HTTP client shared by every resource client. It owns the transport, the optional +/// API token, the User-Agent, and the retry/timeout policy, and applies them to every request. +/// +internal sealed class HttpClientCore +{ + /// Status returned when the per-resource rate limit is hit. + private const int RateLimitExceeded = 429; + + /// Statuses at or above this value are treated as retryable internal server errors. + private const int MinServerError = 500; + + /// Responses with a status below this value are treated as success. + public const int MaxSuccessStatus = 300; + + /// Exponential-backoff multiplier applied to the inter-retry delay after each attempt. + private const int BackoffFactor = 2; + + private const int NotFound = 404; + + private readonly IHttpTransport _transport; + private readonly string? _token; + private readonly RetryConfig _retry; + + public HttpClientCore(IHttpTransport transport, string? token, string userAgent, RetryConfig retry) + { + _transport = transport; + _token = token; + UserAgent = userAgent; + _retry = retry; + } + + /// The User-Agent header value this client sends. + public string UserAgent { get; } + + /// The configured overall per-request timeout budget, in seconds. + public double RequestTimeoutSecs => _retry.TimeoutSecs; + + /// + /// Sends a request with auth, User-Agent and the retry policy applied, returning the successful + /// response (the caller owns and disposes it). + /// + public async Task CallAsync( + HttpMethod method, + string url, + string? body = null, + string contentType = "", + TimeSpan? timeout = null, + bool doNotRetryTimeouts = false, + byte[]? bodyBytes = null, + IReadOnlyDictionary? extraHeaders = null, + CancellationToken cancellationToken = default) + { + var delayMillis = _retry.MinDelayMillis; + var maxAttempts = _retry.MaxRetries + 1; + var path = ExtractPath(url); + var baseTimeout = timeout ?? TimeSpan.FromSeconds(_retry.TimeoutSecs); + Exception? lastError = null; + + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + bool retryable; + try + { + var response = await SendOnceAsync( + method, url, body, bodyBytes, contentType, extraHeaders, + AttemptTimeout(baseTimeout, attempt), cancellationToken).ConfigureAwait(false); + + var status = (int)response.StatusCode; + if (status < MaxSuccessStatus) + { + return response; + } + + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + response.Dispose(); + lastError = BuildApiError(status, errorBody, attempt, method.Method, path); + retryable = IsStatusRetryable(status); + } + catch (ApifyTransportException ex) + { + lastError = ex; + // Network/timeout failures are retryable, unless the caller opted out of retrying timeouts. + retryable = !(doNotRetryTimeouts && ex.IsTimeout); + } + + if (!retryable || attempt == maxAttempts) + { + throw lastError; + } + + await Task.Delay(TimeSpan.FromMilliseconds(RandomizedDelayMillis(delayMillis)), cancellationToken) + .ConfigureAwait(false); + delayMillis = Math.Min(delayMillis * BackoffFactor, _retry.MaxDelayMillis); + } + + // Unreachable in practice (maxAttempts >= 1); defensive. + throw lastError ?? new ApifyTransportException("request failed with no attempts"); + } + + /// Opens a live streaming response (single attempt, no retry). Used by log streaming. + public Task StreamAsync(string url, CancellationToken cancellationToken) + { + var request = BuildRequest(HttpMethod.Get, url, null, null, string.Empty, null); + return _transport.SendAsync(request, TimeSpan.FromSeconds(_retry.TimeoutSecs), streaming: true, cancellationToken); + } + + private async Task SendOnceAsync( + HttpMethod method, + string url, + string? body, + byte[]? bodyBytes, + string contentType, + IReadOnlyDictionary? extraHeaders, + TimeSpan timeout, + CancellationToken cancellationToken) + { + using var request = BuildRequest(method, url, body, bodyBytes, contentType, extraHeaders); + return await _transport.SendAsync(request, timeout, streaming: false, cancellationToken).ConfigureAwait(false); + } + + /// Builds a fully-prepared request with auth, User-Agent, content type and extra headers. + private HttpRequestMessage BuildRequest( + HttpMethod method, + string url, + string? body, + byte[]? bodyBytes, + string contentType, + IReadOnlyDictionary? extraHeaders) + { + var request = new HttpRequestMessage(method, url); + request.Headers.TryAddWithoutValidation("User-Agent", UserAgent); + if (!string.IsNullOrEmpty(_token)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); + } + + if (extraHeaders is not null) + { + foreach (var header in extraHeaders) + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + + // Raw bytes take precedence so binary records (e.g. images, gzip) are sent verbatim; a string body + // is UTF-8 encoded. Setting the content type verbatim (no charset appended unless the caller added one). + HttpContent? content = bodyBytes is not null + ? new ByteArrayContent(bodyBytes) + : body is not null ? new StringContent(body, Encoding.UTF8) : null; + if (content is not null) + { + content.Headers.ContentType = string.IsNullOrEmpty(contentType) + ? null + : MediaTypeHeaderValue.Parse(contentType); + request.Content = content; + } + + return request; + } + + /// + /// Returns min(overall, base * 2^(attempt-1)): the first attempt uses the base timeout; each + /// retry doubles it (a slow-but-progressing connection gets more time) while never exceeding the + /// overall budget. + /// + private TimeSpan AttemptTimeout(TimeSpan baseTimeout, int attempt) + { + var overall = TimeSpan.FromSeconds(_retry.TimeoutSecs); + var scaled = baseTimeout; + for (var i = 1; i < attempt; i++) + { + scaled *= 2; + if (scaled >= overall) + { + return overall; + } + } + + return scaled < overall ? scaled : overall; + } + + private static bool IsStatusRetryable(int status) => status == RateLimitExceeded || status >= MinServerError; + + /// Returns a delay chosen randomly from [delay, 2*delay) (exponential backoff + jitter). + private static double RandomizedDelayMillis(double delayMillis) + { + if (delayMillis <= 0) + { + return delayMillis; + } + + return delayMillis + (Random.Shared.NextDouble() * delayMillis); + } + + /// Builds an from an API error response body. + public static ApifyApiException BuildApiError(int status, string body, int attempt, string method, string path) + { + string? type = null; + string? message = null; + System.Text.Json.Nodes.JsonObject? data = null; + + if (Json.TryDecode(body) is System.Text.Json.Nodes.JsonObject decoded + && decoded.TryGetPropertyValue("error", out var errorNode) + && errorNode is System.Text.Json.Nodes.JsonObject error) + { + type = AsString(error, "type"); + message = AsString(error, "message"); + if (error.TryGetPropertyValue("data", out var dataNode) + && dataNode is System.Text.Json.Nodes.JsonObject dataObj) + { + data = (System.Text.Json.Nodes.JsonObject)dataObj.DeepClone(); + } + } + + message ??= body.Length == 0 + ? "unexpected error with status " + status.ToString(CultureInfo.InvariantCulture) + : "unexpected error: " + body; + + return new ApifyApiException(status, type, message, attempt, method, path, data); + } + + private static string? AsString(System.Text.Json.Nodes.JsonObject obj, string key) + { + if (obj.TryGetPropertyValue(key, out var node) + && node is System.Text.Json.Nodes.JsonValue value + && value.TryGetValue(out var text)) + { + return text; + } + + return null; + } + + /// Returns the path+query portion of a URL, for error reporting. + public static string ExtractPath(string url) + { + var rest = url; + var scheme = rest.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) + { + rest = rest.Substring(scheme + 3); + } + + var slash = rest.IndexOf('/', StringComparison.Ordinal); + return slash >= 0 ? rest.Substring(slash) : string.Empty; + } + + /// Reports whether an exception represents a "resource not found" API error. + public static bool IsNotFound(Exception ex) + { + if (ex is not ApifyApiException apiError || apiError.StatusCode != NotFound) + { + return false; + } + + return apiError.Type is "record-not-found" or "record-or-token-not-found" + || string.Equals(apiError.HttpMethod, "HEAD", StringComparison.Ordinal); + } +} diff --git a/src/Apify.Client/Internal/Json.cs b/src/Apify.Client/Internal/Json.cs new file mode 100644 index 0000000..ddd8f7f --- /dev/null +++ b/src/Apify.Client/Internal/Json.cs @@ -0,0 +1,65 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Apify.Client.Internal; + +/// +/// Shared JSON (de)serialization for the client. +/// +internal static class Json +{ + /// + /// Serialization options matching the reference client's output: slashes and non-ASCII characters + /// are left unescaped (like PHP's JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE). + /// + private static readonly JsonSerializerOptions SerializerOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// Serializes a value to a JSON string. + public static string Encode(object? value) => JsonSerializer.Serialize(value, SerializerOptions); + + /// Serializes a to a JSON string. + public static string Encode(JsonNode? value) => + value?.ToJsonString(SerializerOptions) ?? "null"; + + /// Decodes a JSON string into a (null for an empty body). + public static JsonNode? Decode(string body) + { + if (string.IsNullOrEmpty(body)) + { + return null; + } + + return JsonNode.Parse(body); + } + + /// + /// Decodes a JSON response body wrapped in a {"data": ...} envelope, returning the unwrapped + /// data value (or null if it is absent/null). + /// + public static JsonNode? DecodeData(string body) + { + if (Decode(body) is JsonObject obj && obj.TryGetPropertyValue("data", out var data)) + { + return data; + } + + return null; + } + + /// Attempts to decode a body, returning null on any parse error. + public static JsonNode? TryDecode(string body) + { + try + { + return Decode(body); + } + catch (JsonException) + { + return null; + } + } +} diff --git a/src/Apify.Client/Internal/JsonValues.cs b/src/Apify.Client/Internal/JsonValues.cs new file mode 100644 index 0000000..2893168 --- /dev/null +++ b/src/Apify.Client/Internal/JsonValues.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Apify.Client.Internal; + +/// +/// Small helpers for reading typed values out of a decoded with fallbacks, +/// used by the page/head models. Absent or mistyped fields fall back rather than throwing. +/// +internal static class JsonValues +{ + /// The object as a , or an empty object if it is not one. + public static JsonObject AsObject(JsonNode? node) => node as JsonObject ?? new JsonObject(); + + /// Reads a string field, or null if absent/not a string. + public static string? String(JsonObject obj, string key) + { + return obj.TryGetPropertyValue(key, out var node) && node?.GetValueKind() == JsonValueKind.String + ? node.GetValue() + : null; + } + + /// Reads an integer field, or if absent/not numeric. + public static long IntOr(JsonObject obj, string key, long fallback) + { + if (obj.TryGetPropertyValue(key, out var node) && node?.GetValueKind() == JsonValueKind.Number + && long.TryParse(node.ToJsonString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var value)) + { + return value; + } + + return fallback; + } + + /// Reads a boolean field, or if absent/not a boolean. + public static bool BoolOr(JsonObject obj, string key, bool fallback) + { + return obj.TryGetPropertyValue(key, out var node) + ? node?.GetValueKind() switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => fallback, + } + : fallback; + } + + /// Returns the items array of a decoded object as a list of . + public static IReadOnlyList ObjectItems(JsonObject obj) + { + var result = new List(); + if (obj.TryGetPropertyValue("items", out var node) && node is JsonArray array) + { + foreach (var item in array) + { + result.Add(item as JsonObject ?? new JsonObject()); + } + } + + return result; + } +} diff --git a/src/Apify.Client/Internal/QueryParams.cs b/src/Apify.Client/Internal/QueryParams.cs new file mode 100644 index 0000000..a42e2f6 --- /dev/null +++ b/src/Apify.Client/Internal/QueryParams.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace Apify.Client.Internal; + +/// +/// An ordered collection of query parameters that omits absent (null) values and encodes +/// booleans as 1/0, matching the Apify API conventions. +/// +internal sealed class QueryParams +{ + private readonly List> _pairs = new(); + + /// Adds a string parameter if the value is non-null. + public QueryParams AddString(string key, string? value) + { + if (value is not null) + { + _pairs.Add(new KeyValuePair(key, value)); + } + + return this; + } + + /// Adds an integer parameter if the value is non-null. + public QueryParams AddInt(string key, long? value) + { + if (value is not null) + { + _pairs.Add(new KeyValuePair(key, value.Value.ToString(CultureInfo.InvariantCulture))); + } + + return this; + } + + /// Adds a floating-point parameter if the value is non-null. + public QueryParams AddDouble(string key, double? value) + { + if (value is not null) + { + // Locale-independent representation without a trailing ".0" for whole numbers. + var text = value.Value.ToString("0.##########", CultureInfo.InvariantCulture); + _pairs.Add(new KeyValuePair(key, text)); + } + + return this; + } + + /// + /// Adds a boolean parameter, encoded as 1/0, if the value is non-null. This matches the + /// JS reference client, whose axios paramsSerializer converts booleans via Number(value). + /// + public QueryParams AddBool(string key, bool? value) + { + if (value is not null) + { + _pairs.Add(new KeyValuePair(key, value.Value ? "1" : "0")); + } + + return this; + } + + /// Adds a comma-joined list parameter if the list is non-null and non-empty. + public QueryParams AddCsv(string key, IReadOnlyList? values) + { + if (values is { Count: > 0 }) + { + _pairs.Add(new KeyValuePair(key, string.Join(",", values))); + } + + return this; + } + + /// Appends an already-stringified key/value pair unconditionally. + public QueryParams AddRaw(string key, string value) + { + _pairs.Add(new KeyValuePair(key, value)); + return this; + } + + /// Whether no parameters have been added. + public bool IsEmpty => _pairs.Count == 0; + + /// Returns a shallow copy of this instance. + public QueryParams Copy() + { + var copy = new QueryParams(); + copy._pairs.AddRange(_pairs); + return copy; + } + + /// Appends all pairs from to this instance. + public QueryParams Extend(QueryParams? other) + { + if (other is not null) + { + _pairs.AddRange(other._pairs); + } + + return this; + } + + /// Appends the parameters to as a URL-encoded query string. + public string ApplyToUrl(string rawUrl) + { + if (_pairs.Count == 0) + { + return rawUrl; + } + + var builder = new StringBuilder(rawUrl); + builder.Append(rawUrl.Contains('?', StringComparison.Ordinal) ? '&' : '?'); + for (var i = 0; i < _pairs.Count; i++) + { + if (i > 0) + { + builder.Append('&'); + } + + builder.Append(Uri.EscapeDataString(_pairs[i].Key)); + builder.Append('='); + builder.Append(Uri.EscapeDataString(_pairs[i].Value)); + } + + return builder.ToString(); + } +} diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs new file mode 100644 index 0000000..097c3d0 --- /dev/null +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -0,0 +1,378 @@ +using System; +using System.Diagnostics; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Exceptions; +using Apify.Client.Models; + +namespace Apify.Client.Internal; + +/// +/// The resolved context for a resource client: its base URL and the shared HTTP client. The methods +/// here implement the CRUD primitives once, so each resource client stays small and consistent (DRY). +/// +internal sealed class ResourceContext +{ + public const string ContentTypeJson = "application/json"; + public const string ContentTypeJsonCharset = "application/json; charset=utf-8"; + + /// How long to wait between polls while waiting for a run/build to finish, in seconds. + private const double WaitPollIntervalSecs = 0.25; + + /// Server-side waitForFinish chunk size (the API caps server waiting at 60 seconds). + private const int WaitRequestSecs = 60; + + /// + /// Safety margin subtracted from the configured per-request timeout when choosing the server-side + /// waitForFinish value, so the server responds before the client's socket timeout fires. + /// + private const int WaitTimeoutMarginSecs = 5; + + /// + /// Finite upper bound used when the caller asks to wait "indefinitely" (waitSecs == null). The + /// API will not accept "Infinity" and an unbounded loop can spin forever on a transient 404; 999999s + /// (~11.5 days) is effectively indefinite while guaranteeing termination. + /// + private const int MaxWaitForFinishSecs = 999999; + + private readonly string _apiOrigin; + private string _publicOrigin; + private TimeSpan? _requestTimeout; + + private ResourceContext(HttpClientCore http, string url, string baseUrl) + { + Http = http; + Url = url; + BaseParams = new QueryParams(); + _apiOrigin = OriginOf(baseUrl); + _publicOrigin = _apiOrigin; + } + + /// The shared orchestrating HTTP client. + public HttpClientCore Http { get; } + + /// Fully-qualified base URL of the resource, e.g. https://api.apify.com/v2/actors/ID. + public string Url { get; } + + /// Query parameters inherited by every call made through this context. + public QueryParams BaseParams { get; } + + /// The per-context request timeout, or null to use the client-wide default. + public TimeSpan? RequestTimeout => _requestTimeout; + + /// Creates a context for a collection endpoint: {base}/{resourcePath}. + public static ResourceContext Collection(HttpClientCore http, string baseUrl, string resourcePath) + => new(http, baseUrl + "/" + resourcePath, baseUrl); + + /// Creates a context for a single resource: {base}/{resourcePath}/{safeId}. + public static ResourceContext Single(HttpClientCore http, string baseUrl, string resourcePath, string id) + => new(http, baseUrl + "/" + resourcePath + "/" + ToSafeId(id), baseUrl); + + /// Sets an overall per-request timeout for every call made through this context. + public ResourceContext WithTimeout(TimeSpan? timeout) + { + _requestTimeout = timeout; + return this; + } + + /// Overrides the origin used when building public URLs. + public ResourceContext WithPublicOrigin(string publicBaseUrl) + { + _publicOrigin = OriginOf(publicBaseUrl); + return this; + } + + /// This resource's URL with an optional extra path segment appended. + public string SubUrl(string subPath = "") => subPath.Length == 0 ? Url : Url + "/" + subPath; + + /// The public (shareable) form of this resource's URL, swapping the API origin for the public one. + public string PublicUrl(string subPath) + { + var apiUrl = SubUrl(subPath); + if (string.Equals(_publicOrigin, _apiOrigin, StringComparison.Ordinal)) + { + return apiUrl; + } + + return apiUrl.StartsWith(_apiOrigin, StringComparison.Ordinal) + ? string.Concat(_publicOrigin, apiUrl.AsSpan(_apiOrigin.Length)) + : apiUrl; + } + + /// Merges the inherited base params with per-call params. + public QueryParams MergedParams(QueryParams? p) => BaseParams.Copy().Extend(p); + + // ---- CRUD primitives ------------------------------------------------------ + + /// GET a single resource, returning its decoded data, or null on not-found. + public async Task GetResourceAsync(string subPath, QueryParams p, CancellationToken ct) + { + try + { + return await GetResourceRequiredAsync(subPath, p, ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// GET a single resource, returning its decoded data (propagates errors). + public async Task GetResourceRequiredAsync(string subPath, QueryParams p, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + return Json.DecodeData(body); + } + + /// PUT to update a resource with a JSON-serializable body, returning the decoded data. + public async Task UpdateResourceAsync(string subPath, object? body, CancellationToken ct) + { + var url = MergedParams(new QueryParams()).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Put, url, Json.Encode(body), ContentTypeJson, _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + } + + /// Performs a DELETE; a not-found is treated as a successful no-op. + public async Task DeleteResourceAsync(string subPath, CancellationToken ct) + { + var url = MergedParams(new QueryParams()).ApplyToUrl(SubUrl(subPath)); + try + { + using var response = await Http.CallAsync(HttpMethod.Delete, url, timeout: _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + // A missing resource is a successful no-op for delete. + } + } + + /// GET a paginated listing and build a with each item hydrated. + public async Task> ListResourceAsync(string subPath, QueryParams p, Func hydrate, CancellationToken ct) + { + var data = await GetResourceRequiredAsync(subPath, p, ct).ConfigureAwait(false); + return PaginationList.FromData(data, hydrate); + } + + /// POST to create a resource with a JSON-serializable body, returning the decoded data. + public async Task CreateResourceAsync(QueryParams p, object? body, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(string.Empty)); + using var response = await Http.CallAsync(HttpMethod.Post, url, Json.Encode(body), ContentTypeJson, _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + } + + /// + /// POST that gets-or-creates a named resource (POST {collection}?name=...). An optional + /// is sent as {"schema": ...}, matching the reference client. + /// + public async Task GetOrCreateNamedAsync(string? name, JsonNode? schema, CancellationToken ct) + { + var p = new QueryParams(); + if (!string.IsNullOrEmpty(name)) + { + p.AddString("name", name); + } + + var url = p.ApplyToUrl(SubUrl(string.Empty)); + using var response = schema is not null + ? await Http.CallAsync(HttpMethod.Post, url, Json.Encode(new JsonObject { ["schema"] = schema.DeepClone() }), ContentTypeJson, _requestTimeout, cancellationToken: ct).ConfigureAwait(false) + : await Http.CallAsync(HttpMethod.Post, url, timeout: _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + } + + /// POST with an optional raw body and content type, unwrapping the data envelope. + public async Task PostWithBodyAsync(string subPath, QueryParams p, string? body, string contentType, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + } + + /// + /// POST with a raw body, parsing the response directly without unwrapping a data envelope. + /// Used by endpoints (e.g. actor input validation) whose response is a plain object. + /// + public async Task PostWithBodyNoEnvelopeAsync(string subPath, QueryParams p, string? body, string contentType, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return Json.Decode(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + } + + /// DELETE with a JSON body (used for batch request deletion), unwrapping the data envelope. + public async Task DeleteWithBodyAsync(string subPath, QueryParams p, object? body, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Delete, url, Json.Encode(body), ContentTypeJson, _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + } + + /// + /// GET returning the raw response body (no data envelope). Returns null on not-found. + /// + public async Task GetRawAsync(string subPath, QueryParams p, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + try + { + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// HEAD request; returns whether the resource exists. + public async Task HeadExistsAsync(string subPath, QueryParams p, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + try + { + using var response = await Http.CallAsync(HttpMethod.Head, url, timeout: _requestTimeout, cancellationToken: ct).ConfigureAwait(false); + return true; + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return false; + } + } + + /// PUT with raw bytes and a content type, with an explicit per-request timeout and retry control. + public async Task PutRawAsync(string subPath, QueryParams p, byte[] body, string contentType, TimeSpan? timeout, bool doNotRetryTimeouts, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Put, url, null, contentType, timeout ?? _requestTimeout, doNotRetryTimeouts, bodyBytes: body, cancellationToken: ct).ConfigureAwait(false); + } + + // ---- Wait-for-finish ------------------------------------------------------ + + /// + /// The largest server-side waitForFinish value that is safe to send: below the configured + /// per-request timeout by a safety margin (or the API's 60s cap when no finite timeout is set). + /// + private int ServerWaitCapSecs() + { + var configured = (int)Http.RequestTimeoutSecs; + return configured > 0 ? Math.Max(0, configured - WaitTimeoutMarginSecs) : WaitRequestSecs; + } + + /// + /// Clamps a caller-supplied server-side waitForFinish value (seconds) to the server wait cap, + /// so a synchronous get/wait never asks the server to hold the connection longer than the client's own + /// per-request timeout. Returns null for a null input. + /// + public int? ClampServerWait(int? waitForFinishSecs) + { + if (waitForFinishSecs is null) + { + return null; + } + + return Math.Min(Math.Max(0, waitForFinishSecs.Value), ServerWaitCapSecs()); + } + + /// + /// Polls a GET endpoint with waitForFinish until the resource reaches a terminal state or the + /// wait budget elapses. == null means "wait indefinitely", + /// implemented as a finite but very large bound so the loop always terminates. A transient 404 (replica + /// lag) is treated as "not yet available". + /// + public async Task WaitForFinishAsync(int? waitSecs, string resourceName, Func isTerminal, CancellationToken ct) + { + var effectiveWaitSecs = waitSecs is not null + ? Math.Min(Math.Max(waitSecs.Value, 0), MaxWaitForFinishSecs) + : MaxWaitForFinishSecs; + var budgetMillis = (long)effectiveWaitSecs * 1000; + var stopwatch = Stopwatch.StartNew(); + var serverWaitCap = ServerWaitCapSecs(); + + JsonObject? resource = null; + + while (true) + { + var elapsed = stopwatch.ElapsedMilliseconds; + var remainingSecs = (int)((budgetMillis - elapsed) / 1000); + var requestSecs = Math.Min(Math.Min(Math.Max(remainingSecs, 0), WaitRequestSecs), serverWaitCap); + + var p = new QueryParams(); + p.AddInt("waitForFinish", requestSecs); + + var data = await GetResourceAsync(string.Empty, p, ct).ConfigureAwait(false); + if (data is JsonObject obj) + { + resource = obj; + if (isTerminal(obj)) + { + return obj; + } + } + + if (stopwatch.ElapsedMilliseconds >= budgetMillis) + { + break; + } + + await Task.Delay(TimeSpan.FromSeconds(WaitPollIntervalSecs), ct).ConfigureAwait(false); + } + + if (resource is not null) + { + return resource; + } + + throw new InvalidOperationException( + $"waiting for {resourceName} to finish failed: cannot fetch {resourceName} details from the server"); + } + + /// + /// Coerces a decoded value to a JSON object. Endpoints that return a resource object always decode to + /// an object; a non-object (e.g. an unexpected null data field) becomes an empty object so model + /// construction stays type-safe. + /// + private static JsonObject AsObject(JsonNode? value) => value as JsonObject ?? new JsonObject(); + + // ---- URL / id helpers ----------------------------------------------------- + + /// + /// Encodes a resource id so it is safe to embed in a URL path. Apify uses the + /// username~resourcename form, so the first / of an id is replaced with ~. + /// + public static string ToSafeId(string id) + { + var slash = id.IndexOf('/', StringComparison.Ordinal); + return slash < 0 ? id : string.Concat(id.AsSpan(0, slash), "~", id.AsSpan(slash + 1)); + } + + /// + /// Percent-encodes a single URL path segment, so values interpolated into the path (record keys, + /// request IDs) cannot break out of the segment. + /// + public static string EncodePathSegment(string input) => Uri.EscapeDataString(input); + + /// Extracts the origin (scheme://host[:port]) from a URL, dropping any path. + public static string OriginOf(string rawUrl) + { + var rest = rawUrl; + var scheme = string.Empty; + var pos = rest.IndexOf("://", StringComparison.Ordinal); + if (pos >= 0) + { + scheme = rest.Substring(0, pos + 3); + rest = rest.Substring(pos + 3); + } + + var slash = rest.IndexOf('/', StringComparison.Ordinal); + if (slash >= 0) + { + rest = rest.Substring(0, slash); + } + + return scheme + rest; + } +} diff --git a/src/Apify.Client/Internal/ResponseOwningStream.cs b/src/Apify.Client/Internal/ResponseOwningStream.cs new file mode 100644 index 0000000..b5b16a4 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseOwningStream.cs @@ -0,0 +1,72 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace Apify.Client.Internal; + +/// +/// A read-only stream over an HTTP response body that also owns the , +/// disposing it (and thus releasing the connection) when the stream is disposed. Used for live log +/// streaming, where the response must stay open while the caller reads the body incrementally. +/// +internal sealed class ResponseOwningStream : Stream +{ + private readonly HttpResponseMessage _response; + private readonly Stream _inner; + + private ResponseOwningStream(HttpResponseMessage response, Stream inner) + { + _response = response; + _inner = inner; + } + + /// Wraps a response's body stream, transferring ownership of the response to the returned stream. + public static async Task CreateAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + var inner = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + return new ResponseOwningStream(response, inner); + } + + public override bool CanRead => _inner.CanRead; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _inner.Length; + + public override long Position + { + get => _inner.Position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => _inner.ReadAsync(buffer, offset, count, cancellationToken); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => _inner.ReadAsync(buffer, cancellationToken); + + public override void Flush() => _inner.Flush(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + _response.Dispose(); + } + + base.Dispose(disposing); + } +} diff --git a/src/Apify.Client/Internal/RetryConfig.cs b/src/Apify.Client/Internal/RetryConfig.cs new file mode 100644 index 0000000..b1d9402 --- /dev/null +++ b/src/Apify.Client/Internal/RetryConfig.cs @@ -0,0 +1,32 @@ +namespace Apify.Client.Internal; + +/// +/// Retry/timeout policy for the orchestrating HTTP client. +/// +internal sealed class RetryConfig +{ + /// Creates a retry policy. + /// Maximum retries (the request is attempted up to maxRetries + 1 times). + /// Minimum delay between retries, in ms; doubled on each retry. + /// Upper bound on the (exponentially growing) inter-retry delay, in ms. + /// Overall per-request timeout budget, in seconds. + public RetryConfig(int maxRetries, double minDelayMillis, double maxDelayMillis, double timeoutSecs) + { + MaxRetries = maxRetries; + MinDelayMillis = minDelayMillis; + MaxDelayMillis = maxDelayMillis; + TimeoutSecs = timeoutSecs; + } + + /// Maximum number of retries (the request is attempted up to MaxRetries + 1 times). + public int MaxRetries { get; } + + /// Minimum delay between retries, in milliseconds; doubled on each retry (exponential backoff). + public double MinDelayMillis { get; } + + /// Upper bound on the (exponentially growing) inter-retry delay, in milliseconds. + public double MaxDelayMillis { get; } + + /// Overall per-request timeout budget, in seconds. Each attempt's timeout grows but is capped here. + public double TimeoutSecs { get; } +} diff --git a/src/Apify.Client/Internal/Signatures.cs b/src/Apify.Client/Internal/Signatures.cs new file mode 100644 index 0000000..38d8fb5 --- /dev/null +++ b/src/Apify.Client/Internal/Signatures.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace Apify.Client.Internal; + +/// +/// Apify storage-content URL signing, byte-for-byte compatible with the platform's +/// @apify/utilities implementation that the reference clients rely on. +/// +internal static class Signatures +{ + /// Version tag embedded in storage-content signatures (upstream default). + private const string StorageContentSignatureVersion = "0"; + + /// Number of leading hex characters of the HMAC digest used. + private const int HmacSignatureHexLen = 30; + + /// Base62 alphabet (digits, then lowercase, then uppercase), matching upstream. + private const string Base62Alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + private const int Base = 62; + private const int ByteBase = 256; + + /// + /// Computes an Apify URL-signing signature, byte-for-byte compatible with upstream + /// createHmacSignature: HMAC-SHA256(secret, message) as lowercase hex, take the first 30 hex + /// characters, interpret them as a big integer, then base62-encode (alphabet 0-9a-zA-Z). + /// + public static string CreateHmacSignature(string secretKey, string message) + { + var digest = HMACSHA256.HashData(Encoding.UTF8.GetBytes(secretKey), Encoding.UTF8.GetBytes(message)); + var hex = Convert.ToHexString(digest).ToLowerInvariant(); + var truncated = hex.Substring(0, HmacSignatureHexLen); + return HexToBase62(truncated); + } + + /// + /// Builds a storage-content signature for a resource's public URL, byte-for-byte compatible with + /// upstream createStorageContentSignature. + /// + /// + /// It signs the message "{version}.{expiresAtMillis}.{resourceId}" (expiresAtMillis is + /// the absolute expiry in ms, or 0 for a non-expiring URL) with , + /// then returns the base64url (no padding) encoding of "{version}.{expiresAtMillis}.{hmac}". + /// + /// The store/dataset URL-signing secret key. + /// The resource id being signed. + /// Optional expiry in seconds (null for a non-expiring URL). + public static string SignStorageContent(string secretKey, string resourceId, int? expiresInSecs) + { + var expiresAtMillis = expiresInSecs is not null + ? DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + (expiresInSecs.Value * 1000L) + : 0L; + var version = StorageContentSignatureVersion; + var expiryText = expiresAtMillis.ToString(CultureInfo.InvariantCulture); + var message = version + "." + expiryText + "." + resourceId; + var hmac = CreateHmacSignature(secretKey, message); + var envelope = version + "." + expiryText + "." + hmac; + return Base64UrlNoPadding(Encoding.UTF8.GetBytes(envelope)); + } + + /// + /// Interprets a hex string as a big-endian non-negative integer and encodes it in base62. + /// Implemented with byte-wise long division (base 256 → base 62) so it needs no bignum dependency. + /// + private static string HexToBase62(string hex) + { + var digits = new List(Convert.FromHexString(hex).Length); + foreach (var b in Convert.FromHexString(hex)) + { + digits.Add(b); + } + + if (digits.Count == 0) + { + return "0"; + } + + var result = new StringBuilder(); + while (digits.Count > 0) + { + var remainder = 0; + var quotient = new List(digits.Count); + foreach (var value in digits) + { + var accumulator = (remainder * ByteBase) + value; + var q = accumulator / Base; + remainder = accumulator % Base; + if (quotient.Count > 0 || q != 0) + { + quotient.Add(q); + } + } + + result.Insert(0, Base62Alphabet[remainder]); + digits = quotient; + } + + return result.Length == 0 ? "0" : result.ToString(); + } + + /// Encodes bytes as base64url without padding (+/ → -_, trailing = removed). + private static string Base64UrlNoPadding(byte[] bytes) + { + return Convert.ToBase64String(bytes) + .Replace('+', '-') + .Replace('/', '_') + .TrimEnd('='); + } +} diff --git a/src/Apify.Client/Internal/Statuses.cs b/src/Apify.Client/Internal/Statuses.cs new file mode 100644 index 0000000..160715f --- /dev/null +++ b/src/Apify.Client/Internal/Statuses.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; + +namespace Apify.Client.Internal; + +/// +/// Run/build status helpers. +/// +internal static class Statuses +{ + /// Terminal run/build statuses: a resource in any of these is finished and will not change. + private static readonly HashSet Terminal = new(StringComparer.Ordinal) + { + "SUCCEEDED", + "FAILED", + "ABORTED", + "TIMED-OUT", + }; + + /// Reports whether the status is a terminal (finished) run/build status. + public static bool IsTerminal(string? status) => status is not null && Terminal.Contains(status); +} diff --git a/src/Apify.Client/Models/Actor.cs b/src/Apify.Client/Models/Actor.cs new file mode 100644 index 0000000..1172088 --- /dev/null +++ b/src/Apify.Client/Models/Actor.cs @@ -0,0 +1,41 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// An Actor on the Apify platform. +public sealed class Actor : ApifyResource +{ + /// Wraps a raw Actor object. + /// The raw decoded resource object. + public Actor(JsonObject data) + : base(data) + { + } + + /// The unique Actor ID. + public string? Id => GetString("id"); + + /// The ID of the user who owns the Actor. + public string? UserId => GetString("userId"); + + /// The technical name of the Actor (used in API paths). + public string? Name => GetString("name"); + + /// The username of the Actor's owner. + public string? Username => GetString("username"); + + /// The human-readable title shown in the UI. + public string? Title => GetString("title"); + + /// A description of what the Actor does. + public string? Description => GetString("description"); + + /// Whether the Actor is publicly available in Apify Store. + public bool? IsPublic => GetBool("isPublic"); + + /// When the Actor was created (ISO-8601 string). + public string? CreatedAt => GetString("createdAt"); + + /// When the Actor was last modified (ISO-8601 string). + public string? ModifiedAt => GetString("modifiedAt"); +} diff --git a/src/Apify.Client/Models/ActorEnvVar.cs b/src/Apify.Client/Models/ActorEnvVar.cs new file mode 100644 index 0000000..ae22924 --- /dev/null +++ b/src/Apify.Client/Models/ActorEnvVar.cs @@ -0,0 +1,86 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// An environment variable attached to an Actor version. +public sealed class ActorEnvVar : ApifyResource +{ + private ActorEnvVar(JsonObject data) + : base(data) + { + } + + /// Creates an environment variable. + /// The environment variable name. + /// The environment variable value. + /// Whether the value is stored as a secret. + public ActorEnvVar(string? name = null, string? value = null, bool? isSecret = null) + : this(new JsonObject()) + { + if (name is not null) + { + Name = name; + } + + if (value is not null) + { + Value = value; + } + + if (isSecret is not null) + { + IsSecret = isSecret; + } + } + + /// Wraps a raw env-var object (used when hydrating from the API). + /// The raw decoded env-var object. + public static ActorEnvVar FromJsonObject(JsonObject data) => new(data); + + /// The environment variable name. + public string? Name + { + get => GetString("name"); + set => SetOrRemove("name", value); + } + + /// The environment variable value. + public string? Value + { + get => GetString("value"); + set => SetOrRemove("value", value); + } + + /// Whether the value is stored as a secret. + public bool? IsSecret + { + get => GetBool("isSecret"); + set => SetOrRemove("isSecret", value); + } + + // Honor the documented "null fields are omitted" contract: a null assignment removes the key rather + // than writing a JSON null node. + private void SetOrRemove(string key, string? value) + { + if (value is null) + { + ToJsonObject().Remove(key); + } + else + { + ToJsonObject()[key] = value; + } + } + + private void SetOrRemove(string key, bool? value) + { + if (value is null) + { + ToJsonObject().Remove(key); + } + else + { + ToJsonObject()[key] = value.Value; + } + } +} diff --git a/src/Apify.Client/Models/ActorRun.cs b/src/Apify.Client/Models/ActorRun.cs new file mode 100644 index 0000000..07b7fe6 --- /dev/null +++ b/src/Apify.Client/Models/ActorRun.cs @@ -0,0 +1,60 @@ +using System.Text.Json.Nodes; +using Apify.Client.Internal; + +namespace Apify.Client.Models; + +/// A single execution of an Actor. +public sealed class ActorRun : ApifyResource +{ + /// Wraps a raw run object. + /// The raw decoded resource object. + public ActorRun(JsonObject data) + : base(data) + { + } + + /// The unique run ID. + public string? Id => GetString("id"); + + /// The ID of the Actor that produced this run. + public string? ActId => GetString("actId"); + + /// The ID of the task that started this run, if any. + public string? ActorTaskId => GetString("actorTaskId"); + + /// The ID of the user who owns the run. + public string? UserId => GetString("userId"); + + /// + /// The current run status. One of the eight ActorJobStatus values: READY, RUNNING, + /// SUCCEEDED, FAILED, TIMING-OUT, TIMED-OUT, ABORTING, ABORTED. + /// + public string? Status => GetString("status"); + + /// An optional human-readable status message. + public string? StatusMessage => GetString("statusMessage"); + + /// When the run started (ISO-8601 string). + public string? StartedAt => GetString("startedAt"); + + /// When the run finished (absent while still running). + public string? FinishedAt => GetString("finishedAt"); + + /// The ID of the build used for the run. + public string? BuildId => GetString("buildId"); + + /// The ID of the run's default dataset. + public string? DefaultDatasetId => GetString("defaultDatasetId"); + + /// The ID of the run's default key-value store. + public string? DefaultKeyValueStoreId => GetString("defaultKeyValueStoreId"); + + /// The ID of the run's default request queue. + public string? DefaultRequestQueueId => GetString("defaultRequestQueueId"); + + /// The URL of the run's container (for live access). + public string? ContainerUrl => GetString("containerUrl"); + + /// Whether the run has reached a terminal (finished) status. + public bool IsTerminal => Statuses.IsTerminal(Status); +} diff --git a/src/Apify.Client/Models/ActorStoreListItem.cs b/src/Apify.Client/Models/ActorStoreListItem.cs new file mode 100644 index 0000000..dde8e52 --- /dev/null +++ b/src/Apify.Client/Models/ActorStoreListItem.cs @@ -0,0 +1,26 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// An Actor as listed in the Apify Store. +public sealed class ActorStoreListItem : ApifyResource +{ + /// Wraps a raw store-list item. + /// The raw decoded resource object. + public ActorStoreListItem(JsonObject data) + : base(data) + { + } + + /// The unique Actor ID. + public string? Id => GetString("id"); + + /// The technical name of the Actor. + public string? Name => GetString("name"); + + /// The username of the Actor's owner. + public string? Username => GetString("username"); + + /// The human-readable title. + public string? Title => GetString("title"); +} diff --git a/src/Apify.Client/Models/ActorTask.cs b/src/Apify.Client/Models/ActorTask.cs new file mode 100644 index 0000000..abed185 --- /dev/null +++ b/src/Apify.Client/Models/ActorTask.cs @@ -0,0 +1,41 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// +/// A pre-configured Actor run (an Actor task). +/// +/// +/// Named ActorTask rather than Task to avoid colliding with +/// ; it corresponds to the reference client's task resource. +/// +public sealed class ActorTask : ApifyResource +{ + /// Wraps a raw task object. + /// The raw decoded resource object. + public ActorTask(JsonObject data) + : base(data) + { + } + + /// The unique task ID. + public string? Id => GetString("id"); + + /// The ID of the Actor this task runs. + public string? ActId => GetString("actId"); + + /// The ID of the user who owns the task. + public string? UserId => GetString("userId"); + + /// The technical name of the task. + public string? Name => GetString("name"); + + /// The human-readable title shown in the UI. + public string? Title => GetString("title"); + + /// When the task was created (ISO-8601 string). + public string? CreatedAt => GetString("createdAt"); + + /// When the task was last modified (ISO-8601 string). + public string? ModifiedAt => GetString("modifiedAt"); +} diff --git a/src/Apify.Client/Models/ActorVersion.cs b/src/Apify.Client/Models/ActorVersion.cs new file mode 100644 index 0000000..b9c9732 --- /dev/null +++ b/src/Apify.Client/Models/ActorVersion.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A single version of an Actor. +public sealed class ActorVersion : ApifyResource +{ + /// Wraps a raw version object. + /// The raw decoded resource object. + public ActorVersion(JsonObject data) + : base(data) + { + } + + /// The version identifier (e.g. 0.1). + public string? VersionNumber => GetString("versionNumber"); + + /// How the version's source is provided (e.g. SOURCE_FILES). + public string? SourceType => GetString("sourceType"); +} diff --git a/src/Apify.Client/Models/ApifyResource.cs b/src/Apify.Client/Models/ApifyResource.cs new file mode 100644 index 0000000..93659fe --- /dev/null +++ b/src/Apify.Client/Models/ApifyResource.cs @@ -0,0 +1,115 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// +/// Base class for API resource models. +/// +/// +/// Each model wraps the raw decoded JSON object and exposes commonly-used fields as typed properties. +/// The full payload — including any field the API adds that is not modelled here — is always available +/// via and , so additive API changes never lose data. +/// +public abstract class ApifyResource +{ + private readonly JsonObject _data; + + /// Wraps the raw decoded resource object. + /// The raw decoded resource object. + protected ApifyResource(JsonObject data) + { + _data = data; + } + + /// The full raw resource object, including fields not mapped to a typed property. + public JsonObject ToJsonObject() => _data; + + /// A single raw field by key (null if absent). + /// The field name. + public JsonNode? Get(string key) => _data.TryGetPropertyValue(key, out var value) ? value : null; + + /// Reads a string field, coercing numbers to their text form; null if absent or unsupported. + protected string? GetString(string key) + { + var node = Get(key); + return node?.GetValueKind() switch + { + JsonValueKind.String => node.GetValue(), + JsonValueKind.Number => node.ToJsonString(), + _ => null, + }; + } + + /// Reads an integer field, coercing numeric strings and fractional numbers; null if absent. + protected long? GetInt(string key) + { + var node = Get(key); + if (node is null) + { + return null; + } + + var text = node.GetValueKind() switch + { + JsonValueKind.Number => node.ToJsonString(), + JsonValueKind.String => node.GetValue(), + _ => null, + }; + if (text is null) + { + return null; + } + + if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var longValue)) + { + return longValue; + } + + if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var doubleValue)) + { + return (long)doubleValue; + } + + return null; + } + + /// Reads a boolean field; null if absent or not a JSON boolean. + protected bool? GetBool(string key) + { + return Get(key)?.GetValueKind() switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => null, + }; + } + + /// Reads a string-array field (numbers coerced to text); null if absent or not an array. + protected IReadOnlyList? GetStringList(string key) + { + if (Get(key) is not JsonArray array) + { + return null; + } + + var result = new List(array.Count); + foreach (var item in array) + { + var text = item?.GetValueKind() switch + { + JsonValueKind.String => item.GetValue(), + JsonValueKind.Number => item.ToJsonString(), + _ => null, + }; + if (text is not null) + { + result.Add(text); + } + } + + return result; + } +} diff --git a/src/Apify.Client/Models/BatchAddResult.cs b/src/Apify.Client/Models/BatchAddResult.cs new file mode 100644 index 0000000..aadb043 --- /dev/null +++ b/src/Apify.Client/Models/BatchAddResult.cs @@ -0,0 +1,42 @@ +using System.Collections.Generic; + +namespace Apify.Client.Models; + +/// +/// The result of a batch request-add: the accepted (processed) and the unprocessed requests. +/// +public sealed class BatchAddResult +{ + private List _processed; + private List _unprocessed; + + /// Creates a result with the given processed and unprocessed requests. + /// The requests the API successfully added. + /// The requests the API did not process. + public BatchAddResult( + IEnumerable? processedRequests = null, + IEnumerable? unprocessedRequests = null) + { + _processed = processedRequests is null ? new List() : new List(processedRequests); + _unprocessed = unprocessedRequests is null ? new List() : new List(unprocessedRequests); + } + + /// The requests the API successfully added. + public IReadOnlyList ProcessedRequests => _processed; + + /// The requests the API did not process. + public IReadOnlyList UnprocessedRequests => _unprocessed; + + /// Replaces the processed requests. + internal void SetProcessedRequests(List processedRequests) => _processed = processedRequests; + + /// Replaces the unprocessed requests. + internal void SetUnprocessedRequests(List unprocessedRequests) => _unprocessed = unprocessedRequests; + + /// Appends another result's requests into this one (used to merge per-chunk batch results). + internal void Merge(BatchAddResult other) + { + _processed.AddRange(other._processed); + _unprocessed.AddRange(other._unprocessed); + } +} diff --git a/src/Apify.Client/Models/Build.cs b/src/Apify.Client/Models/Build.cs new file mode 100644 index 0000000..8b9301f --- /dev/null +++ b/src/Apify.Client/Models/Build.cs @@ -0,0 +1,39 @@ +using System.Text.Json.Nodes; +using Apify.Client.Internal; + +namespace Apify.Client.Models; + +/// A single build of an Actor. +public sealed class Build : ApifyResource +{ + /// Wraps a raw build object. + /// The raw decoded resource object. + public Build(JsonObject data) + : base(data) + { + } + + /// The unique build ID. + public string? Id => GetString("id"); + + /// The ID of the Actor this build belongs to. + public string? ActId => GetString("actId"); + + /// + /// The current build status. One of the eight ActorJobStatus values: READY, RUNNING, + /// SUCCEEDED, FAILED, TIMING-OUT, TIMED-OUT, ABORTING, ABORTED. + /// + public string? Status => GetString("status"); + + /// When the build started (ISO-8601 string). + public string? StartedAt => GetString("startedAt"); + + /// When the build finished (absent while still building). + public string? FinishedAt => GetString("finishedAt"); + + /// The human-readable build number (e.g. 0.1.2). + public string? BuildNumber => GetString("buildNumber"); + + /// Whether the build has reached a terminal (finished) status. + public bool IsTerminal => Statuses.IsTerminal(Status); +} diff --git a/src/Apify.Client/Models/Dataset.cs b/src/Apify.Client/Models/Dataset.cs new file mode 100644 index 0000000..c51aec1 --- /dev/null +++ b/src/Apify.Client/Models/Dataset.cs @@ -0,0 +1,32 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A dataset stores structured results from Actor runs. +public sealed class Dataset : ApifyResource +{ + /// Wraps a raw dataset object. + /// The raw decoded resource object. + public Dataset(JsonObject data) + : base(data) + { + } + + /// The unique dataset ID. + public string? Id => GetString("id"); + + /// The dataset name (empty for unnamed datasets). + public string? Name => GetString("name"); + + /// The ID of the user who owns the dataset. + public string? UserId => GetString("userId"); + + /// When the dataset was created (ISO-8601 string). + public string? CreatedAt => GetString("createdAt"); + + /// When the dataset was last modified (ISO-8601 string). + public string? ModifiedAt => GetString("modifiedAt"); + + /// The number of items currently stored. + public long? ItemCount => GetInt("itemCount"); +} diff --git a/src/Apify.Client/Models/KeyValueStore.cs b/src/Apify.Client/Models/KeyValueStore.cs new file mode 100644 index 0000000..14ab72d --- /dev/null +++ b/src/Apify.Client/Models/KeyValueStore.cs @@ -0,0 +1,29 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A key-value store holds arbitrary data records. +public sealed class KeyValueStore : ApifyResource +{ + /// Wraps a raw key-value store object. + /// The raw decoded resource object. + public KeyValueStore(JsonObject data) + : base(data) + { + } + + /// The unique store ID. + public string? Id => GetString("id"); + + /// The store name (empty for unnamed stores). + public string? Name => GetString("name"); + + /// The ID of the user who owns the store. + public string? UserId => GetString("userId"); + + /// When the store was created (ISO-8601 string). + public string? CreatedAt => GetString("createdAt"); + + /// When the store was last modified (ISO-8601 string). + public string? ModifiedAt => GetString("modifiedAt"); +} diff --git a/src/Apify.Client/Models/KeyValueStoreKey.cs b/src/Apify.Client/Models/KeyValueStoreKey.cs new file mode 100644 index 0000000..1e54ed4 --- /dev/null +++ b/src/Apify.Client/Models/KeyValueStoreKey.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A single key listed from a key-value store. +public sealed class KeyValueStoreKey : ApifyResource +{ + /// Wraps a raw key object. + /// The raw decoded resource object. + public KeyValueStoreKey(JsonObject data) + : base(data) + { + } + + /// The record key. + public string? Key => GetString("key"); + + /// The record size in bytes. + public long? Size => GetInt("size"); +} diff --git a/src/Apify.Client/Models/KeyValueStoreKeysPage.cs b/src/Apify.Client/Models/KeyValueStoreKeysPage.cs new file mode 100644 index 0000000..6c5ab2c --- /dev/null +++ b/src/Apify.Client/Models/KeyValueStoreKeysPage.cs @@ -0,0 +1,57 @@ +using System.Collections.Generic; +using System.Text.Json.Nodes; +using Apify.Client.Internal; + +namespace Apify.Client.Models; + +/// A page of keys from a key-value store. +public sealed class KeyValueStoreKeysPage +{ + private KeyValueStoreKeysPage( + IReadOnlyList items, + long limit, + bool isTruncated, + string? exclusiveStartKey, + string? nextExclusiveStartKey) + { + Items = items; + Limit = limit; + IsTruncated = isTruncated; + ExclusiveStartKey = exclusiveStartKey; + NextExclusiveStartKey = nextExclusiveStartKey; + } + + /// Builds a page from the decoded keys-page object. + /// The decoded keys-page object. + public static KeyValueStoreKeysPage FromData(JsonNode? data) + { + var obj = JsonValues.AsObject(data); + var items = new List(); + foreach (var item in JsonValues.ObjectItems(obj)) + { + items.Add(new KeyValueStoreKey(item)); + } + + return new KeyValueStoreKeysPage( + items, + JsonValues.IntOr(obj, "limit", items.Count), + JsonValues.BoolOr(obj, "isTruncated", false), + JsonValues.String(obj, "exclusiveStartKey"), + JsonValues.String(obj, "nextExclusiveStartKey")); + } + + /// The listed keys. + public IReadOnlyList Items { get; } + + /// The maximum number of keys requested. + public long Limit { get; } + + /// Whether more keys are available. + public bool IsTruncated { get; } + + /// The key the listing started after. + public string? ExclusiveStartKey { get; } + + /// The key to pass to fetch the next page. + public string? NextExclusiveStartKey { get; } +} diff --git a/src/Apify.Client/Models/KeyValueStoreRecord.cs b/src/Apify.Client/Models/KeyValueStoreRecord.cs new file mode 100644 index 0000000..8738880 --- /dev/null +++ b/src/Apify.Client/Models/KeyValueStoreRecord.cs @@ -0,0 +1,37 @@ +namespace Apify.Client.Models; + +/// +/// A single record retrieved from a key-value store. +/// +/// +/// holds the record's raw bytes exactly as stored, so binary records (images, +/// gzip, protobuf, XLSX, …) survive a round-trip intact. Decode it according to : +/// for text use System.Text.Encoding.UTF8.GetString(record.Value), and for JSON +/// (e.g. records written with ) +/// deserialize the bytes with System.Text.Json.JsonSerializer.Deserialize<T>(record.Value). +/// +public sealed class KeyValueStoreRecord +{ + /// Creates a record. + /// The record key. + /// The raw record bytes. + /// The record's MIME type, as reported by the API. + public KeyValueStoreRecord(string key, byte[] value, string? contentType) + { + Key = key; + Value = value; + ContentType = contentType; + } + + /// The record key. + public string Key { get; } + + /// + /// The raw record bytes, exactly as stored. Decode according to (see the + /// class remarks for text/JSON decoding). + /// + public byte[] Value { get; } + + /// The record's MIME type, as reported by the API. + public string? ContentType { get; } +} diff --git a/src/Apify.Client/Models/PaginationList.cs b/src/Apify.Client/Models/PaginationList.cs new file mode 100644 index 0000000..7c79a1c --- /dev/null +++ b/src/Apify.Client/Models/PaginationList.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text.Json.Nodes; +using Apify.Client.Internal; + +namespace Apify.Client.Models; + +/// +/// A single page of an offset/limit-paginated list. +/// +/// +/// The pagination metadata (, , , +/// , ) accompanies the . is +/// always the number of items in this page (so this[i] is valid for +/// 0 <= i < Count); the API's total across all pages is exposed separately as +/// . Note: reflects the API's reported total, which can briefly lag +/// immediately after a write (the count is computed asynchronously) — re-read after a short delay if you +/// need an exact post-write total. +/// +/// The hydrated item type. +public sealed class PaginationList : IReadOnlyList +{ + private readonly IReadOnlyList _items; + + private PaginationList(IReadOnlyList items, long total, long offset, long limit, bool desc) + { + _items = items; + Total = total; + Offset = offset; + Limit = limit; + Desc = desc; + } + + /// Builds a page from a decoded paginated object, hydrating each item. + /// The decoded paginated object. + /// Maps each raw item to a model. + internal static PaginationList FromData(JsonNode? data, Func hydrate) + { + var obj = JsonValues.AsObject(data); + var rawItems = JsonValues.ObjectItems(obj); + var items = new List(rawItems.Count); + foreach (var raw in rawItems) + { + items.Add(hydrate(raw)); + } + + return new PaginationList( + items, + JsonValues.IntOr(obj, "total", items.Count), + JsonValues.IntOr(obj, "offset", 0), + JsonValues.IntOr(obj, "limit", items.Count), + JsonValues.BoolOr(obj, "desc", false)); + } + + /// + /// Builds a page directly from items and metadata (used by the dataset-items endpoint, which returns + /// a bare array with pagination in response headers). + /// + /// The page items. + /// Total number of items available across all pages. + /// Number of items skipped at the start. + /// Maximum number of items the API would return. + /// Whether the items are in descending order. + internal static PaginationList FromItems(IReadOnlyList items, long total, long offset, long limit, bool desc) + => new(items, total, offset, limit, desc); + + /// The items of this page (never null). + public IReadOnlyList Items => _items; + + /// Total number of items available across all pages. + public long Total { get; } + + /// Number of items skipped at the start. + public long Offset { get; } + + /// Maximum number of items the API would return for this request. + public long Limit { get; } + + /// + /// Number of items in this page (always equal to Items.Count). Use for the + /// count across all pages. + /// + public long Count => _items.Count; + + /// Whether the items are in descending order. + public bool Desc { get; } + + /// + int IReadOnlyCollection.Count => _items.Count; + + /// + public T this[int index] => _items[index]; + + /// + public IEnumerator GetEnumerator() => _items.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/src/Apify.Client/Models/RequestQueue.cs b/src/Apify.Client/Models/RequestQueue.cs new file mode 100644 index 0000000..662a9c4 --- /dev/null +++ b/src/Apify.Client/Models/RequestQueue.cs @@ -0,0 +1,32 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A request queue stores URLs to be crawled. +public sealed class RequestQueue : ApifyResource +{ + /// Wraps a raw request queue object. + /// The raw decoded resource object. + public RequestQueue(JsonObject data) + : base(data) + { + } + + /// The unique queue ID. + public string? Id => GetString("id"); + + /// The queue name (empty for unnamed queues). + public string? Name => GetString("name"); + + /// The ID of the user who owns the queue. + public string? UserId => GetString("userId"); + + /// When the queue was created (ISO-8601 string). + public string? CreatedAt => GetString("createdAt"); + + /// When the queue was last modified (ISO-8601 string). + public string? ModifiedAt => GetString("modifiedAt"); + + /// The total number of requests ever added. + public long? TotalRequestCount => GetInt("totalRequestCount"); +} diff --git a/src/Apify.Client/Models/RequestQueueHead.cs b/src/Apify.Client/Models/RequestQueueHead.cs new file mode 100644 index 0000000..6f3ea4d --- /dev/null +++ b/src/Apify.Client/Models/RequestQueueHead.cs @@ -0,0 +1,42 @@ +using System.Collections.Generic; +using System.Text.Json.Nodes; +using Apify.Client.Internal; + +namespace Apify.Client.Models; + +/// The head (front) of a request queue. +public sealed class RequestQueueHead +{ + private RequestQueueHead(IReadOnlyList items, long limit, bool hadMultipleClients) + { + Items = items; + Limit = limit; + HadMultipleClients = hadMultipleClients; + } + + /// Builds a head from the decoded queue-head object. + /// The decoded queue-head object. + public static RequestQueueHead FromData(JsonNode? data) + { + var obj = JsonValues.AsObject(data); + var items = new List(); + foreach (var item in JsonValues.ObjectItems(obj)) + { + items.Add(RequestQueueRequest.FromJsonObject(item)); + } + + return new RequestQueueHead( + items, + JsonValues.IntOr(obj, "limit", items.Count), + JsonValues.BoolOr(obj, "hadMultipleClients", false)); + } + + /// The requests at the head of the queue. + public IReadOnlyList Items { get; } + + /// The maximum number of requests requested. + public long Limit { get; } + + /// Whether multiple clients have accessed the queue. + public bool HadMultipleClients { get; } +} diff --git a/src/Apify.Client/Models/RequestQueueOperationInfo.cs b/src/Apify.Client/Models/RequestQueueOperationInfo.cs new file mode 100644 index 0000000..9f2270a --- /dev/null +++ b/src/Apify.Client/Models/RequestQueueOperationInfo.cs @@ -0,0 +1,29 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// Returned when adding or updating a request in a queue. +public sealed class RequestQueueOperationInfo : ApifyResource +{ + /// Wraps a raw operation-info object. + /// The raw decoded resource object. + public RequestQueueOperationInfo(JsonObject data) + : base(data) + { + } + + /// The ID of the affected request. + public string? RequestId => GetString("requestId"); + + /// + /// The unique key of the affected request. Populated for batch-add results; may be null for + /// single add/update operations. + /// + public string? UniqueKey => GetString("uniqueKey"); + + /// Whether the request was already in the queue. + public bool? WasAlreadyPresent => GetBool("wasAlreadyPresent"); + + /// Whether the request had already been handled. + public bool? WasAlreadyHandled => GetBool("wasAlreadyHandled"); +} diff --git a/src/Apify.Client/Models/RequestQueueRequest.cs b/src/Apify.Client/Models/RequestQueueRequest.cs new file mode 100644 index 0000000..d4e12a3 --- /dev/null +++ b/src/Apify.Client/Models/RequestQueueRequest.cs @@ -0,0 +1,93 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// +/// A single request stored in a request queue. Fields left null are omitted when the request is +/// sent to the API. Construct one for adding to a queue, or receive one when reading a queue. +/// +public sealed class RequestQueueRequest : ApifyResource +{ + private RequestQueueRequest(JsonObject data) + : base(data) + { + } + + /// Creates a request, optionally with a URL and unique (deduplication) key. + /// The request URL. + /// The deduplication key for the request. + public RequestQueueRequest(string? url = null, string? uniqueKey = null) + : this(new JsonObject()) + { + if (url is not null) + { + Url = url; + } + + if (uniqueKey is not null) + { + UniqueKey = uniqueKey; + } + } + + /// Wraps a raw request object (used when hydrating from the API). + /// The raw decoded request object. + public static RequestQueueRequest FromJsonObject(JsonObject data) => new(data); + + /// The unique request ID (assigned by the API; absent on create). + public string? Id + { + get => GetString("id"); + set => SetString("id", value); + } + + /// The request URL. + public string? Url + { + get => GetString("url"); + set => SetString("url", value); + } + + /// The deduplication key for the request. + public string? UniqueKey + { + get => GetString("uniqueKey"); + set => SetString("uniqueKey", value); + } + + /// The HTTP method (e.g. GET, POST). + public string? Method + { + get => GetString("method"); + set => SetString("method", value); + } + + /// Arbitrary user-attached metadata. + public JsonNode? UserData + { + get => Get("userData"); + set + { + if (value is null) + { + ToJsonObject().Remove("userData"); + } + else + { + ToJsonObject()["userData"] = value.DeepClone(); + } + } + } + + private void SetString(string key, string? value) + { + if (value is null) + { + ToJsonObject().Remove(key); + } + else + { + ToJsonObject()[key] = value; + } + } +} diff --git a/src/Apify.Client/Models/Schedule.cs b/src/Apify.Client/Models/Schedule.cs new file mode 100644 index 0000000..14d4676 --- /dev/null +++ b/src/Apify.Client/Models/Schedule.cs @@ -0,0 +1,29 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A schedule automatically starts Actor or task runs at specified times. +public sealed class Schedule : ApifyResource +{ + /// Wraps a raw schedule object. + /// The raw decoded resource object. + public Schedule(JsonObject data) + : base(data) + { + } + + /// The unique schedule ID. + public string? Id => GetString("id"); + + /// The ID of the user who owns the schedule. + public string? UserId => GetString("userId"); + + /// The schedule name. + public string? Name => GetString("name"); + + /// The cron expression governing when the schedule fires. + public string? CronExpression => GetString("cronExpression"); + + /// Whether the schedule is currently active. + public bool? IsEnabled => GetBool("isEnabled"); +} diff --git a/src/Apify.Client/Models/User.cs b/src/Apify.Client/Models/User.cs new file mode 100644 index 0000000..af21248 --- /dev/null +++ b/src/Apify.Client/Models/User.cs @@ -0,0 +1,23 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// +/// An Apify user account. Private account details for me (email, plan, proxy settings, …) are +/// available via . +/// +public sealed class User : ApifyResource +{ + /// Wraps a raw user object. + /// The raw decoded resource object. + public User(JsonObject data) + : base(data) + { + } + + /// The unique user ID. + public string? Id => GetString("id"); + + /// The user's username. + public string? Username => GetString("username"); +} diff --git a/src/Apify.Client/Models/Webhook.cs b/src/Apify.Client/Models/Webhook.cs new file mode 100644 index 0000000..05f0d46 --- /dev/null +++ b/src/Apify.Client/Models/Webhook.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A webhook notifies an external service when specific events occur. +public sealed class Webhook : ApifyResource +{ + /// Wraps a raw webhook object. + /// The raw decoded resource object. + public Webhook(JsonObject data) + : base(data) + { + } + + /// The unique webhook ID. + public string? Id => GetString("id"); + + /// The ID of the user who owns the webhook. + public string? UserId => GetString("userId"); + + /// The URL the webhook posts to. + public string? RequestUrl => GetString("requestUrl"); + + /// The events that trigger the webhook. + public IReadOnlyList? EventTypes => GetStringList("eventTypes"); +} diff --git a/src/Apify.Client/Models/WebhookDispatch.cs b/src/Apify.Client/Models/WebhookDispatch.cs new file mode 100644 index 0000000..6175bb4 --- /dev/null +++ b/src/Apify.Client/Models/WebhookDispatch.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Nodes; + +namespace Apify.Client.Models; + +/// A single invocation of a webhook. +public sealed class WebhookDispatch : ApifyResource +{ + /// Wraps a raw webhook dispatch object. + /// The raw decoded resource object. + public WebhookDispatch(JsonObject data) + : base(data) + { + } + + /// The unique dispatch ID. + public string? Id => GetString("id"); + + /// The ID of the webhook that produced this dispatch. + public string? WebhookId => GetString("webhookId"); +} diff --git a/src/Apify.Client/Options/ActorBuildOptions.cs b/src/Apify.Client/Options/ActorBuildOptions.cs new file mode 100644 index 0000000..69e43a9 --- /dev/null +++ b/src/Apify.Client/Options/ActorBuildOptions.cs @@ -0,0 +1,27 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures building an Actor version. +public sealed class ActorBuildOptions +{ + /// If true, use beta versions of Apify packages. + public bool? BetaPackages { get; init; } + + /// The tag to apply to the build (e.g. latest). + public string? Tag { get; init; } + + /// Whether to use the Docker build cache (default true). + public bool? UseCache { get; init; } + + /// Maximum seconds to wait server-side for the build (max 60). + public int? WaitForFinish { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddBool("betaPackages", BetaPackages) + .AddString("tag", Tag) + .AddBool("useCache", UseCache) + .AddInt("waitForFinish", WaitForFinish); + } +} diff --git a/src/Apify.Client/Options/ActorListOptions.cs b/src/Apify.Client/Options/ActorListOptions.cs new file mode 100644 index 0000000..3acf7a8 --- /dev/null +++ b/src/Apify.Client/Options/ActorListOptions.cs @@ -0,0 +1,31 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Options for listing the account's Actors. +public sealed class ActorListOptions +{ + /// Number of Actors to skip. + public int? Offset { get; init; } + + /// Maximum number of Actors to return. + public int? Limit { get; init; } + + /// If true, return Actors newest-first. + public bool? Desc { get; init; } + + /// If true, return only Actors owned by the current user. + public bool? My { get; init; } + + /// The sort field (e.g. createdAt, stats.lastRunStartedAt). + public string? SortBy { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddInt("offset", Offset) + .AddInt("limit", Limit) + .AddBool("desc", Desc) + .AddBool("my", My) + .AddString("sortBy", SortBy); + } +} diff --git a/src/Apify.Client/Options/ActorStartOptions.cs b/src/Apify.Client/Options/ActorStartOptions.cs new file mode 100644 index 0000000..3665dbe --- /dev/null +++ b/src/Apify.Client/Options/ActorStartOptions.cs @@ -0,0 +1,71 @@ +using System; +using System.Text; +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures starting an Actor run. All fields are optional. +public sealed class ActorStartOptions +{ + /// The tag or number of the build to run (e.g. latest, 0.1.2). + public string? Build { get; init; } + + /// Memory in megabytes allocated for the run. + public int? MemoryMbytes { get; init; } + + /// Timeout for the run in seconds (0 means no timeout). + public int? TimeoutSecs { get; init; } + + /// Maximum seconds to wait server-side for the run to finish (max 60). + public int? WaitForFinish { get; init; } + + /// Maximum number of dataset items to charge (pay-per-result Actors). + public int? MaxItems { get; init; } + + /// Maximum total charge in USD (pay-per-event Actors). + public double? MaxTotalChargeUsd { get; init; } + + /// The content type of the input body. Defaults to application/json. + public string? ContentType { get; init; } + + /// If true, restart the run if it fails. + public bool? RestartOnError { get; init; } + + /// + /// Override the Actor's permission level for this run (LIMITED_PERMISSIONS/FULL_PERMISSIONS). + /// + public string? ForcePermissionLevel { get; init; } + + /// + /// Ad-hoc webhooks to attach to this run; a JSON-serializable list serialized to base64-encoded JSON + /// as the webhooks query parameter. + /// + public object? Webhooks { get; init; } + + /// The configured content type, or the JSON default when unset. + internal string ContentTypeOrDefault() => + string.IsNullOrEmpty(ContentType) ? ResourceContext.ContentTypeJson : ContentType; + + internal void AppendTo(QueryParams q) + { + q.AddString("build", Build) + .AddInt("memory", MemoryMbytes) + .AddInt("timeout", TimeoutSecs) + .AddInt("waitForFinish", WaitForFinish) + .AddInt("maxItems", MaxItems) + .AddDouble("maxTotalChargeUsd", MaxTotalChargeUsd) + .AddBool("restartOnError", RestartOnError) + .AddString("forcePermissionLevel", ForcePermissionLevel) + .AddString("webhooks", EncodeWebhooks(Webhooks)); + } + + /// + /// Encodes an ad-hoc webhooks list as base64-encoded JSON, as the API's webhooks query + /// parameter requires. Returns null for a null list. Shared by Actor and task start + /// options. + /// + internal static string? EncodeWebhooks(object? webhooks) + { + return webhooks is null ? null : Convert.ToBase64String(Encoding.UTF8.GetBytes(Json.Encode(webhooks))); + } +} diff --git a/src/Apify.Client/Options/BatchAddRequestsOptions.cs b/src/Apify.Client/Options/BatchAddRequestsOptions.cs new file mode 100644 index 0000000..3c67ec0 --- /dev/null +++ b/src/Apify.Client/Options/BatchAddRequestsOptions.cs @@ -0,0 +1,42 @@ +using System; + +namespace Apify.Client.Options; + +/// +/// Tuning options for batch request adding, mirroring the reference client. Requests the API reports as +/// unprocessed (typically due to rate limiting) are automatically retried with exponential backoff. +/// +public sealed class BatchAddRequestsOptions +{ + /// Default number of retry rounds for unprocessed requests (matches the reference client). + public const int DefaultMaxUnprocessedRetries = 3; + + /// Default maximum number of batch API calls made in parallel (matches the reference client). + public const int DefaultMaxParallel = 5; + + /// Default minimum delay before retrying unprocessed requests (matches the reference client). + public const int DefaultMinDelayMillis = 500; + + /// Creates batch-add options, clamping values to their valid ranges. + /// Number of retry rounds for unprocessed requests. + /// Maximum number of batch API calls made in parallel. + /// Minimum delay before retrying unprocessed requests. + public BatchAddRequestsOptions( + int maxUnprocessedRequestsRetries = DefaultMaxUnprocessedRetries, + int maxParallel = DefaultMaxParallel, + int minDelayBetweenUnprocessedRequestsRetriesMillis = DefaultMinDelayMillis) + { + MaxUnprocessedRequestsRetries = Math.Max(0, maxUnprocessedRequestsRetries); + MaxParallel = Math.Max(1, maxParallel); + MinDelayBetweenUnprocessedRequestsRetriesMillis = Math.Max(0, minDelayBetweenUnprocessedRequestsRetriesMillis); + } + + /// Number of retry rounds for requests the API reports as unprocessed. + public int MaxUnprocessedRequestsRetries { get; } + + /// Maximum number of batch API calls made in parallel. + public int MaxParallel { get; } + + /// Minimum delay before retrying unprocessed requests, in milliseconds. + public int MinDelayBetweenUnprocessedRequestsRetriesMillis { get; } +} diff --git a/src/Apify.Client/Options/DatasetDownloadOptions.cs b/src/Apify.Client/Options/DatasetDownloadOptions.cs new file mode 100644 index 0000000..eab665f --- /dev/null +++ b/src/Apify.Client/Options/DatasetDownloadOptions.cs @@ -0,0 +1,50 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// Adds format-specific options for downloading dataset items on top of the shared item +/// filtering/projection options (). +/// +public sealed class DatasetDownloadOptions +{ + /// The shared filtering/projection options. + public DatasetListItemsOptions? Items { get; init; } + + /// Set Content-Disposition: attachment on the response. + public bool? Attachment { get; init; } + + /// Prepend a UTF-8 BOM (useful for Excel-compatible CSV). + public bool? Bom { get; init; } + + /// The CSV field delimiter (default ,). + public string? Delimiter { get; init; } + + /// Omit the CSV header row. + public bool? SkipHeaderRow { get; init; } + + /// The name of the root XML element (default items). + public string? XmlRoot { get; init; } + + /// The name of the per-item XML element (default item). + public string? XmlRow { get; init; } + + /// The title used for RSS/Atom feed exports. + public string? FeedTitle { get; init; } + + /// The description used for RSS/Atom feed exports. + public string? FeedDescription { get; init; } + + internal void AppendTo(QueryParams q) + { + Items?.AppendTo(q); + q.AddBool("attachment", Attachment) + .AddBool("bom", Bom) + .AddString("delimiter", Delimiter) + .AddBool("skipHeaderRow", SkipHeaderRow) + .AddString("xmlRoot", XmlRoot) + .AddString("xmlRow", XmlRow) + .AddString("feedTitle", FeedTitle) + .AddString("feedDescription", FeedDescription); + } +} diff --git a/src/Apify.Client/Options/DatasetListItemsOptions.cs b/src/Apify.Client/Options/DatasetListItemsOptions.cs new file mode 100644 index 0000000..c8f4edf --- /dev/null +++ b/src/Apify.Client/Options/DatasetListItemsOptions.cs @@ -0,0 +1,75 @@ +using System.Collections.Generic; +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// Configures listing or downloading dataset items (GET /v2/datasets/{datasetId}/items). All +/// fields are optional. +/// +public sealed class DatasetListItemsOptions +{ + /// Number of items to skip. + public int? Offset { get; init; } + + /// Maximum number of items to return. + public int? Limit { get; init; } + + /// Return items newest-first. + public bool? Desc { get; init; } + + /// Restrict the output to these fields. + public IReadOnlyList? Fields { get; init; } + + /// Positionally rename the selected (requires ). + public IReadOnlyList? OutputFields { get; init; } + + /// Exclude these fields from the output. + public IReadOnlyList? Omit { get; init; } + + /// Skip empty items. + public bool? SkipEmpty { get; init; } + + /// Skip hidden fields (those starting with #). + public bool? SkipHidden { get; init; } + + /// Return only clean (non-empty, non-hidden) items. + public bool? Clean { get; init; } + + /// Expand these fields (each array element becomes a separate item). + public IReadOnlyList? Unwind { get; init; } + + /// Flatten these nested fields into dot-notation keys. + public IReadOnlyList? Flatten { get; init; } + + /// Select a predefined dataset view for field selection. + public string? View { get; init; } + + /// Return simplified (flattened, cleaned) items. + public bool? Simplified { get; init; } + + /// Skip items that come from failed pages. + public bool? SkipFailedPages { get; init; } + + /// A pre-shared URL signature granting access without an API token. + public string? Signature { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddInt("offset", Offset) + .AddInt("limit", Limit) + .AddBool("desc", Desc) + .AddCsv("fields", Fields) + .AddCsv("outputFields", OutputFields) + .AddCsv("omit", Omit) + .AddBool("skipEmpty", SkipEmpty) + .AddBool("skipHidden", SkipHidden) + .AddBool("clean", Clean) + .AddCsv("unwind", Unwind) + .AddCsv("flatten", Flatten) + .AddString("view", View) + .AddBool("simplified", Simplified) + .AddBool("skipFailedPages", SkipFailedPages) + .AddString("signature", Signature); + } +} diff --git a/src/Apify.Client/Options/DownloadItemsFormat.cs b/src/Apify.Client/Options/DownloadItemsFormat.cs new file mode 100644 index 0000000..8668b97 --- /dev/null +++ b/src/Apify.Client/Options/DownloadItemsFormat.cs @@ -0,0 +1,45 @@ +using System; + +namespace Apify.Client.Options; + +/// An output format for downloading dataset items. +public enum DownloadItemsFormat +{ + /// JSON array. + Json, + + /// Newline-delimited JSON. + Jsonl, + + /// Comma-separated values. + Csv, + + /// Microsoft Excel (XLSX) workbook. + Xlsx, + + /// XML. + Xml, + + /// RSS feed. + Rss, + + /// HTML table. + Html, +} + +/// Maps values to their API wire representation. +internal static class DownloadItemsFormatExtensions +{ + /// The lowercase wire value the API expects for the format query parameter. + public static string ToWireValue(this DownloadItemsFormat format) => format switch + { + DownloadItemsFormat.Json => "json", + DownloadItemsFormat.Jsonl => "jsonl", + DownloadItemsFormat.Csv => "csv", + DownloadItemsFormat.Xlsx => "xlsx", + DownloadItemsFormat.Xml => "xml", + DownloadItemsFormat.Rss => "rss", + DownloadItemsFormat.Html => "html", + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "unknown download format"), + }; +} diff --git a/src/Apify.Client/Options/GetRecordOptions.cs b/src/Apify.Client/Options/GetRecordOptions.cs new file mode 100644 index 0000000..449999e --- /dev/null +++ b/src/Apify.Client/Options/GetRecordOptions.cs @@ -0,0 +1,18 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures fetching a key-value-store record. +public sealed class GetRecordOptions +{ + /// Controls the Content-Disposition: attachment behaviour. + public bool? Attachment { get; init; } + + /// A pre-shared URL signature granting access without an API token. + public string? Signature { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddBool("attachment", Attachment).AddString("signature", Signature); + } +} diff --git a/src/Apify.Client/Options/LastRunOptions.cs b/src/Apify.Client/Options/LastRunOptions.cs new file mode 100644 index 0000000..cfb5a61 --- /dev/null +++ b/src/Apify.Client/Options/LastRunOptions.cs @@ -0,0 +1,19 @@ +namespace Apify.Client.Options; + +/// +/// Filters which "last" run the last-run accessors resolve to. Leave a field null to leave that +/// filter unset. +/// +/// +/// Origin is an Apify-platform convenience exposed by the reference client but not documented as a +/// query parameter in the OpenAPI spec; it is included for parity, threaded to the same runs/last +/// endpoint. +/// +public sealed class LastRunOptions +{ + /// Filter by run status (e.g. SUCCEEDED, FAILED, RUNNING). + public string? Status { get; init; } + + /// Filter by how the run was started (e.g. DEVELOPMENT, WEB, API). + public string? Origin { get; init; } +} diff --git a/src/Apify.Client/Options/ListKeysOptions.cs b/src/Apify.Client/Options/ListKeysOptions.cs new file mode 100644 index 0000000..72c0d66 --- /dev/null +++ b/src/Apify.Client/Options/ListKeysOptions.cs @@ -0,0 +1,31 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures listing keys in a key-value store. +public sealed class ListKeysOptions +{ + /// Maximum number of keys to return. + public int? Limit { get; init; } + + /// List keys after this one (for pagination). + public string? ExclusiveStartKey { get; init; } + + /// Restrict the listing to keys with this prefix. + public string? Prefix { get; init; } + + /// Restrict the listing to a named collection of keys. + public string? Collection { get; init; } + + /// A pre-shared URL signature granting access without an API token. + public string? Signature { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddInt("limit", Limit) + .AddString("exclusiveStartKey", ExclusiveStartKey) + .AddString("prefix", Prefix) + .AddString("collection", Collection) + .AddString("signature", Signature); + } +} diff --git a/src/Apify.Client/Options/ListOptions.cs b/src/Apify.Client/Options/ListOptions.cs new file mode 100644 index 0000000..e028c4d --- /dev/null +++ b/src/Apify.Client/Options/ListOptions.cs @@ -0,0 +1,25 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// The standard offset/limit pagination shared by most list endpoints (builds, runs, tasks, +/// schedules, webhooks, Actor versions). All fields are optional; leave one null to use the API +/// default. +/// +public sealed class ListOptions +{ + /// Number of items to skip from the beginning of the list. + public int? Offset { get; init; } + + /// Maximum number of items to return. + public int? Limit { get; init; } + + /// If true, return items newest-first. + public bool? Desc { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddInt("offset", Offset).AddInt("limit", Limit).AddBool("desc", Desc); + } +} diff --git a/src/Apify.Client/Options/ListRequestsOptions.cs b/src/Apify.Client/Options/ListRequestsOptions.cs new file mode 100644 index 0000000..144f5ef --- /dev/null +++ b/src/Apify.Client/Options/ListRequestsOptions.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures listing a request queue's requests. +public sealed class ListRequestsOptions +{ + /// Filter value: currently locked requests. + public const string FilterLocked = "locked"; + + /// Filter value: pending (not-yet-handled) requests. + public const string FilterPending = "pending"; + + /// Maximum number of requests to return. + public int? Limit { get; init; } + + /// List requests after this ID. + public string? ExclusiveStartId { get; init; } + + /// An opaque pagination cursor (alternative to ). + public string? Cursor { get; init; } + + /// + /// Restrict the listing to requests in the given states; each value must be + /// or . + /// + public IReadOnlyList? Filter { get; init; } + + /// Validates the options for API-level constraints. + internal void Validate() + { + if (ExclusiveStartId is not null && Cursor is not null) + { + throw new ArgumentException("ListRequestsOptions: ExclusiveStartId and Cursor are mutually exclusive"); + } + + if (Filter is not null) + { + foreach (var f in Filter) + { + if (f != FilterLocked && f != FilterPending) + { + throw new ArgumentException(string.Format( + CultureInfo.InvariantCulture, + "ListRequestsOptions: filter entries must be \"{0}\" or \"{1}\", got \"{2}\"", + FilterLocked, + FilterPending, + f)); + } + } + } + } + + internal void AppendTo(QueryParams q) + { + q.AddInt("limit", Limit) + .AddString("exclusiveStartId", ExclusiveStartId) + .AddString("cursor", Cursor) + .AddCsv("filter", Filter); + } +} diff --git a/src/Apify.Client/Options/LogOptions.cs b/src/Apify.Client/Options/LogOptions.cs new file mode 100644 index 0000000..6aac3bc --- /dev/null +++ b/src/Apify.Client/Options/LogOptions.cs @@ -0,0 +1,18 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures log retrieval/streaming. +public sealed class LogOptions +{ + /// If true, return the unprocessed log content (no platform post-processing). + public bool? Raw { get; init; } + + /// If true, set Content-Disposition so the log is served as a download. + public bool? Download { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddBool("raw", Raw).AddBool("download", Download); + } +} diff --git a/src/Apify.Client/Options/MetamorphOptions.cs b/src/Apify.Client/Options/MetamorphOptions.cs new file mode 100644 index 0000000..ff2c354 --- /dev/null +++ b/src/Apify.Client/Options/MetamorphOptions.cs @@ -0,0 +1,17 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures a run metamorph. +public sealed class MetamorphOptions +{ + /// Optionally pins the target Actor's build (unset for default). + public string? Build { get; init; } + + /// The content type of the input body. Defaults to application/json. + public string? ContentType { get; init; } + + /// The configured content type, or the JSON default when unset. + internal string ContentTypeOrDefault() => + string.IsNullOrEmpty(ContentType) ? ResourceContext.ContentTypeJson : ContentType; +} diff --git a/src/Apify.Client/Options/PaginateRequestsOptions.cs b/src/Apify.Client/Options/PaginateRequestsOptions.cs new file mode 100644 index 0000000..6db4d7f --- /dev/null +++ b/src/Apify.Client/Options/PaginateRequestsOptions.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace Apify.Client.Options; + +/// +/// Configures lazy iteration over a request queue's requests +/// (), mirroring the +/// reference client's paginateRequests({ limit, maxPageLimit, exclusiveStartId, cursor, filter }). +/// +public sealed class PaginateRequestsOptions +{ + /// Default maximum number of requests fetched per page (matches the reference client). + public const int DefaultMaxPageLimit = 1000; + + /// Filter value: currently locked requests. + public const string FilterLocked = "locked"; + + /// Filter value: pending (not-yet-handled) requests. + public const string FilterPending = "pending"; + + /// Maximum total number of requests to iterate across all pages (null for no bound). + public int? Limit { get; init; } + + /// Maximum number of requests fetched per page (defaults to ). + public int? MaxPageLimit { get; init; } + + /// Start iterating after this request ID (first page only; mutually exclusive with cursor). + public string? ExclusiveStartId { get; init; } + + /// An opaque pagination cursor to start from (mutually exclusive with ). + public string? Cursor { get; init; } + + /// + /// Restrict the iteration to requests in the given states; each value must be + /// or . + /// + public IReadOnlyList? Filter { get; init; } + + /// Validates the options for API-level constraints. + internal void Validate() + { + if (ExclusiveStartId is not null && Cursor is not null) + { + throw new ArgumentException("PaginateRequestsOptions: ExclusiveStartId and Cursor are mutually exclusive"); + } + + if (Filter is not null) + { + foreach (var f in Filter) + { + if (f != FilterLocked && f != FilterPending) + { + throw new ArgumentException(string.Format( + CultureInfo.InvariantCulture, + "PaginateRequestsOptions: filter entries must be \"{0}\" or \"{1}\", got \"{2}\"", + FilterLocked, + FilterPending, + f)); + } + } + } + } +} diff --git a/src/Apify.Client/Options/RequestQueueClientOptions.cs b/src/Apify.Client/Options/RequestQueueClientOptions.cs new file mode 100644 index 0000000..16f1a51 --- /dev/null +++ b/src/Apify.Client/Options/RequestQueueClientOptions.cs @@ -0,0 +1,21 @@ +namespace Apify.Client.Options; + +/// +/// Per-client options for a , mirroring the +/// reference client's requestQueue(id, { clientKey, timeoutSecs }). +/// +public sealed class RequestQueueClientOptions +{ + /// + /// A stable client key identifying this client to the queue. Required to operate on locks the client + /// itself created, and lets the API detect whether multiple clients access the queue. + /// + public string? ClientKey { get; init; } + + /// + /// Per-request timeout (seconds) for this queue client's calls. It shortens the wait for each call and + /// is capped at the client-wide overall timeout, so a value larger than that timeout has no effect. When + /// null the shared client-wide timeout is used. + /// + public double? TimeoutSecs { get; init; } +} diff --git a/src/Apify.Client/Options/RunChargeOptions.cs b/src/Apify.Client/Options/RunChargeOptions.cs new file mode 100644 index 0000000..3c6d829 --- /dev/null +++ b/src/Apify.Client/Options/RunChargeOptions.cs @@ -0,0 +1,31 @@ +namespace Apify.Client.Options; + +/// Configures charging for a pay-per-event Actor run. +public sealed class RunChargeOptions +{ + /// Creates charge options. + /// The name of the event to charge for. Required. + /// The number of times to charge the event (defaults to 1). + /// + /// A key that deduplicates the charge across retries. If unset, one is auto-generated as + /// "{runId}-{eventName}-{timestampMillis}-{random}", matching the reference client. + /// + public RunChargeOptions(string eventName, int? count = null, string? idempotencyKey = null) + { + EventName = eventName; + Count = count; + IdempotencyKey = idempotencyKey; + } + + /// The name of the event to charge for. + public string EventName { get; } + + /// The number of times to charge the event (defaults to 1). + public int? Count { get; } + + /// A key that deduplicates the charge across retries. + public string? IdempotencyKey { get; } + + /// The count to send, defaulting to 1. + internal int CountValue() => Count ?? 1; +} diff --git a/src/Apify.Client/Options/RunListOptions.cs b/src/Apify.Client/Options/RunListOptions.cs new file mode 100644 index 0000000..75c9b6a --- /dev/null +++ b/src/Apify.Client/Options/RunListOptions.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// Run-specific filters for listing runs. The StartedAfter/StartedBefore filters are only +/// honoured by the Actor-scoped and task-scoped run collections. +/// +public sealed class RunListOptions +{ + /// + /// Filter by one or more run statuses (e.g. SUCCEEDED, RUNNING); sent as a + /// comma-separated list. + /// + public IReadOnlyList? Status { get; init; } + + /// Filter to runs started after this ISO-8601 timestamp. + public string? StartedAfter { get; init; } + + /// Filter to runs started before this ISO-8601 timestamp. + public string? StartedBefore { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddCsv("status", Status) + .AddString("startedAfter", StartedAfter) + .AddString("startedBefore", StartedBefore); + } +} diff --git a/src/Apify.Client/Options/RunResurrectOptions.cs b/src/Apify.Client/Options/RunResurrectOptions.cs new file mode 100644 index 0000000..1b2699d --- /dev/null +++ b/src/Apify.Client/Options/RunResurrectOptions.cs @@ -0,0 +1,35 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures resurrecting a finished run. +public sealed class RunResurrectOptions +{ + /// The tag or number of the build to resurrect with. + public string? Build { get; init; } + + /// Memory in megabytes to allocate. + public int? MemoryMbytes { get; init; } + + /// The run timeout in seconds. + public int? TimeoutSecs { get; init; } + + /// Maximum number of dataset items to charge (pay-per-result Actors). + public int? MaxItems { get; init; } + + /// Maximum total charge in USD (pay-per-event Actors). + public double? MaxTotalChargeUsd { get; init; } + + /// If true, restart the run if it fails. + public bool? RestartOnError { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddString("build", Build) + .AddInt("memory", MemoryMbytes) + .AddInt("timeout", TimeoutSecs) + .AddInt("maxItems", MaxItems) + .AddDouble("maxTotalChargeUsd", MaxTotalChargeUsd) + .AddBool("restartOnError", RestartOnError); + } +} diff --git a/src/Apify.Client/Options/SetRecordOptions.cs b/src/Apify.Client/Options/SetRecordOptions.cs new file mode 100644 index 0000000..55892bc --- /dev/null +++ b/src/Apify.Client/Options/SetRecordOptions.cs @@ -0,0 +1,18 @@ +namespace Apify.Client.Options; + +/// +/// Write options for storing a key-value-store record, mirroring the reference client's +/// timeoutSecs/doNotRetryTimeouts. +/// +public sealed class SetRecordOptions +{ + /// + /// Per-request timeout for the upload, in seconds. Use it to shorten the wait for this upload; defaults + /// to (and is capped at) the client's configured overall request timeout, so a value larger than that + /// timeout has no effect. + /// + public int? TimeoutSecs { get; init; } + + /// If true, do not retry the upload when it fails with a request timeout. + public bool DoNotRetryTimeouts { get; init; } +} diff --git a/src/Apify.Client/Options/StorageListOptions.cs b/src/Apify.Client/Options/StorageListOptions.cs new file mode 100644 index 0000000..caf3043 --- /dev/null +++ b/src/Apify.Client/Options/StorageListOptions.cs @@ -0,0 +1,35 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// Options for the storage collection list endpoints (GET /v2/datasets, +/// /v2/key-value-stores, /v2/request-queues), which add unnamed and ownership +/// filters on top of the standard pagination. +/// +public sealed class StorageListOptions +{ + /// Number of items to skip from the beginning of the list. + public int? Offset { get; init; } + + /// Maximum number of items to return. + public int? Limit { get; init; } + + /// If true, return items newest-first. + public bool? Desc { get; init; } + + /// If true, include unnamed storages in the result. + public bool? Unnamed { get; init; } + + /// Filter by ownership (e.g. OWNED / ACCESSIBLE). + public string? Ownership { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddInt("offset", Offset) + .AddInt("limit", Limit) + .AddBool("desc", Desc) + .AddBool("unnamed", Unnamed) + .AddString("ownership", Ownership); + } +} diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs new file mode 100644 index 0000000..5b8070b --- /dev/null +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -0,0 +1,69 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Options for listing/iterating the Apify Store (GET /v2/store). +public sealed class StoreListOptions +{ + /// Number of Actors to skip. + public int? Offset { get; init; } + + /// Maximum number of Actors to return (also the per-page size when iterating). + public int? Limit { get; init; } + + /// Full-text search query. + public string? Search { get; init; } + + /// The sort field (e.g. popularity, newest). + public string? SortBy { get; init; } + + /// Filter Actors by category. + public string? Category { get; init; } + + /// Filter Actors by owner username. + public string? Username { get; init; } + + /// + /// Filter Actors by pricing model (FREE, FLAT_PRICE_PER_MONTH, + /// PRICE_PER_DATASET_ITEM, PAY_PER_EVENT). + /// + public string? PricingModel { get; init; } + + /// Include Actors the current user cannot run. + public bool? IncludeUnrunnableActors { get; init; } + + /// Filter to Actors that allow agentic users. + public bool? AllowsAgenticUsers { get; init; } + + /// The response format (full, agent). + public string? ResponseFormat { get; init; } + + /// Returns a copy of these options with a new (used by lazy iteration). + internal StoreListOptions WithOffset(int? offset) => new() + { + Offset = offset, + Limit = Limit, + Search = Search, + SortBy = SortBy, + Category = Category, + Username = Username, + PricingModel = PricingModel, + IncludeUnrunnableActors = IncludeUnrunnableActors, + AllowsAgenticUsers = AllowsAgenticUsers, + ResponseFormat = ResponseFormat, + }; + + internal void AppendTo(QueryParams q) + { + q.AddInt("offset", Offset) + .AddInt("limit", Limit) + .AddString("search", Search) + .AddString("sortBy", SortBy) + .AddString("category", Category) + .AddString("username", Username) + .AddString("pricingModel", PricingModel) + .AddBool("includeUnrunnableActors", IncludeUnrunnableActors) + .AddBool("allowsAgenticUsers", AllowsAgenticUsers) + .AddString("responseFormat", ResponseFormat); + } +} diff --git a/src/Apify.Client/Options/TaskStartOptions.cs b/src/Apify.Client/Options/TaskStartOptions.cs new file mode 100644 index 0000000..0a6fd03 --- /dev/null +++ b/src/Apify.Client/Options/TaskStartOptions.cs @@ -0,0 +1,49 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// +/// Configures starting a task run. +/// +/// +/// It mirrors but omits the fields the task run endpoint does not accept +/// (the Actor-only ContentType and ForcePermissionLevel), matching the reference client. +/// +public sealed class TaskStartOptions +{ + /// The tag or number of the build to run (e.g. latest, 0.1.2). + public string? Build { get; init; } + + /// Memory in megabytes allocated for the run. + public int? MemoryMbytes { get; init; } + + /// Timeout for the run in seconds (0 means no timeout). + public int? TimeoutSecs { get; init; } + + /// Maximum seconds to wait server-side for the run to finish (max 60). + public int? WaitForFinish { get; init; } + + /// Maximum number of dataset items to charge (pay-per-result Actors). + public int? MaxItems { get; init; } + + /// Maximum total charge in USD (pay-per-event Actors). + public double? MaxTotalChargeUsd { get; init; } + + /// If true, restart the run if it fails. + public bool? RestartOnError { get; init; } + + /// Ad-hoc webhooks to attach (a JSON-serializable list serialized to base64-encoded JSON). + public object? Webhooks { get; init; } + + internal void AppendTo(QueryParams q) + { + q.AddString("build", Build) + .AddInt("memory", MemoryMbytes) + .AddInt("timeout", TimeoutSecs) + .AddInt("waitForFinish", WaitForFinish) + .AddInt("maxItems", MaxItems) + .AddDouble("maxTotalChargeUsd", MaxTotalChargeUsd) + .AddBool("restartOnError", RestartOnError) + .AddString("webhooks", ActorStartOptions.EncodeWebhooks(Webhooks)); + } +} diff --git a/src/Apify.Client/Options/ValidateInputOptions.cs b/src/Apify.Client/Options/ValidateInputOptions.cs new file mode 100644 index 0000000..6a40e91 --- /dev/null +++ b/src/Apify.Client/Options/ValidateInputOptions.cs @@ -0,0 +1,22 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Options; + +/// Configures Actor input validation. All fields are optional. +public sealed class ValidateInputOptions +{ + /// The tag or number of the build whose input schema is used for validation. + public string? Build { get; init; } + + /// The content type of the input body. Defaults to application/json. + public string? ContentType { get; init; } + + /// The configured content type, or the JSON default when unset. + internal string ContentTypeOrDefault() => + string.IsNullOrEmpty(ContentType) ? ResourceContext.ContentTypeJson : ContentType; + + internal void AppendTo(QueryParams q) + { + q.AddString("build", Build); + } +} diff --git a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs new file mode 100644 index 0000000..7617f62 --- /dev/null +++ b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs @@ -0,0 +1,33 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// Shared read-only behavior for webhook collections. Both the account-wide collection +/// () and the read-only collections nested under an Actor or task +/// () can list webhooks; only the account-wide collection can +/// create them. +/// +public abstract class AbstractWebhookCollectionClient +{ + private protected readonly ResourceContext Ctx; + + private protected AbstractWebhookCollectionClient(HttpClientCore http, string baseUrl) + { + Ctx = ResourceContext.Collection(http, baseUrl, "webhooks"); + } + + /// Lists webhooks. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return Ctx.ListResourceAsync("", q, static d => new Webhook(d), cancellationToken); + } +} diff --git a/src/Apify.Client/Resources/ActorClient.cs b/src/Apify.Client/Resources/ActorClient.cs new file mode 100644 index 0000000..9374f30 --- /dev/null +++ b/src/Apify.Client/Resources/ActorClient.cs @@ -0,0 +1,152 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a specific Actor. +/// +/// +/// It provides CRUD methods plus convenience helpers to start/call the Actor, build it, and access its +/// runs, builds, versions and webhooks. +/// +public sealed class ActorClient +{ + private readonly ApifyClient _root; + private readonly HttpClientCore _http; + private readonly string _baseUrl; + private readonly ResourceContext _ctx; + + internal ActorClient(ApifyClient root, HttpClientCore http, string baseUrl, string id) + { + _root = root; + _http = http; + _baseUrl = baseUrl; + Id = id; + _ctx = ResourceContext.Single(http, baseUrl, "actors", id); + } + + /// The Actor's ID (or username~name) as provided. + public string Id { get; } + + /// Fetches the Actor object, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new Actor(obj) : null; + } + + /// Updates the Actor with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new Actor(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the Actor. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// Starts the Actor and returns immediately with the created run. + /// Any JSON-serializable value (or null for no input). + /// Optional run-start options. + /// A token to cancel the request. + public async Task StartAsync(object? input = null, ActorStartOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ActorStartOptions(); + var q = new QueryParams(); + options.AppendTo(q); + var body = input is null ? null : Json.Encode(input); + return new ActorRun(await _ctx.PostWithBodyAsync("runs", q, body, options.ContentTypeOrDefault(), cancellationToken).ConfigureAwait(false)); + } + + /// Starts the Actor and waits (client-side polling) for it to finish. + /// Any JSON-serializable value (or null for no input). + /// Optional run-start options. + /// Bounds the wait; null waits indefinitely. + /// A token to cancel the request. + public async Task CallAsync( + object? input = null, + ActorStartOptions? options = null, + int? waitSecs = null, + CancellationToken cancellationToken = default) + { + var run = await StartAsync(input, options, cancellationToken).ConfigureAwait(false); + return await _root.Run(run.Id ?? string.Empty).WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + } + + /// Validates against the Actor's input schema and returns whether it is valid. + /// Any JSON-serializable value (or null). + /// Optional validation options. + /// A token to cancel the request. + public async Task ValidateInputAsync(object? input = null, ValidateInputOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ValidateInputOptions(); + var q = new QueryParams(); + options.AppendTo(q); + var body = input is null ? null : Json.Encode(input); + // The validate-input endpoint returns a bare {"valid": } object, not the standard + // {"data": ...} envelope, so parse it without unwrapping. + var result = await _ctx.PostWithBodyNoEnvelopeAsync("validate-input", q, body, options.ContentTypeOrDefault(), cancellationToken).ConfigureAwait(false); + return result is JsonObject obj && obj.TryGetPropertyValue("valid", out var valid) + && valid?.GetValueKind() == System.Text.Json.JsonValueKind.True; + } + + /// Builds the given version of the Actor and returns the created build. + /// The version to build (e.g. 0.0). + /// Optional build options. + /// A token to cancel the request. + public async Task BuildAsync(string versionNumber, ActorBuildOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddString("version", versionNumber); + (options ?? new ActorBuildOptions()).AppendTo(q); + return new Build(await _ctx.PostWithBodyAsync("builds", q, null, ResourceContext.ContentTypeJson, cancellationToken).ConfigureAwait(false)); + } + + /// + /// Resolves the Actor's default build and returns a client for it. + /// optionally bounds how long (seconds) the API waits for the build to finish before responding. + /// + /// Optional server-side wait in seconds. + /// A token to cancel the request. + public async Task DefaultBuildAsync(int? waitForFinish = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddInt("waitForFinish", waitForFinish); + var data = await _ctx.GetResourceRequiredAsync("builds/default", q, cancellationToken).ConfigureAwait(false); + var build = new Build(data as JsonObject ?? new JsonObject()); + return new BuildClient(_http, _baseUrl, build.Id ?? string.Empty); + } + + /// Returns a client for the last run of this Actor, optionally filtered by status and/or origin. + /// Optional last-run filters. + public RunClient LastRun(LastRunOptions? options = null) + { + var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + client.SetLastRunParams(options ?? new LastRunOptions()); + return client; + } + + /// A client for this Actor's build collection. + public BuildCollectionClient Builds() => new(_http, _ctx.SubUrl(""), "builds"); + + /// A client for this Actor's run collection. + public RunCollectionClient Runs() => new(_http, _ctx.SubUrl(""), "runs"); + + /// A client for a specific version of this Actor. + /// The version identifier (e.g. 0.1). + public ActorVersionClient Version(string versionNumber) => new(_http, _ctx.SubUrl(""), versionNumber); + + /// A client for this Actor's version collection. + public ActorVersionCollectionClient Versions() => new(_http, _ctx.SubUrl("")); + + /// A read-only client for this Actor's webhook collection (GET /v2/actors/{id}/webhooks). + public NestedWebhookCollectionClient Webhooks() => new(_http, _ctx.SubUrl("")); +} diff --git a/src/Apify.Client/Resources/ActorCollectionClient.cs b/src/Apify.Client/Resources/ActorCollectionClient.cs new file mode 100644 index 0000000..a7b8293 --- /dev/null +++ b/src/Apify.Client/Resources/ActorCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the Actor collection (GET/POST /v2/actors). +public sealed class ActorCollectionClient +{ + private readonly ResourceContext _ctx; + + internal ActorCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "actors"); + } + + /// Lists the account's Actors. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public Task> ListAsync(ActorListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ActorListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new Actor(d), cancellationToken); + } + + /// Creates a new Actor. + /// Any JSON-serializable Actor definition. + /// A token to cancel the request. + public async Task CreateAsync(object actor, CancellationToken cancellationToken = default) + { + return new Actor(await _ctx.CreateResourceAsync(new QueryParams(), actor, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/ActorEnvVarClient.cs b/src/Apify.Client/Resources/ActorEnvVarClient.cs new file mode 100644 index 0000000..f046066 --- /dev/null +++ b/src/Apify.Client/Resources/ActorEnvVarClient.cs @@ -0,0 +1,42 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// +/// A client for a single environment variable +/// (GET/PUT/DELETE /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{name}). +/// +public sealed class ActorEnvVarClient +{ + private readonly ResourceContext _ctx; + + internal ActorEnvVarClient(HttpClientCore http, string versionUrl, string name) + { + _ctx = ResourceContext.Single(http, versionUrl, "env-vars", name); + } + + /// Fetches the environment variable, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? ActorEnvVar.FromJsonObject(obj) : null; + } + + /// Updates the environment variable and returns the updated object. + /// The new environment variable state. + /// A token to cancel the request. + public async Task UpdateAsync(ActorEnvVar envVar, CancellationToken cancellationToken = default) + { + return ActorEnvVar.FromJsonObject( + await _ctx.UpdateResourceAsync("", envVar.ToJsonObject(), cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the environment variable. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); +} diff --git a/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs b/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs new file mode 100644 index 0000000..34fcef9 --- /dev/null +++ b/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// +/// A client for an Actor version's environment variable collection +/// (GET/POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars). +/// +public sealed class ActorEnvVarCollectionClient +{ + private readonly ResourceContext _ctx; + + internal ActorEnvVarCollectionClient(HttpClientCore http, string versionUrl) + { + _ctx = ResourceContext.Collection(http, versionUrl, "env-vars"); + } + + /// Lists the version's environment variables. + /// A token to cancel the request. + public Task> ListAsync(CancellationToken cancellationToken = default) + { + return _ctx.ListResourceAsync("", new QueryParams(), static d => ActorEnvVar.FromJsonObject(d), cancellationToken); + } + + /// Creates a new environment variable. + /// The environment variable to create. + /// A token to cancel the request. + public async Task CreateAsync(ActorEnvVar envVar, CancellationToken cancellationToken = default) + { + return ActorEnvVar.FromJsonObject( + await _ctx.CreateResourceAsync(new QueryParams(), envVar.ToJsonObject(), cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/ActorVersionClient.cs b/src/Apify.Client/Resources/ActorVersionClient.cs new file mode 100644 index 0000000..fed6eba --- /dev/null +++ b/src/Apify.Client/Resources/ActorVersionClient.cs @@ -0,0 +1,52 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// +/// A client for a specific Actor version +/// (GET/PUT/DELETE /v2/actors/{actorId}/versions/{versionNumber}). +/// +public sealed class ActorVersionClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + private readonly string _versionUrl; + + internal ActorVersionClient(HttpClientCore http, string actorUrl, string versionNumber) + { + _http = http; + _ctx = ResourceContext.Single(http, actorUrl, "versions", versionNumber); + _versionUrl = _ctx.SubUrl(""); + } + + /// Fetches the version, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new ActorVersion(obj) : null; + } + + /// Updates the version with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new ActorVersion(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the version. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// A client for a specific environment variable of this version. + /// The environment variable name. + public ActorEnvVarClient EnvVar(string name) => new(_http, _versionUrl, name); + + /// A client for this version's environment variable collection. + public ActorEnvVarCollectionClient EnvVars() => new(_http, _versionUrl); +} diff --git a/src/Apify.Client/Resources/ActorVersionCollectionClient.cs b/src/Apify.Client/Resources/ActorVersionCollectionClient.cs new file mode 100644 index 0000000..771b156 --- /dev/null +++ b/src/Apify.Client/Resources/ActorVersionCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for an Actor's version collection (GET/POST /v2/actors/{actorId}/versions). +public sealed class ActorVersionCollectionClient +{ + private readonly ResourceContext _ctx; + + internal ActorVersionCollectionClient(HttpClientCore http, string actorUrl) + { + _ctx = ResourceContext.Collection(http, actorUrl, "versions"); + } + + /// Lists the Actor's versions. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new ActorVersion(d), cancellationToken); + } + + /// Creates a new Actor version. + /// Any JSON-serializable version definition. + /// A token to cancel the request. + public async Task CreateAsync(object version, CancellationToken cancellationToken = default) + { + return new ActorVersion(await _ctx.CreateResourceAsync(new QueryParams(), version, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/BuildClient.cs b/src/Apify.Client/Resources/BuildClient.cs new file mode 100644 index 0000000..c6bb007 --- /dev/null +++ b/src/Apify.Client/Resources/BuildClient.cs @@ -0,0 +1,69 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// A client for a specific Actor build (/v2/actor-builds/{buildId}). +public sealed class BuildClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + internal BuildClient(HttpClientCore http, string baseUrl, string id) + { + _http = http; + _ctx = ResourceContext.Single(http, baseUrl, "actor-builds", id); + } + + /// + /// Fetches the build, optionally asking the API to wait up to + /// seconds (max 60) for the build to finish before responding. Returns null if it does not exist. + /// + /// Optional server-side wait in seconds. + /// A token to cancel the request. + public async Task GetAsync(int? waitForFinishSecs = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. + q.AddInt("waitForFinish", _ctx.ClampServerWait(waitForFinishSecs)); + var data = await _ctx.GetResourceAsync("", q, cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new Build(obj) : null; + } + + /// Aborts the build and returns its updated state. + /// A token to cancel the request. + public async Task AbortAsync(CancellationToken cancellationToken = default) + { + return new Build(await _ctx.PostWithBodyAsync("abort", new QueryParams(), null, "", cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the build. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// + /// Polls until the build reaches a terminal state or elapses (null + /// waits indefinitely). Returns the latest build. + /// + /// The wait budget in seconds, or null to wait indefinitely. + /// A token to cancel the wait. + public async Task WaitForFinishAsync(int? waitSecs = null, CancellationToken cancellationToken = default) + { + var data = await _ctx.WaitForFinishAsync(waitSecs, "build", static d => new Build(d).IsTerminal, cancellationToken).ConfigureAwait(false); + return new Build(data); + } + + /// Returns the OpenAPI definition generated for the build, or null if unavailable. + /// A token to cancel the request. + public async Task GetOpenApiDefinitionAsync(CancellationToken cancellationToken = default) + { + var body = await _ctx.GetRawAsync("openapi.json", new QueryParams(), cancellationToken).ConfigureAwait(false); + return body is null ? null : Json.Decode(body) as JsonObject; + } + + /// A client for accessing this build's log. + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl("")); +} diff --git a/src/Apify.Client/Resources/BuildCollectionClient.cs b/src/Apify.Client/Resources/BuildCollectionClient.cs new file mode 100644 index 0000000..ce4d187 --- /dev/null +++ b/src/Apify.Client/Resources/BuildCollectionClient.cs @@ -0,0 +1,31 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a build collection: the account-wide collection (GET /v2/actor-builds) or an +/// Actor's builds (GET /v2/actors/{id}/builds). +/// +public sealed class BuildCollectionClient +{ + private readonly ResourceContext _ctx; + + internal BuildCollectionClient(HttpClientCore http, string baseUrl, string resourcePath) + { + _ctx = ResourceContext.Collection(http, baseUrl, resourcePath); + } + + /// Lists builds. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new Build(d), cancellationToken); + } +} diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs new file mode 100644 index 0000000..6476d98 --- /dev/null +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -0,0 +1,185 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for a specific dataset (and run-nested variants). +public sealed class DatasetClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + private DatasetClient(HttpClientCore http, ResourceContext ctx) + { + _http = http; + _ctx = ctx; + } + + internal static DatasetClient ForId(HttpClientCore http, string baseUrl, string id) + => new(http, ResourceContext.Single(http, baseUrl, "datasets", id)); + + internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath) + => new(http, ResourceContext.Collection(http, baseUrl, subPath)); + + internal DatasetClient WithPublicBase(string publicBaseUrl) + { + _ctx.WithPublicOrigin(publicBaseUrl); + return this; + } + + /// Fetches the dataset metadata, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new Dataset(obj) : null; + } + + /// Updates the dataset metadata (e.g. name, title) and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new Dataset(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the dataset. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// + /// Lists items from the dataset, each decoded to a (objects become + /// ). + /// + /// + /// The dataset items endpoint returns a bare JSON array (not a data envelope) and reports pagination via + /// X-Apify-Pagination-* headers, surfaced in the returned page. + /// + /// Optional item filtering/projection and pagination. + /// A token to cancel the request. + public async Task> ListItemsAsync(DatasetListItemsOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new DatasetListItemsOptions(); + var q = new QueryParams(); + options.AppendTo(q); + var url = q.ApplyToUrl(_ctx.SubUrl("items")); + using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + var items = new List(); + if (Json.Decode(body) is JsonArray array) + { + foreach (var item in array) + { + items.Add(item); + } + } + + var count = items.Count; + return PaginationList.FromItems( + items, + HeaderInt(response, "X-Apify-Pagination-Total", count), + HeaderInt(response, "X-Apify-Pagination-Offset", 0), + HeaderInt(response, "X-Apify-Pagination-Limit", count), + options.Desc ?? false); + } + + /// + /// Downloads dataset items serialized in the given format, returning the raw bytes. Unlike + /// (parsed items), this returns the items already serialized to JSON, CSV, + /// XLSX, XML, RSS or HTML — useful for exporting. Bytes (not a decoded string) are returned so binary + /// formats such as (a ZIP-based export) are not corrupted; decode + /// text formats yourself, e.g. System.Text.Encoding.UTF8.GetString(bytes). + /// + /// The output format. + /// Optional format-specific and filtering options. + /// A token to cancel the request. + public async Task DownloadItemsAsync(DownloadItemsFormat format, DatasetDownloadOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddString("format", format.ToWireValue()); + (options ?? new DatasetDownloadOptions()).AppendTo(q); + var url = q.ApplyToUrl(_ctx.SubUrl("items")); + using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + } + + /// Pushes one or more items to the dataset. + /// Must serialize to a JSON object or an array of objects. + /// A token to cancel the request. + public async Task PushItemsAsync(object items, CancellationToken cancellationToken = default) + { + using var response = await _http.CallAsync( + HttpMethod.Post, + _ctx.SubUrl("items"), + Json.Encode(items), + ResourceContext.ContentTypeJsonCharset, + timeout: _ctx.RequestTimeout, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// Returns statistical information about the dataset, or null if unavailable. + /// A token to cancel the request. + public async Task GetStatisticsAsync(CancellationToken cancellationToken = default) + { + var body = await _ctx.GetRawAsync("statistics", new QueryParams(), cancellationToken).ConfigureAwait(false); + return body is null ? null : Json.DecodeData(body) as JsonObject; + } + + /// + /// Builds a public URL for downloading this dataset's items. + /// + /// + /// It fetches the dataset, and if the dataset exposes a URL-signing secret key (i.e. it is private), + /// appends an HMAC-SHA256 signature so the URL grants access without an API token. + /// optionally bounds the validity of a signed URL (null for + /// non-expiring). The URL is built from the configured public base URL. + /// + /// Optional item filtering/projection options forwarded into the URL. + /// Optional expiry in seconds for a signed URL. + /// A token to cancel the request. + public async Task CreateItemsPublicUrlAsync( + DatasetListItemsOptions? options = null, + int? expiresInSecs = null, + CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new DatasetListItemsOptions()).AppendTo(q); + var dataset = await GetAsync(cancellationToken).ConfigureAwait(false); + if (dataset is not null) + { + var secret = JsonValues.String(dataset.ToJsonObject(), "urlSigningSecretKey"); + if (secret is not null) + { + var signature = Signatures.SignStorageContent(secret, dataset.Id ?? string.Empty, expiresInSecs); + q.AddString("signature", signature); + } + } + + return q.ApplyToUrl(_ctx.PublicUrl("items")); + } + + private static long HeaderInt(HttpResponseMessage response, string name, long fallback) + { + if (response.Headers.TryGetValues(name, out var values) + || response.Content.Headers.TryGetValues(name, out values)) + { + foreach (var value in values) + { + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + } + } + + return fallback; + } +} diff --git a/src/Apify.Client/Resources/DatasetCollectionClient.cs b/src/Apify.Client/Resources/DatasetCollectionClient.cs new file mode 100644 index 0000000..cecf4e7 --- /dev/null +++ b/src/Apify.Client/Resources/DatasetCollectionClient.cs @@ -0,0 +1,42 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the dataset collection (GET/POST /v2/datasets). +public sealed class DatasetCollectionClient +{ + private readonly ResourceContext _ctx; + + internal DatasetCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "datasets"); + } + + /// Lists datasets. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public Task> ListAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new StorageListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new Dataset(d), cancellationToken); + } + + /// + /// Gets the dataset with the given name, creating it if it does not exist. An empty/null name + /// creates a new unnamed dataset. An optional is sent when creating the + /// dataset, mirroring the reference client's getOrCreate(name, { schema }). + /// + /// The dataset name, or null for a new unnamed dataset. + /// An optional dataset schema to send on creation. + /// A token to cancel the request. + public async Task GetOrCreateAsync(string? name = null, JsonNode? schema = null, CancellationToken cancellationToken = default) + { + return new Dataset(await _ctx.GetOrCreateNamedAsync(name, schema, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/KeyValueStoreClient.cs b/src/Apify.Client/Resources/KeyValueStoreClient.cs new file mode 100644 index 0000000..ff09446 --- /dev/null +++ b/src/Apify.Client/Resources/KeyValueStoreClient.cs @@ -0,0 +1,186 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for a specific key-value store (and run-nested variants). +public sealed class KeyValueStoreClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + private KeyValueStoreClient(HttpClientCore http, ResourceContext ctx) + { + _http = http; + _ctx = ctx; + } + + internal static KeyValueStoreClient ForId(HttpClientCore http, string baseUrl, string id) + => new(http, ResourceContext.Single(http, baseUrl, "key-value-stores", id)); + + internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath) + => new(http, ResourceContext.Collection(http, baseUrl, subPath)); + + internal KeyValueStoreClient WithPublicBase(string publicBaseUrl) + { + _ctx.WithPublicOrigin(publicBaseUrl); + return this; + } + + /// Fetches the store metadata, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is System.Text.Json.Nodes.JsonObject obj ? new KeyValueStore(obj) : null; + } + + /// Updates the store metadata (e.g. name) and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new KeyValueStore(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the store. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// Lists the keys stored in this key-value store. + /// Optional key-listing filters and pagination. + /// A token to cancel the request. + public async Task ListKeysAsync(ListKeysOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListKeysOptions()).AppendTo(q); + return KeyValueStoreKeysPage.FromData(await _ctx.GetResourceRequiredAsync("keys", q, cancellationToken).ConfigureAwait(false)); + } + + /// Reports whether a record with the given key exists. + /// The record key. + /// A token to cancel the request. + public Task RecordExistsAsync(string key, CancellationToken cancellationToken = default) + => _ctx.HeadExistsAsync("records/" + ResourceContext.EncodePathSegment(key), new QueryParams(), cancellationToken); + + /// + /// Fetches a record by key, or null if it does not exist. Like the reference client, it requests + /// the record as an attachment so the API returns the raw bytes directly. + /// + /// The record key. + /// Optional fetch options. + /// A token to cancel the request. + public async Task GetRecordAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new GetRecordOptions { Attachment = true }; + var q = new QueryParams(); + options.AppendTo(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("records/" + ResourceContext.EncodePathSegment(key))); + try + { + using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); + // Read the raw bytes (not a decoded string) so binary records survive the round-trip intact. + var body = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var contentType = response.Content.Headers.ContentType?.ToString(); + return new KeyValueStoreRecord(key, body, string.IsNullOrEmpty(contentType) ? null : contentType); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// + /// Stores a record with raw bytes and the given content type, honoring the given write options + /// (TimeoutSecs, DoNotRetryTimeouts). + /// + /// The record key. + /// The raw record bytes. + /// The record's MIME type. + /// Optional write options. + /// A token to cancel the request. + public Task SetRecordAsync(string key, byte[] value, string contentType, SetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new SetRecordOptions(); + var timeout = options.TimeoutSecs is not null ? TimeSpan.FromSeconds(options.TimeoutSecs.Value) : (TimeSpan?)null; + return _ctx.PutRawAsync( + "records/" + ResourceContext.EncodePathSegment(key), + new QueryParams(), + value, + contentType, + timeout, + options.DoNotRetryTimeouts, + cancellationToken); + } + + /// Stores a record holding the JSON serialization of . + /// The record key. + /// Any JSON-serializable value. + /// A token to cancel the request. + public Task SetRecordJsonAsync(string key, object? value, CancellationToken cancellationToken = default) + => SetRecordAsync(key, System.Text.Encoding.UTF8.GetBytes(Json.Encode(value)), ResourceContext.ContentTypeJsonCharset, null, cancellationToken); + + /// Deletes a record by key. + /// The record key. + /// A token to cancel the request. + public Task DeleteRecordAsync(string key, CancellationToken cancellationToken = default) + => _ctx.DeleteResourceAsync("records/" + ResourceContext.EncodePathSegment(key), cancellationToken); + + /// + /// Builds a public URL for fetching the given record. It fetches the store, and if the store exposes a + /// URL-signing secret key (i.e. it is private), appends an HMAC-SHA256 signature so the URL grants + /// access without an API token. The URL is built from the configured public base URL. + /// + /// The record key. + /// A token to cancel the request. + public async Task GetRecordPublicUrlAsync(string key, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + var store = await GetAsync(cancellationToken).ConfigureAwait(false); + if (store is not null) + { + var secret = JsonValues.String(store.ToJsonObject(), "urlSigningSecretKey"); + if (secret is not null) + { + q.AddString("signature", Signatures.CreateHmacSignature(secret, key)); + } + } + + return q.ApplyToUrl(_ctx.PublicUrl("records/" + ResourceContext.EncodePathSegment(key))); + } + + /// + /// Builds a public URL for listing this store's keys, forwarding the given key-listing filters into the + /// URL. As with , a signature is appended for private stores unless + /// the caller already supplied one. optionally bounds a signed URL. + /// + /// Optional key-listing filters forwarded into the URL. + /// Optional expiry in seconds for a signed URL. + /// A token to cancel the request. + public async Task CreateKeysPublicUrlAsync(ListKeysOptions? options = null, int? expiresInSecs = null, CancellationToken cancellationToken = default) + { + options ??= new ListKeysOptions(); + var q = new QueryParams(); + options.AppendTo(q); + if (options.Signature is null) + { + var store = await GetAsync(cancellationToken).ConfigureAwait(false); + if (store is not null) + { + var secret = JsonValues.String(store.ToJsonObject(), "urlSigningSecretKey"); + if (secret is not null) + { + q.AddString("signature", Signatures.SignStorageContent(secret, store.Id ?? string.Empty, expiresInSecs)); + } + } + } + + return q.ApplyToUrl(_ctx.PublicUrl("keys")); + } +} diff --git a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs new file mode 100644 index 0000000..065d18f --- /dev/null +++ b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs @@ -0,0 +1,41 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the key-value store collection (GET/POST /v2/key-value-stores). +public sealed class KeyValueStoreCollectionClient +{ + private readonly ResourceContext _ctx; + + internal KeyValueStoreCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "key-value-stores"); + } + + /// Lists key-value stores. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public Task> ListAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new StorageListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new KeyValueStore(d), cancellationToken); + } + + /// + /// Gets the store with the given name, creating it if it does not exist. An empty/null name + /// creates a new unnamed store. An optional is sent when creating the store. + /// + /// The store name, or null for a new unnamed store. + /// An optional store schema to send on creation. + /// A token to cancel the request. + public async Task GetOrCreateAsync(string? name = null, JsonNode? schema = null, CancellationToken cancellationToken = default) + { + return new KeyValueStore(await _ctx.GetOrCreateNamedAsync(name, schema, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/LogClient.cs b/src/Apify.Client/Resources/LogClient.cs new file mode 100644 index 0000000..bac7b31 --- /dev/null +++ b/src/Apify.Client/Resources/LogClient.cs @@ -0,0 +1,69 @@ +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for accessing the log of an Actor build or run (/v2/logs/{buildOrRunId}, or the +/// run/build-nested .../log). +/// +public sealed class LogClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + private LogClient(HttpClientCore http, ResourceContext ctx) + { + _http = http; + _ctx = ctx; + } + + internal static LogClient ForId(HttpClientCore http, string baseUrl, string id) + => new(http, ResourceContext.Single(http, baseUrl, "logs", id)); + + internal static LogClient Nested(HttpClientCore http, string baseUrl) + => new(http, ResourceContext.Collection(http, baseUrl, "log")); + + /// Fetches the log as text, or null if the log does not exist. + /// Optional log-content options. + /// A token to cancel the request. + public Task GetAsync(LogOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new LogOptions()).AppendTo(q); + return _ctx.GetRawAsync("", q, cancellationToken); + } + + /// + /// Opens a live, streaming connection to the log and returns a stream over the log bytes. + /// + /// + /// Unlike , this bypasses the buffered/retrying transport so the log can be + /// followed in real time as the run produces it (the stream=1 query parameter). Because the + /// response is consumed incrementally, it is not retried. The caller must dispose the returned stream. + /// + /// Optional log-content options. + /// A token to cancel the request. + public async Task StreamAsync(LogOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddBool("stream", true); + (options ?? new LogOptions()).AppendTo(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("")); + + var response = await _http.StreamAsync(url, cancellationToken).ConfigureAwait(false); + var status = (int)response.StatusCode; + if (status >= HttpClientCore.MaxSuccessStatus) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + response.Dispose(); + throw HttpClientCore.BuildApiError(status, body, 1, "GET", HttpClientCore.ExtractPath(url)); + } + + return await ResponseOwningStream.CreateAsync(response, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Apify.Client/Resources/NestedWebhookCollectionClient.cs b/src/Apify.Client/Resources/NestedWebhookCollectionClient.cs new file mode 100644 index 0000000..da488db --- /dev/null +++ b/src/Apify.Client/Resources/NestedWebhookCollectionClient.cs @@ -0,0 +1,17 @@ +using Apify.Client.Internal; + +namespace Apify.Client.Resources; + +/// +/// A read-only client for the webhooks nested under an Actor (GET /v2/actors/{id}/webhooks) or a +/// task (GET /v2/actor-tasks/{id}/webhooks). These endpoints only support listing; webhooks are +/// created through the account-wide (which targets an Actor or task +/// via the webhook's condition), so Create is intentionally not exposed. +/// +public sealed class NestedWebhookCollectionClient : AbstractWebhookCollectionClient +{ + internal NestedWebhookCollectionClient(HttpClientCore http, string baseUrl) + : base(http, baseUrl) + { + } +} diff --git a/src/Apify.Client/Resources/RequestQueueClient.cs b/src/Apify.Client/Resources/RequestQueueClient.cs new file mode 100644 index 0000000..275f291 --- /dev/null +++ b/src/Apify.Client/Resources/RequestQueueClient.cs @@ -0,0 +1,589 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Exceptions; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for a specific request queue (and run-nested variants). +public sealed class RequestQueueClient +{ + /// The API limit on requests per batch call; larger inputs are split into chunks of this size. + private const int MaxRequestsPerBatch = 25; + + /// + /// The API's maximum accepted request payload size (9 MiB). Batches are additionally split so no single + /// batch call exceeds this, matching the reference client's sliceArrayByByteLength. + /// + private const int MaxPayloadSizeBytes = 9 * 1024 * 1024; + + /// Safety margin (0.01%) subtracted from the payload limit, matching the reference client. + private const double PayloadSafetyBufferPercent = 0.0001; + + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + private readonly string? _clientKey; + private readonly TimeSpan? _timeout; + + private RequestQueueClient(HttpClientCore http, ResourceContext ctx, string? clientKey, TimeSpan? timeout) + { + _http = http; + _ctx = ctx; + _clientKey = clientKey; + _timeout = timeout; + } + + internal static RequestQueueClient ForId(HttpClientCore http, string baseUrl, string id, RequestQueueClientOptions? options) + { + var ctx = ResourceContext.Single(http, baseUrl, "request-queues", id); + var timeout = options?.TimeoutSecs is not null ? TimeSpan.FromSeconds(options.TimeoutSecs.Value) : (TimeSpan?)null; + ctx.WithTimeout(timeout); + return new RequestQueueClient(http, ctx, options?.ClientKey, timeout); + } + + internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath) + => new(http, ResourceContext.Collection(http, baseUrl, subPath), null, null); + + /// + /// Returns a copy of the client that identifies its requests with . A + /// stable client key is required to operate on locks the client itself created, and lets the API detect + /// whether multiple clients access a queue. + /// + /// The stable client key. + public RequestQueueClient WithClientKey(string clientKey) => new(_http, _ctx, clientKey, _timeout); + + /// Fetches the queue metadata, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new RequestQueue(obj) : null; + } + + /// Updates the queue metadata (e.g. name) and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new RequestQueue(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the queue. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// + /// Returns the requests at the head (front) of the queue, up to (null + /// for the server default). + /// + /// The maximum number of requests to return. + /// A token to cancel the request. + public async Task ListHeadAsync(int? limit = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddInt("limit", limit); + ApplyClientKey(q); + return RequestQueueHead.FromData(await _ctx.GetResourceRequiredAsync("head", q, cancellationToken).ConfigureAwait(false)); + } + + /// Adds a request to the queue. If is true, it is added to the front. + /// The request to add. + /// Whether to add to the front of the queue. + /// A token to cancel the request. + public async Task AddRequestAsync(RequestQueueRequest request, bool forefront = false, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddBool("forefront", forefront); + ApplyClientKey(q); + var data = await _ctx.PostWithBodyAsync("requests", q, Json.Encode(request.ToJsonObject()), ResourceContext.ContentTypeJson, cancellationToken).ConfigureAwait(false); + return new RequestQueueOperationInfo(data); + } + + /// Fetches a request by ID, or null if it does not exist. + /// The request ID. + /// A token to cancel the request. + public async Task GetRequestAsync(string id, CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("requests/" + ResourceContext.EncodePathSegment(id), new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? RequestQueueRequest.FromJsonObject(obj) : null; + } + + /// + /// Updates an existing request (identified by its ID field) and returns the operation info. If + /// is true, the request is moved to the front of the queue. + /// + /// The request to update (must have an ID). + /// Whether to move the request to the front. + /// A token to cancel the request. + public async Task UpdateRequestAsync(RequestQueueRequest request, bool forefront = false, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddBool("forefront", forefront); + ApplyClientKey(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(request.Id ?? string.Empty))); + using var response = await _http.CallAsync(HttpMethod.Put, url, Json.Encode(request.ToJsonObject()), ResourceContext.ContentTypeJson, _timeout, cancellationToken: cancellationToken).ConfigureAwait(false); + var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + return new RequestQueueOperationInfo(data as JsonObject ?? new JsonObject()); + } + + /// Deletes a request by ID. + /// The request ID. + /// A token to cancel the request. + public async Task DeleteRequestAsync(string id, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + ApplyClientKey(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(id))); + try + { + using var response = await _http.CallAsync(HttpMethod.Delete, url, timeout: _timeout, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + // A missing request is a successful no-op for delete. + } + } + + /// + /// Atomically returns and locks up to requests from the head of the queue for + /// seconds. Returns the raw locked-head object. + /// + /// How long to lock the returned requests, in seconds. + /// The maximum number of requests to lock. + /// A token to cancel the request. + public Task ListAndLockHeadAsync(int lockSecs, int? limit = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddInt("lockSecs", lockSecs).AddInt("limit", limit); + ApplyClientKey(q); + return _ctx.PostWithBodyAsync("head/lock", q, null, "", cancellationToken); + } + + /// + /// Adds multiple requests to the queue. If is true, they are added to the + /// front. + /// + /// + /// The input is automatically split into chunks of at most 25 requests (the API count limit) that + /// additionally respect the API's ~9 MiB payload-size limit. Chunks are dispatched using up to + /// concurrent API calls (set it to 1 for sequential + /// dispatch). Requests the API returns as unprocessed in a successful response (typically rate-limited) + /// are retried with exponential backoff; the per-chunk results are merged in input order. Every request + /// must carry a non-empty UniqueKey. Consistent with the reference client, this method does not + /// throw on API errors: if a batch call fails and the transport did not retry, that chunk's + /// not-yet-processed requests are returned in . Invalid + /// input (empty uniqueKey, oversized request) is rejected up front with . + /// + /// The requests to add. + /// Whether to add to the front of the queue. + /// Optional batch-add tuning. + /// A token to cancel the request. + public async Task BatchAddRequestsAsync( + IReadOnlyList requests, + bool forefront = false, + BatchAddRequestsOptions? options = null, + CancellationToken cancellationToken = default) + { + options ??= new BatchAddRequestsOptions(); + var list = new List(requests); + + for (var i = 0; i < list.Count; i++) + { + if (string.IsNullOrEmpty(list[i].UniqueKey)) + { + throw new ArgumentException( + $"BatchAddRequests: the request at index {i} is missing a non-empty UniqueKey", nameof(requests)); + } + } + + var payloadSizeLimitBytes = MaxPayloadSizeBytes - (int)Math.Ceiling(MaxPayloadSizeBytes * PayloadSafetyBufferPercent); + + // Pre-compute all chunks up front (bounded first by the count limit of 25, then by payload byte size) + // so they can be dispatched sequentially or with bounded parallelism. + var chunks = new List>(); + var index = 0; + while (index < list.Count) + { + var countSlice = list.GetRange(index, Math.Min(MaxRequestsPerBatch, list.Count - index)); + var chunk = SliceByByteLength(countSlice, payloadSizeLimitBytes, index); + chunks.Add(chunk); + index += chunk.Count; + } + + var merged = new BatchAddResult(); + if (chunks.Count == 0) + { + return merged; + } + + // Sequential path when parallelism is disabled or there is only a single chunk. + if (options.MaxParallel <= 1 || chunks.Count == 1) + { + foreach (var chunk in chunks) + { + merged.Merge(await BatchAddChunkWithRetriesAsync(chunk, forefront, options, cancellationToken).ConfigureAwait(false)); + } + + return merged; + } + + // Bounded-parallel dispatch: at most MaxParallel chunk calls run concurrently, gated by a semaphore. + // Results are merged in chunk (input) order so the output stays deterministic regardless of which + // chunk finishes first. + using var gate = new SemaphoreSlim(options.MaxParallel); + var tasks = new List>(chunks.Count); + foreach (var chunk in chunks) + { + tasks.Add(DispatchChunkAsync(chunk, forefront, options, gate, cancellationToken)); + } + + foreach (var result in await Task.WhenAll(tasks).ConfigureAwait(false)) + { + merged.Merge(result); + } + + return merged; + } + + /// Runs one chunk's add-with-retries under the concurrency gate. + private async Task DispatchChunkAsync( + List chunk, + bool forefront, + BatchAddRequestsOptions options, + SemaphoreSlim gate, + CancellationToken cancellationToken) + { + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await BatchAddChunkWithRetriesAsync(chunk, forefront, options, cancellationToken).ConfigureAwait(false); + } + finally + { + gate.Release(); + } + } + + /// + /// Returns the longest leading run of whose combined JSON payload stays + /// under , always keeping at least one request so iteration makes + /// progress. Ports the reference client's sliceArrayByByteLength. + /// + private static List SliceByByteLength(List requests, int maxByteLength, int startIndex) + { + var payloads = new List(requests.Count); + foreach (var r in requests) + { + payloads.Add(r.ToJsonObject()); + } + + if (Encoding.UTF8.GetByteCount(Json.Encode(payloads)) < maxByteLength) + { + return requests; + } + + var sliced = new List(); + var byteLength = 2; // the two bytes of an empty array "[]" + for (var i = 0; i < requests.Count; i++) + { + var itemBytes = Encoding.UTF8.GetByteCount(Json.Encode(requests[i].ToJsonObject())); + if (itemBytes > maxByteLength) + { + throw new ArgumentException( + $"BatchAddRequests: the request at index {startIndex + i} exceeds the maximum payload size ({maxByteLength} bytes)"); + } + + if (byteLength + itemBytes >= maxByteLength) + { + break; + } + + byteLength += itemBytes; + sliced.Add(requests[i]); + } + + // Guarantee forward progress: keep at least the first request (it fits under the hard max). + if (sliced.Count == 0) + { + sliced.Add(requests[0]); + } + + return sliced; + } + + private async Task BatchAddChunkWithRetriesAsync(List chunk, bool forefront, BatchAddRequestsOptions options, CancellationToken cancellationToken) + { + var maxRetries = options.MaxUnprocessedRequestsRetries; + var minDelayMillis = options.MinDelayBetweenUnprocessedRequestsRetriesMillis; + + var remaining = chunk; + var processed = new List(); + var unprocessed = new List(); + + for (var attempt = 0; attempt <= maxRetries; attempt++) + { + BatchAddResult response; + try + { + response = await BatchAddChunkAsync(remaining, forefront, cancellationToken).ConfigureAwait(false); + } + catch (ApifyApiException) + { + // Matches the JS reference: when the HTTP call fails and the transport did not (or was told + // not to) retry, the requests not yet processed in THIS chunk are reported as unprocessed and + // we stop — keeping the method's non-throwing contract so a multi-chunk call still returns + // every earlier chunk's already-merged results instead of aborting the whole operation. + unprocessed = RequestsNotYetProcessed(chunk, processed); + break; + } + + processed.AddRange(response.ProcessedRequests); + // Only requests the API reports as unprocessed in this SUCCESSFUL response are retried. + unprocessed = new List(response.UnprocessedRequests); + remaining = RequestsNotYetProcessed(chunk, processed); + if (remaining.Count == 0) + { + break; + } + + if (attempt < maxRetries) + { + await SleepBackoffAsync(attempt, minDelayMillis, cancellationToken).ConfigureAwait(false); + } + } + + var result = new BatchAddResult(); + result.SetProcessedRequests(processed); + result.SetUnprocessedRequests(unprocessed); + return result; + } + + private async Task BatchAddChunkAsync(List requests, bool forefront, CancellationToken cancellationToken) + { + var q = new QueryParams(); + q.AddBool("forefront", forefront); + ApplyClientKey(q); + var payload = new List(requests.Count); + foreach (var r in requests) + { + payload.Add(r.ToJsonObject()); + } + + var data = await _ctx.PostWithBodyAsync("requests/batch", q, Json.Encode(payload), ResourceContext.ContentTypeJson, cancellationToken).ConfigureAwait(false); + + var processed = new List(); + if (data.TryGetPropertyValue("processedRequests", out var pNode) && pNode is JsonArray pArray) + { + foreach (var item in pArray) + { + processed.Add(new RequestQueueOperationInfo(item as JsonObject ?? new JsonObject())); + } + } + + var unprocessed = new List(); + if (data.TryGetPropertyValue("unprocessedRequests", out var uNode) && uNode is JsonArray uArray) + { + foreach (var item in uArray) + { + unprocessed.Add(RequestQueueRequest.FromJsonObject(item as JsonObject ?? new JsonObject())); + } + } + + return new BatchAddResult(processed, unprocessed); + } + + private static List RequestsNotYetProcessed(List chunk, List processed) + { + var processedKeys = new HashSet(StringComparer.Ordinal); + foreach (var info in processed) + { + if (info.UniqueKey is not null) + { + processedKeys.Add(info.UniqueKey); + } + } + + var remaining = new List(); + foreach (var request in chunk) + { + if (!processedKeys.Contains(request.UniqueKey ?? string.Empty)) + { + remaining.Add(request); + } + } + + return remaining; + } + + private static Task SleepBackoffAsync(int attempt, int minDelayMillis, CancellationToken cancellationToken) + { + if (minDelayMillis <= 0) + { + return Task.CompletedTask; + } + + // (1 + random) * 2^attempt * minDelay — exponential backoff with jitter, matching the reference. + var factor = (1 + Random.Shared.NextDouble()) * Math.Pow(2, attempt); + var delayMillis = (int)Math.Floor(factor * minDelayMillis); + return Task.Delay(delayMillis, cancellationToken); + } + + /// + /// Deletes multiple requests in a single call. Each entry identifies a request (e.g. by id or + /// uniqueKey). Returns the raw batch result. + /// + /// A JSON-serializable list identifying the requests to delete. + /// A token to cancel the request. + public Task BatchDeleteRequestsAsync(object requests, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + ApplyClientKey(q); + return _ctx.DeleteWithBodyAsync("requests/batch", q, requests, cancellationToken); + } + + /// Lists the queue's requests with pagination. Returns the raw response. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public async Task ListRequestsAsync(ListRequestsOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListRequestsOptions(); + options.Validate(); + var q = new QueryParams(); + options.AppendTo(q); + ApplyClientKey(q); + var data = await _ctx.GetResourceRequiredAsync("requests", q, cancellationToken).ConfigureAwait(false); + return data as JsonObject ?? new JsonObject(); + } + + /// + /// Extends the lock on a request by seconds. If + /// is true, the request is moved to the front when its lock expires. Returns the raw response. + /// + /// The request ID. + /// How much longer to hold the lock, in seconds. + /// Whether to move the request to the front when the lock expires. + /// A token to cancel the request. + public async Task ProlongRequestLockAsync(string id, int lockSecs, bool forefront = false, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddInt("lockSecs", lockSecs).AddBool("forefront", forefront); + ApplyClientKey(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(id) + "/lock")); + using var response = await _http.CallAsync(HttpMethod.Put, url, null, "", _timeout, cancellationToken: cancellationToken).ConfigureAwait(false); + var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + return data as JsonObject ?? new JsonObject(); + } + + /// + /// Releases the lock on a request. If is true, the request is moved to the + /// front of the queue. + /// + /// The request ID. + /// Whether to move the request to the front. + /// A token to cancel the request. + public async Task DeleteRequestLockAsync(string id, bool forefront = false, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddBool("forefront", forefront); + ApplyClientKey(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(id) + "/lock")); + try + { + using var response = await _http.CallAsync(HttpMethod.Delete, url, timeout: _timeout, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + // A missing lock is a successful no-op. + } + } + + /// Releases all locks the client holds on this queue's requests. Returns the raw response. + /// A token to cancel the request. + public Task UnlockRequestsAsync(CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + ApplyClientKey(q); + return _ctx.PostWithBodyAsync("requests/unlock", q, null, "", cancellationToken); + } + + /// + /// Lazily iterates over the queue's requests, transparently following pagination. + /// + /// + /// With no options it fetches pages of up to + /// requests until the queue is exhausted. The options mirror the reference client: Limit caps the + /// total number of requests yielded across all pages, MaxPageLimit caps the page size, + /// ExclusiveStartId/Cursor choose the starting point (first page only), and Filter + /// restricts to locked/pending requests. + /// + /// Optional iteration options. + /// A token to cancel the iteration. + public async IAsyncEnumerable PaginateRequestsAsync( + PaginateRequestsOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new PaginateRequestsOptions(); + options.Validate(); + + var maxPageLimit = options.MaxPageLimit ?? PaginateRequestsOptions.DefaultMaxPageLimit; + var limit = options.Limit; // total across all pages; null = unbounded + var nextCursor = options.Cursor; + var nextExclusiveStartId = options.ExclusiveStartId; // used for the first page only + var iterated = 0; + + while (true) + { + var pageLimit = limit is not null ? Math.Min(maxPageLimit, limit.Value - iterated) : maxPageLimit; + + var page = await ListRequestsAsync( + new ListRequestsOptions + { + Limit = pageLimit, + ExclusiveStartId = nextExclusiveStartId, + Cursor = nextCursor, + Filter = options.Filter, + }, + cancellationToken).ConfigureAwait(false); + + var items = page.TryGetPropertyValue("items", out var itemsNode) && itemsNode is JsonArray array + ? array + : new JsonArray(); + if (items.Count == 0) + { + yield break; + } + + foreach (var item in items) + { + yield return RequestQueueRequest.FromJsonObject(item as JsonObject ?? new JsonObject()); + } + + iterated += items.Count; + + nextCursor = JsonValues.String(page, "nextCursor"); + if ((limit is not null && iterated >= limit.Value) || string.IsNullOrEmpty(nextCursor)) + { + yield break; + } + + // After the first page, paginate purely by cursor. + nextExclusiveStartId = null; + } + } + + private void ApplyClientKey(QueryParams q) + { + if (!string.IsNullOrEmpty(_clientKey)) + { + q.AddString("clientKey", _clientKey); + } + } +} diff --git a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs new file mode 100644 index 0000000..ffc3cf9 --- /dev/null +++ b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs @@ -0,0 +1,39 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the request queue collection (GET/POST /v2/request-queues). +public sealed class RequestQueueCollectionClient +{ + private readonly ResourceContext _ctx; + + internal RequestQueueCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "request-queues"); + } + + /// Lists request queues. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public Task> ListAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new StorageListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new RequestQueue(d), cancellationToken); + } + + /// + /// Gets the queue with the given name, creating it if it does not exist. An empty/null name + /// creates a new unnamed queue. + /// + /// The queue name, or null for a new unnamed queue. + /// A token to cancel the request. + public async Task GetOrCreateAsync(string? name = null, CancellationToken cancellationToken = default) + { + return new RequestQueue(await _ctx.GetOrCreateNamedAsync(name, null, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs new file mode 100644 index 0000000..f3ced69 --- /dev/null +++ b/src/Apify.Client/Resources/RunClient.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a specific Actor run. +/// +/// +/// It provides CRUD methods plus convenience helpers (abort, metamorph, reboot, resurrect, charge, +/// wait-for-finish) and accessors for the run's default storages and log. +/// +public sealed class RunClient +{ + /// Header the API uses to deduplicate charge requests. + private const string ChargeIdempotencyHeader = "idempotency-key"; + + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + private readonly string _id; + + internal RunClient(HttpClientCore http, string baseUrl, string resourcePath, string id) + { + _http = http; + _id = id; + _ctx = ResourceContext.Single(http, baseUrl, resourcePath, id); + } + + /// + /// Pins the status/origin query parameters inherited by all calls on this client (used by + /// the last-run accessors). Empty values are skipped. + /// + internal void SetLastRunParams(LastRunOptions options) + { + if (!string.IsNullOrEmpty(options.Status)) + { + _ctx.BaseParams.AddRaw("status", options.Status); + } + + if (!string.IsNullOrEmpty(options.Origin)) + { + _ctx.BaseParams.AddRaw("origin", options.Origin); + } + } + + /// + /// Fetches the run, optionally asking the API to wait up to + /// seconds (max 60) for the run to reach a terminal state. Returns null if it does not exist. + /// + /// Optional server-side wait in seconds. + /// A token to cancel the request. + public async Task GetAsync(int? waitForFinishSecs = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddInt("waitForFinish", _ctx.ClampServerWait(waitForFinishSecs)); + var data = await _ctx.GetResourceAsync("", q, cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new ActorRun(obj) : null; + } + + /// Updates the run with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new ActorRun(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the run. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// + /// Aborts the run. If is true, the run is signalled so it can + /// finish the current request before terminating; false aborts immediately. null omits + /// the parameter and lets the server apply its default (immediate abort). + /// + /// Whether to abort gracefully, or null for the server default. + /// A token to cancel the request. + public async Task AbortAsync(bool? gracefully = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddBool("gracefully", gracefully); + return new ActorRun(await _ctx.PostWithBodyAsync("abort", q, null, "", cancellationToken).ConfigureAwait(false)); + } + + /// Transforms the run into a run of another Actor with a new input. + /// The Actor to metamorph into. + /// The new input (null for none). + /// Optional metamorph options. + /// A token to cancel the request. + public async Task MetamorphAsync( + string targetActorId, + object? input = null, + MetamorphOptions? options = null, + CancellationToken cancellationToken = default) + { + options ??= new MetamorphOptions(); + var q = new QueryParams(); + q.AddString("targetActorId", targetActorId); + if (!string.IsNullOrEmpty(options.Build)) + { + q.AddString("build", options.Build); + } + + var body = input is null ? null : Json.Encode(input); + return new ActorRun(await _ctx.PostWithBodyAsync("metamorph", q, body, options.ContentTypeOrDefault(), cancellationToken).ConfigureAwait(false)); + } + + /// Reboots the run (restarts its container while keeping the same run). + /// A token to cancel the request. + public async Task RebootAsync(CancellationToken cancellationToken = default) + { + return new ActorRun(await _ctx.PostWithBodyAsync("reboot", new QueryParams(), null, "", cancellationToken).ConfigureAwait(false)); + } + + /// Resurrects a finished run, starting it again from the beginning. + /// Optional resurrect options. + /// A token to cancel the request. + public async Task ResurrectAsync(RunResurrectOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new RunResurrectOptions()).AppendTo(q); + return new ActorRun(await _ctx.PostWithBodyAsync("resurrect", q, null, "", cancellationToken).ConfigureAwait(false)); + } + + /// + /// Charges for a pay-per-event Actor run: records occurrences of a named event. Only meaningful for + /// runs of pay-per-event Actors. + /// + /// + /// An idempotency key is always sent (auto-generated if not provided), so a charge that is retried by + /// the transport is applied at most once, matching the reference client. + /// + /// The charge event details. + /// A token to cancel the request. + public async Task ChargeAsync(RunChargeOptions options, CancellationToken cancellationToken = default) + { + if (options.EventName.Length == 0) + { + throw new ArgumentException("RunChargeOptions.EventName is required and must not be empty", nameof(options)); + } + + var idempotencyKey = options.IdempotencyKey; + if (string.IsNullOrEmpty(idempotencyKey)) + { + idempotencyKey = GenerateIdempotencyKey(options.EventName); + } + + var body = new JsonObject + { + ["eventName"] = options.EventName, + ["count"] = options.CountValue(), + }; + using var response = await _http.CallAsync( + HttpMethod.Post, + _ctx.SubUrl("charge"), + Json.Encode(body), + ResourceContext.ContentTypeJson, + extraHeaders: new Dictionary { [ChargeIdempotencyHeader] = idempotencyKey }, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// + /// Builds a per-charge idempotency key of the form "{runId}-{eventName}-{millis}-{random}". It + /// need not be cryptographically secure, only unique enough to avoid collisions within a request. + /// + private string GenerateIdempotencyKey(string eventName) + { + return string.Format( + CultureInfo.InvariantCulture, + "{0}-{1}-{2}-{3}", + _id, + eventName, + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + Random.Shared.Next(0, 1000000)); + } + + /// + /// Polls until the run reaches a terminal state or elapses (null + /// waits indefinitely). Returns the latest run. + /// + /// The wait budget in seconds, or null to wait indefinitely. + /// A token to cancel the wait. + public async Task WaitForFinishAsync(int? waitSecs = null, CancellationToken cancellationToken = default) + { + var data = await _ctx.WaitForFinishAsync(waitSecs, "run", static d => new ActorRun(d).IsTerminal, cancellationToken).ConfigureAwait(false); + return new ActorRun(data); + } + + /// A client for this run's default dataset. + public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset"); + + /// A client for this run's default key-value store. + public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store"); + + /// A client for this run's default request queue. + public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue"); + + /// A client for accessing this run's log. + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl("")); + + /// + /// Opens a live stream of this run's raw log, for convenient log redirection. The caller reads (and + /// disposes) the returned stream. + /// + /// A token to cancel the request. + public Task GetStreamedLogAsync(CancellationToken cancellationToken = default) + => Log().StreamAsync(new LogOptions { Raw = true }, cancellationToken); +} diff --git a/src/Apify.Client/Resources/RunCollectionClient.cs b/src/Apify.Client/Resources/RunCollectionClient.cs new file mode 100644 index 0000000..28cc84a --- /dev/null +++ b/src/Apify.Client/Resources/RunCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a run collection: the account-wide collection (GET /v2/actor-runs), an Actor's +/// runs (GET /v2/actors/{id}/runs), or a task's runs (GET /v2/actor-tasks/{id}/runs). +/// +public sealed class RunCollectionClient +{ + private readonly ResourceContext _ctx; + + internal RunCollectionClient(HttpClientCore http, string baseUrl, string resourcePath) + { + _ctx = ResourceContext.Collection(http, baseUrl, resourcePath); + } + + /// Lists runs, applying the standard pagination and the run-specific filters. + /// Optional pagination. + /// Optional run-specific filters. + /// A token to cancel the request. + public Task> ListAsync( + ListOptions? options = null, + RunListOptions? filter = null, + CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + (filter ?? new RunListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new ActorRun(d), cancellationToken); + } +} diff --git a/src/Apify.Client/Resources/ScheduleClient.cs b/src/Apify.Client/Resources/ScheduleClient.cs new file mode 100644 index 0000000..0cb0430 --- /dev/null +++ b/src/Apify.Client/Resources/ScheduleClient.cs @@ -0,0 +1,43 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// A client for a specific schedule (/v2/schedules/{scheduleId}). +public sealed class ScheduleClient +{ + private readonly ResourceContext _ctx; + + internal ScheduleClient(HttpClientCore http, string baseUrl, string id) + { + _ctx = ResourceContext.Single(http, baseUrl, "schedules", id); + } + + /// Fetches the schedule, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new Schedule(obj) : null; + } + + /// Updates the schedule with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new Schedule(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the schedule. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// Fetches the schedule's invocation log as text, or null if absent. + /// A token to cancel the request. + public Task GetLogAsync(CancellationToken cancellationToken = default) + => _ctx.GetRawAsync("log", new QueryParams(), cancellationToken); +} diff --git a/src/Apify.Client/Resources/ScheduleCollectionClient.cs b/src/Apify.Client/Resources/ScheduleCollectionClient.cs new file mode 100644 index 0000000..3b35052 --- /dev/null +++ b/src/Apify.Client/Resources/ScheduleCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the schedule collection (GET/POST /v2/schedules). +public sealed class ScheduleCollectionClient +{ + private readonly ResourceContext _ctx; + + internal ScheduleCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "schedules"); + } + + /// Lists the account's schedules. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new Schedule(d), cancellationToken); + } + + /// Creates a new schedule. + /// Any JSON-serializable schedule definition. + /// A token to cancel the request. + public async Task CreateAsync(object schedule, CancellationToken cancellationToken = default) + { + return new Schedule(await _ctx.CreateResourceAsync(new QueryParams(), schedule, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/StoreCollectionClient.cs b/src/Apify.Client/Resources/StoreCollectionClient.cs new file mode 100644 index 0000000..deaf518 --- /dev/null +++ b/src/Apify.Client/Resources/StoreCollectionClient.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for browsing the Apify Store (GET /v2/store). +public sealed class StoreCollectionClient +{ + private readonly ResourceContext _ctx; + + internal StoreCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "store"); + } + + /// Returns a single page of Store Actors matching the options. + /// Optional listing filters and pagination. + /// A token to cancel the request. + public Task> ListAsync(StoreListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new StoreListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new ActorStoreListItem(d), cancellationToken); + } + + /// + /// Lazily iterates over all Store Actors matching the options, fetching pages on demand. The options' + /// Limit (if set) is used as the per-page size. + /// + /// Optional listing filters; Limit is used as the page size. + /// A token to cancel the iteration. + public async IAsyncEnumerable IterateAsync( + StoreListOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new StoreListOptions(); + var offset = options.Offset ?? 0; + while (true) + { + var page = await ListAsync(options.WithOffset(offset), cancellationToken).ConfigureAwait(false); + var items = page.Items; + foreach (var item in items) + { + yield return item; + } + + offset += items.Count; + if (items.Count == 0 || offset >= page.Total) + { + yield break; + } + } + } +} diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs new file mode 100644 index 0000000..b1f45fe --- /dev/null +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -0,0 +1,114 @@ +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a specific Actor task. +/// +/// +/// Tasks are pre-configured Actor runs with stored input. The client provides CRUD methods plus +/// convenience helpers to start/call the task and access its input, runs and webhooks. +/// +public sealed class TaskClient +{ + private readonly ApifyClient _root; + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + internal TaskClient(ApifyClient root, HttpClientCore http, string baseUrl, string id) + { + _root = root; + _http = http; + _ctx = ResourceContext.Single(http, baseUrl, "actor-tasks", id); + } + + /// Fetches the task object, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new ActorTask(obj) : null; + } + + /// Updates the task with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new ActorTask(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the task. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// Starts the task and returns immediately with the created run. + /// Optionally overrides the task's stored input (null to use it). + /// Optional run-start options. + /// A token to cancel the request. + public async Task StartAsync(object? input = null, TaskStartOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new TaskStartOptions()).AppendTo(q); + var body = input is null ? null : Json.Encode(input); + return new ActorRun(await _ctx.PostWithBodyAsync("runs", q, body, ResourceContext.ContentTypeJson, cancellationToken).ConfigureAwait(false)); + } + + /// Starts the task and waits (client-side polling) for it to finish. + /// Optionally overrides the task's stored input. + /// Optional run-start options. + /// Bounds the wait; null waits indefinitely. + /// A token to cancel the request. + public async Task CallAsync( + object? input = null, + TaskStartOptions? options = null, + int? waitSecs = null, + CancellationToken cancellationToken = default) + { + var run = await StartAsync(input, options, cancellationToken).ConfigureAwait(false); + return await _root.Run(run.Id ?? string.Empty).WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + } + + /// Fetches the task's stored input, or null if none is set. + /// A token to cancel the request. + public async Task GetInputAsync(CancellationToken cancellationToken = default) + { + var body = await _ctx.GetRawAsync("input", new QueryParams(), cancellationToken).ConfigureAwait(false); + return body is null ? null : Json.Decode(body); + } + + /// Replaces the task's stored input and returns the updated input. + /// Any JSON-serializable value. + /// A token to cancel the request. + public async Task UpdateInputAsync(object input, CancellationToken cancellationToken = default) + { + using var response = await _http.CallAsync( + HttpMethod.Put, + _ctx.SubUrl("input"), + Json.Encode(input), + ResourceContext.ContentTypeJson, + cancellationToken: cancellationToken).ConfigureAwait(false); + return Json.Decode(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + } + + /// Returns a client for the last run of this task, optionally filtered by status and/or origin. + /// Optional last-run filters. + public RunClient LastRun(LastRunOptions? options = null) + { + var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + client.SetLastRunParams(options ?? new LastRunOptions()); + return client; + } + + /// A client for this task's run collection. + public RunCollectionClient Runs() => new(_http, _ctx.SubUrl(""), "runs"); + + /// A read-only client for this task's webhook collection (GET /v2/actor-tasks/{id}/webhooks). + public NestedWebhookCollectionClient Webhooks() => new(_http, _ctx.SubUrl("")); +} diff --git a/src/Apify.Client/Resources/TaskCollectionClient.cs b/src/Apify.Client/Resources/TaskCollectionClient.cs new file mode 100644 index 0000000..0800d14 --- /dev/null +++ b/src/Apify.Client/Resources/TaskCollectionClient.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// A client for the Actor task collection (GET/POST /v2/actor-tasks). +public sealed class TaskCollectionClient +{ + private readonly ResourceContext _ctx; + + internal TaskCollectionClient(HttpClientCore http, string baseUrl) + { + _ctx = ResourceContext.Collection(http, baseUrl, "actor-tasks"); + } + + /// Lists the account's tasks. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new ActorTask(d), cancellationToken); + } + + /// Creates a new task. + /// Any JSON-serializable task definition. + /// A token to cancel the request. + public async Task CreateAsync(object task, CancellationToken cancellationToken = default) + { + return new ActorTask(await _ctx.CreateResourceAsync(new QueryParams(), task, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/UserClient.cs b/src/Apify.Client/Resources/UserClient.cs new file mode 100644 index 0000000..73983d9 --- /dev/null +++ b/src/Apify.Client/Resources/UserClient.cs @@ -0,0 +1,93 @@ +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// +/// A client for accessing user data (/v2/users/{userId} or /v2/users/me). +/// +/// +/// For the current user (me), it also exposes account usage and limits. Those endpoints only exist +/// for me and throw if called on another user's client. +/// +public sealed class UserClient +{ + private const string Me = "me"; + + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + private readonly bool _isMe; + + internal UserClient(HttpClientCore http, string baseUrl, string id) + { + _http = http; + _ctx = ResourceContext.Single(http, baseUrl, "users", id); + _isMe = id == Me; + } + + /// + /// Fetches the user. For me it returns private account details (via + /// ); for other users it returns the public profile. Returns + /// null if the user does not exist. + /// + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new User(obj) : null; + } + + /// + /// Fetches the current account's monthly usage for the month containing the given date (formatted as + /// YYYY-MM-DD). An empty/null date reports the current month. Only available for me. + /// + /// The date whose month to report, or null for the current month. + /// A token to cancel the request. + public async Task MonthlyUsageAsync(string? date = null, CancellationToken cancellationToken = default) + { + RequireMe(); + var q = new QueryParams(); + if (!string.IsNullOrEmpty(date)) + { + q.AddString("date", date); + } + + var data = await _ctx.GetResourceRequiredAsync("usage/monthly", q, cancellationToken).ConfigureAwait(false); + return data as JsonObject ?? new JsonObject(); + } + + /// Fetches the current account's resource limits. Only available for me. + /// A token to cancel the request. + public async Task LimitsAsync(CancellationToken cancellationToken = default) + { + RequireMe(); + var data = await _ctx.GetResourceRequiredAsync("limits", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data as JsonObject ?? new JsonObject(); + } + + /// Updates the current account's resource limits. Only available for me. + /// Any JSON-serializable limits object. + /// A token to cancel the request. + public async Task UpdateLimitsAsync(object newLimits, CancellationToken cancellationToken = default) + { + RequireMe(); + using var response = await _http.CallAsync( + HttpMethod.Put, + _ctx.SubUrl("limits"), + Json.Encode(newLimits), + ResourceContext.ContentTypeJson, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private void RequireMe() + { + if (!_isMe) + { + throw new System.InvalidOperationException("this operation is only available for the current user (use Me())"); + } + } +} diff --git a/src/Apify.Client/Resources/WebhookClient.cs b/src/Apify.Client/Resources/WebhookClient.cs new file mode 100644 index 0000000..14949a5 --- /dev/null +++ b/src/Apify.Client/Resources/WebhookClient.cs @@ -0,0 +1,50 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// A client for a specific webhook (/v2/webhooks/{webhookId}). +public sealed class WebhookClient +{ + private readonly HttpClientCore _http; + private readonly ResourceContext _ctx; + + internal WebhookClient(HttpClientCore http, string baseUrl, string id) + { + _http = http; + _ctx = ResourceContext.Single(http, baseUrl, "webhooks", id); + } + + /// Fetches the webhook, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new Webhook(obj) : null; + } + + /// Updates the webhook with the given fields and returns the updated object. + /// Any JSON-serializable set of fields to update. + /// A token to cancel the request. + public async Task UpdateAsync(object newFields, CancellationToken cancellationToken = default) + { + return new Webhook(await _ctx.UpdateResourceAsync("", newFields, cancellationToken).ConfigureAwait(false)); + } + + /// Deletes the webhook. + /// A token to cancel the request. + public Task DeleteAsync(CancellationToken cancellationToken = default) => _ctx.DeleteResourceAsync("", cancellationToken); + + /// Dispatches the webhook immediately and returns the resulting dispatch. + /// A token to cancel the request. + public async Task TestAsync(CancellationToken cancellationToken = default) + { + return new WebhookDispatch(await _ctx.PostWithBodyAsync("test", new QueryParams(), null, "", cancellationToken).ConfigureAwait(false)); + } + + /// A client for this webhook's dispatch collection. + public WebhookDispatchCollectionClient Dispatches() => new(_http, _ctx.SubUrl(""), "dispatches"); +} diff --git a/src/Apify.Client/Resources/WebhookCollectionClient.cs b/src/Apify.Client/Resources/WebhookCollectionClient.cs new file mode 100644 index 0000000..33a9c84 --- /dev/null +++ b/src/Apify.Client/Resources/WebhookCollectionClient.cs @@ -0,0 +1,27 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// +/// A client for the account-wide webhook collection (GET/POST /v2/webhooks), supporting both +/// listing and creation. Webhooks nested under an Actor or task are read-only and use +/// instead. +/// +public sealed class WebhookCollectionClient : AbstractWebhookCollectionClient +{ + internal WebhookCollectionClient(HttpClientCore http, string baseUrl) + : base(http, baseUrl) + { + } + + /// Creates a new webhook. + /// Any JSON-serializable webhook definition. + /// A token to cancel the request. + public async Task CreateAsync(object webhook, CancellationToken cancellationToken = default) + { + return new Webhook(await Ctx.CreateResourceAsync(new QueryParams(), webhook, cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/Apify.Client/Resources/WebhookDispatchClient.cs b/src/Apify.Client/Resources/WebhookDispatchClient.cs new file mode 100644 index 0000000..7c09d10 --- /dev/null +++ b/src/Apify.Client/Resources/WebhookDispatchClient.cs @@ -0,0 +1,26 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Internal; +using Apify.Client.Models; + +namespace Apify.Client.Resources; + +/// A client for a specific webhook dispatch (/v2/webhook-dispatches/{dispatchId}). +public sealed class WebhookDispatchClient +{ + private readonly ResourceContext _ctx; + + internal WebhookDispatchClient(HttpClientCore http, string baseUrl, string id) + { + _ctx = ResourceContext.Single(http, baseUrl, "webhook-dispatches", id); + } + + /// Fetches the dispatch, or null if it does not exist. + /// A token to cancel the request. + public async Task GetAsync(CancellationToken cancellationToken = default) + { + var data = await _ctx.GetResourceAsync("", new QueryParams(), cancellationToken).ConfigureAwait(false); + return data is JsonObject obj ? new WebhookDispatch(obj) : null; + } +} diff --git a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs new file mode 100644 index 0000000..0563698 --- /dev/null +++ b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs @@ -0,0 +1,31 @@ +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Internal; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Resources; + +/// +/// A client for a webhook dispatch collection: the account-wide collection +/// (GET /v2/webhook-dispatches) or dispatches nested under a webhook. +/// +public sealed class WebhookDispatchCollectionClient +{ + private readonly ResourceContext _ctx; + + internal WebhookDispatchCollectionClient(HttpClientCore http, string baseUrl, string resourcePath) + { + _ctx = ResourceContext.Collection(http, baseUrl, resourcePath); + } + + /// Lists webhook dispatches. + /// Optional pagination. + /// A token to cancel the request. + public Task> ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new ListOptions()).AppendTo(q); + return _ctx.ListResourceAsync("", q, static d => new WebhookDispatch(d), cancellationToken); + } +} diff --git a/tests/Apify.Client.Tests/Apify.Client.Tests.csproj b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj new file mode 100644 index 0000000..4c62faf --- /dev/null +++ b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj @@ -0,0 +1,23 @@ + + + + false + true + + false + false + + + + + + + + + + + + + + + diff --git a/tests/Apify.Client.Tests/Examples/CreateBuildRunActorExample.cs b/tests/Apify.Client.Tests/Examples/CreateBuildRunActorExample.cs new file mode 100644 index 0000000..93673ec --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/CreateBuildRunActorExample.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Apify.Client.Options; + +namespace Apify.Client.Tests.Examples; + +/// Create a new Actor, build it, run it, wait, and print the finished run log. +public static class CreateBuildRunActorExample +{ + public static async Task RunAsync(ApifyClient client) + { + var suffix = Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(4)).ToLowerInvariant(); + var created = await client.Actors().CreateAsync(new + { + name = "dotnet-example-actor-" + suffix, + isPublic = false, + versions = new[] + { + new + { + versionNumber = "0.0", + sourceType = "SOURCE_FILES", + buildTag = "latest", + sourceFiles = new object[] + { + new { name = "Dockerfile", format = "TEXT", content = "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" }, + new { name = "main.js", format = "TEXT", content = "console.log('hi');" }, + }, + }, + }, + }); + + try + { + var build = await client.Actor(created.Id!).BuildAsync("0.0", new ActorBuildOptions()); + await client.Build(build.Id!).WaitForFinishAsync(300); + var run = await client.Actor(created.Id!).CallAsync(null, null, 120); + var log = await client.Run(run.Id!).Log().GetAsync(); + if (log is not null) + { + Console.WriteLine(log); + } + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Examples/ExamplesTests.cs b/tests/Apify.Client.Tests/Examples/ExamplesTests.cs new file mode 100644 index 0000000..e7f2e07 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/ExamplesTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Xunit; + +namespace Apify.Client.Tests.Examples; + +/// +/// Runs each documentation example end-to-end against the live API, proving the snippets in the docs +/// actually work. Skipped when APIFY_TOKEN is not set. This is the "Test examples" CI step. +/// +[Trait("Category", "Examples")] +public sealed class ExamplesTests +{ + [SkippableFact] + public Task RunStoreActor() => RunStoreActorExample.RunAsync(Client()); + + [SkippableFact] + public Task Storages() => StoragesExample.RunAsync(Client()); + + [SkippableFact] + public Task GetAccount() => GetAccountExample.RunAsync(Client()); + + [SkippableFact] + public Task CreateBuildRunActor() => CreateBuildRunActorExample.RunAsync(Client()); + + [SkippableFact] + public Task RunAndLastRunStorages() => RunAndLastRunStoragesExample.RunAsync(Client()); + + [SkippableFact] + public Task IterateStore() => IterateStoreExample.RunAsync(Client()); + + [SkippableFact] + public Task LogRedirection() => LogRedirectionExample.RunAsync(Client()); + + private static ApifyClient Client() + { + var token = Environment.GetEnvironmentVariable("APIFY_TOKEN"); + Skip.If(string.IsNullOrEmpty(token), "skipping: APIFY_TOKEN is not set"); + + var apiUrl = Environment.GetEnvironmentVariable("APIFY_API_URL"); + var baseUrl = string.IsNullOrEmpty(apiUrl) ? ApifyClient.DefaultBaseUrl : apiUrl.TrimEnd('/'); + if (baseUrl.EndsWith("/v2", StringComparison.Ordinal)) + { + baseUrl = baseUrl[..^"/v2".Length]; + } + + return new ApifyClient(new ApifyClientOptions { Token = token, BaseUrl = baseUrl }); + } +} diff --git a/tests/Apify.Client.Tests/Examples/GetAccountExample.cs b/tests/Apify.Client.Tests/Examples/GetAccountExample.cs new file mode 100644 index 0000000..b507b00 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/GetAccountExample.cs @@ -0,0 +1,18 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; + +namespace Apify.Client.Tests.Examples; + +/// Get own account details. +public static class GetAccountExample +{ + public static async Task RunAsync(ApifyClient client) + { + var user = await client.Me().GetAsync(); + if (user is not null) + { + Console.WriteLine("Account " + user.Id + " / " + user.Username); + } + } +} diff --git a/tests/Apify.Client.Tests/Examples/IterateStoreExample.cs b/tests/Apify.Client.Tests/Examples/IterateStoreExample.cs new file mode 100644 index 0000000..a0ef650 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/IterateStoreExample.cs @@ -0,0 +1,23 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Apify.Client.Options; + +namespace Apify.Client.Tests.Examples; + +/// Lazy iteration of Store Actors using the convenience iterator. +public static class IterateStoreExample +{ + public static async Task RunAsync(ApifyClient client) + { + var shown = 0; + await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Limit = 10 })) + { + Console.WriteLine(item.Name); + if (++shown >= 5) + { + break; + } + } + } +} diff --git a/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs b/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs new file mode 100644 index 0000000..bcf359e --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs @@ -0,0 +1,20 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Apify.Client; + +namespace Apify.Client.Tests.Examples; + +/// Run an Actor with log redirection turned on (stream the run's log). +public static class LogRedirectionExample +{ + public static async Task RunAsync(ApifyClient client) + { + var run = await client.Actor("apify/hello-world").StartAsync(); + // Wait for the run to finish so the full log is available, then stream it to stdout. + await client.Run(run.Id!).WaitForFinishAsync(120); + using var stream = await client.Run(run.Id!).GetStreamedLogAsync(); + using var reader = new StreamReader(stream); + Console.WriteLine(await reader.ReadToEndAsync()); + } +} diff --git a/tests/Apify.Client.Tests/Examples/RunAndLastRunStoragesExample.cs b/tests/Apify.Client.Tests/Examples/RunAndLastRunStoragesExample.cs new file mode 100644 index 0000000..f709d21 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/RunAndLastRunStoragesExample.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Apify.Client.Options; + +namespace Apify.Client.Tests.Examples; + +/// Start a run, wait, then fetch the Actor's last run and its storages. +public static class RunAndLastRunStoragesExample +{ + public static async Task RunAsync(ApifyClient client) + { + await client.Actor("apify/hello-world").CallAsync(null, null, 120); + var last = await client.Actor("apify/hello-world").LastRun(new LastRunOptions { Status = "SUCCEEDED" }).GetAsync(); + if (last is not null) + { + await client.Dataset(last.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); + await client.KeyValueStore(last.DefaultKeyValueStoreId!).GetRecordAsync("OUTPUT"); + Console.WriteLine("Last run: " + last.Id); + } + } +} diff --git a/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs b/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs new file mode 100644 index 0000000..61d6db5 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs @@ -0,0 +1,17 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Apify.Client.Options; + +namespace Apify.Client.Tests.Examples; + +/// Run a store Actor and read its default dataset. +public static class RunStoreActorExample +{ + public static async Task RunAsync(ApifyClient client) + { + var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); + Console.WriteLine("Item count: " + items.Count); + } +} diff --git a/tests/Apify.Client.Tests/Examples/StoragesExample.cs b/tests/Apify.Client.Tests/Examples/StoragesExample.cs new file mode 100644 index 0000000..d410b2f --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/StoragesExample.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Apify.Client; +using Apify.Client.Models; +using Apify.Client.Options; + +namespace Apify.Client.Tests.Examples; + +/// Each storage: create, push data, read data back. +public static class StoragesExample +{ + public static async Task RunAsync(ApifyClient client) + { + // Dataset + var dataset = await client.Datasets().GetOrCreateAsync("dotnet-example-ds-" + Suffix()); + try + { + await client.Dataset(dataset.Id!).PushItemsAsync(new[] { new { hello = "world" } }); + var items = await client.Dataset(dataset.Id!).ListItemsAsync(new DatasetListItemsOptions()); + Console.WriteLine("Dataset items: " + items.Count); + } + finally + { + await client.Dataset(dataset.Id!).DeleteAsync(); + } + + // Key-value store + var store = await client.KeyValueStores().GetOrCreateAsync("dotnet-example-kvs-" + Suffix()); + try + { + await client.KeyValueStore(store.Id!).SetRecordJsonAsync("OUTPUT", new { answer = 42 }); + var record = await client.KeyValueStore(store.Id!).GetRecordAsync("OUTPUT"); + // Value is the raw bytes; decode JSON/text records via the reported content type. + var recordText = record is null ? string.Empty : System.Text.Encoding.UTF8.GetString(record.Value); + Console.WriteLine("KVS record: " + recordText); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + + // Request queue + var queue = await client.RequestQueues().GetOrCreateAsync("dotnet-example-rq-" + Suffix()); + try + { + await client.RequestQueue(queue.Id!).AddRequestAsync(new RequestQueueRequest("https://example.com", "example")); + var head = await client.RequestQueue(queue.Id!).ListHeadAsync(10); + Console.WriteLine("Queue head size: " + head.Items.Count); + } + finally + { + await client.RequestQueue(queue.Id!).DeleteAsync(); + } + } + + private static string Suffix() + => Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(4)).ToLowerInvariant(); +} diff --git a/tests/Apify.Client.Tests/Integration/ActorIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorIntegrationTests.cs new file mode 100644 index 0000000..3c695ca --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/ActorIntegrationTests.cs @@ -0,0 +1,119 @@ +using System.Threading.Tasks; +using Apify.Client.Models; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class ActorIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListActors() + { + var client = RequireClient(); + var page = await client.Actors().ListAsync(new ActorListOptions { My = true, Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetActor() + { + var client = RequireClient(); + var created = await client.Actors().CreateAsync(MinimalActor(UniqueName("get"))); + try + { + var got = await client.Actor(created.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(created.Id, got!.Id); + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ActorCrudFlow() + { + var client = RequireClient(); + var created = await client.Actors().CreateAsync(MinimalActor(UniqueName("crud"))); + try + { + var actor = client.Actor(created.Id!); + Assert.NotNull(await actor.GetAsync()); + var updated = await actor.UpdateAsync(new { title = "Updated Title" }); + Assert.Equal("Updated Title", updated.Title); + await actor.Builds().ListAsync(new ListOptions()); + await actor.Versions().ListAsync(new ListOptions()); + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ActorVersionCrudFlow() + { + var client = RequireClient(); + var created = await client.Actors().CreateAsync(MinimalActor(UniqueName("ver"))); + try + { + var actor = client.Actor(created.Id!); + var version = await actor.Versions().CreateAsync(new + { + versionNumber = "0.1", + sourceType = "SOURCE_FILES", + buildTag = "latest", + sourceFiles = System.Array.Empty(), + }); + Assert.Equal("0.1", version.VersionNumber); + Assert.NotNull(await actor.Version("0.1").GetAsync()); + await actor.Versions().ListAsync(new ListOptions()); + await actor.Version("0.1").UpdateAsync(new + { + buildTag = "beta", + sourceType = "SOURCE_FILES", + sourceFiles = System.Array.Empty(), + }); + await actor.Version("0.1").DeleteAsync(); + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ValidateInput() + { + var client = RequireClient(); + // apify/hello-world is a public store Actor; validate-input is read-only and returns + // {"valid": }. A well-formed input validates true. + Assert.True(await client.Actor("apify/hello-world").ValidateInputAsync(new { firstNumber = 1 })); + } + + [SkippableFact] + public async Task ActorEnvVarCrudFlow() + { + var client = RequireClient(); + var created = await client.Actors().CreateAsync(MinimalActor(UniqueName("env"))); + try + { + var actor = client.Actor(created.Id!); + var envVars = actor.Version("0.0").EnvVars(); + await envVars.CreateAsync(new ActorEnvVar("MY_VAR", "value1")); + Assert.NotNull(await actor.Version("0.0").EnvVar("MY_VAR").GetAsync()); + await envVars.ListAsync(); + await actor.Version("0.0").EnvVar("MY_VAR").UpdateAsync(new ActorEnvVar("MY_VAR", "value2")); + await actor.Version("0.0").EnvVar("MY_VAR").DeleteAsync(); + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs new file mode 100644 index 0000000..ceb84c2 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -0,0 +1,53 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class ActorRunIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListRuns() + { + var client = RequireClient(); + var page = await client.Runs().ListAsync(new ListOptions { Limit = 5 }, new RunListOptions()); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task RunActorAndReadOutputs() + { + var client = RequireClient(); + var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + Assert.Equal("SUCCEEDED", run.Status); + + Assert.NotNull(await client.Run(run.Id!).GetAsync()); + + var log = await client.Run(run.Id!).Log().GetAsync(); + Assert.NotNull(log); + Assert.NotEqual(string.Empty, log); + + await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions()); + await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); + } + + [SkippableFact] + public async Task LastRunAccess() + { + var client = RequireClient(); + await client.Actor("apify/hello-world").CallAsync(null, null, 120); + + var lastRun = await client.Actor("apify/hello-world").LastRun(new LastRunOptions { Status = "SUCCEEDED" }).GetAsync(); + Assert.NotNull(lastRun); + Assert.Equal("SUCCEEDED", lastRun!.Status); + + var byOrigin = await client.Actor("apify/hello-world") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .GetAsync(); + Assert.NotNull(byOrigin); + Assert.Equal("SUCCEEDED", byOrigin!.Status); + } +} diff --git a/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs new file mode 100644 index 0000000..aa8e2ae --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs @@ -0,0 +1,40 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class BuildIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListBuilds() + { + var client = RequireClient(); + var page = await client.Builds().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task BuildActorFlow() + { + var client = RequireClient(); + var created = await client.Actors().CreateAsync(MinimalActor(UniqueName("build"))); + try + { + var build = await client.Actor(created.Id!).BuildAsync("0.0", new ActorBuildOptions()); + var finished = await client.Build(build.Id!).WaitForFinishAsync(300); + Assert.True(finished.IsTerminal, "build did not finish: " + finished.Status); + + Assert.NotNull(await client.Build(build.Id!).GetAsync()); + await client.Build(build.Id!).Log().GetAsync(); + await client.Build(build.Id!).GetOpenApiDefinitionAsync(); + } + finally + { + await client.Actor(created.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs new file mode 100644 index 0000000..e73473c --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs @@ -0,0 +1,82 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class DatasetIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListDatasets() + { + var client = RequireClient(); + var page = await client.Datasets().ListAsync(new StorageListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetDataset() + { + var client = RequireClient(); + var ds = await client.Datasets().GetOrCreateAsync(UniqueName("ds-get")); + try + { + var got = await client.Dataset(ds.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(ds.Id, got!.Id); + } + finally + { + await client.Dataset(ds.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task DatasetCrudFlow() + { + var client = RequireClient(); + var ds = await client.Datasets().GetOrCreateAsync(UniqueName("ds-crud")); + try + { + var dataset = client.Dataset(ds.Id!); + Assert.NotNull(await dataset.GetAsync()); + + await dataset.PushItemsAsync(new object[] + { + new { url = "https://a.com", n = 1 }, + new { url = "https://b.com", n = 2 }, + new { url = "https://c.com", n = 3 }, + }); + + var page = await dataset.ListItemsAsync(new DatasetListItemsOptions()); + Assert.Equal(3, (int)page.Count); + Assert.Equal(3, page.Items.Count); + Assert.Equal(1, page.Items[0]!["n"]!.GetValue()); + + var csvBytes = await dataset.DownloadItemsAsync(DownloadItemsFormat.Csv, new DatasetDownloadOptions { Bom = true }); + Assert.NotEmpty(csvBytes); + Assert.Contains("url", System.Text.Encoding.UTF8.GetString(csvBytes), System.StringComparison.Ordinal); + + // XLSX is a binary (ZIP-based) export; verify the raw bytes are returned uncorrupted by checking + // the ZIP local-file-header magic (PK\x03\x04). A string-based download would mangle these bytes. + var xlsxBytes = await dataset.DownloadItemsAsync(DownloadItemsFormat.Xlsx); + Assert.True(xlsxBytes.Length >= 4, "expected non-empty XLSX bytes"); + Assert.Equal(new byte[] { 0x50, 0x4B, 0x03, 0x04 }, xlsxBytes[..4]); + + var url = await dataset.CreateItemsPublicUrlAsync(new DatasetListItemsOptions()); + Assert.NotEqual(string.Empty, url); + + await dataset.GetStatisticsAsync(); + + var updated = await dataset.UpdateAsync(new { name = UniqueName("ds-renamed") }); + Assert.False(string.IsNullOrEmpty(updated.Name)); + } + finally + { + await client.Dataset(ds.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs new file mode 100644 index 0000000..f8fa16b --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs @@ -0,0 +1,82 @@ +using System; +using System.Security.Cryptography; +using Apify.Client; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +/// +/// Shared setup for the integration test suite. +/// +/// +/// All integration tests require a valid APIFY_TOKEN for the test account. The API base URL is +/// taken from APIFY_API_URL (which includes the /v2 suffix) and falls back to +/// https://api.apify.com/v2. Tests are designed to run concurrently — including against the same +/// test account from several language clients at once — so every test creates uniquely-named resources +/// and cleans them up. +/// +public abstract class IntegrationTestBase +{ + /// The integration-test contract fallback base URL. + private const string DefaultApiUrl = "https://api.apify.com/v2"; + + /// + /// Derives the client base URL from an optional APIFY_API_URL. The variable includes the + /// /v2 suffix (per the integration-test contract) and falls back to the default. Since the client + /// appends /v2 itself, the suffix is stripped here. + /// + protected static string ResolveBaseUrl(string? apiUrl) + { + if (string.IsNullOrEmpty(apiUrl)) + { + apiUrl = DefaultApiUrl; + } + + var trimmed = apiUrl.TrimEnd('/'); + if (trimmed.EndsWith("/v2", StringComparison.Ordinal)) + { + trimmed = trimmed[..^"/v2".Length]; + } + + return trimmed; + } + + /// Returns a configured client, or skips the test if APIFY_TOKEN is unset. + protected static ApifyClient RequireClient() + { + var token = Environment.GetEnvironmentVariable("APIFY_TOKEN"); + Skip.If(string.IsNullOrEmpty(token), "skipping: APIFY_TOKEN is not set"); + + var apiUrl = Environment.GetEnvironmentVariable("APIFY_API_URL"); + return new ApifyClient(new ApifyClientOptions { Token = token, BaseUrl = ResolveBaseUrl(apiUrl) }); + } + + /// + /// Generates a collision-resistant resource name for test isolation. The random component lets the same + /// test run in parallel (across processes and languages) without clobbering shared state. + /// + protected static string UniqueName(string prefix) + => "dotnet-test-" + prefix + "-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(6)).ToLowerInvariant(); + + /// A minimal Actor definition; the API requires at least one version. + protected static object MinimalActor(string name) => new + { + name, + isPublic = false, + description = "Integration test actor", + versions = new[] + { + new + { + versionNumber = "0.0", + sourceType = "SOURCE_FILES", + buildTag = "latest", + sourceFiles = new object[] + { + new { name = "Dockerfile", format = "TEXT", content = "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" }, + new { name = "main.js", format = "TEXT", content = "console.log('hello from dotnet client test');" }, + }, + }, + }, + }; +} diff --git a/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs new file mode 100644 index 0000000..b86c0b3 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class KeyValueStoreIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListKeyValueStores() + { + var client = RequireClient(); + var page = await client.KeyValueStores().ListAsync(new StorageListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetKeyValueStore() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-get")); + try + { + var got = await client.KeyValueStore(store.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(store.Id, got!.Id); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RecordKeyWithSpecialChars() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-special")); + try + { + var kvs = client.KeyValueStore(store.Id!); + const string key = "weird-key!'()"; + await kvs.SetRecordJsonAsync(key, new { ok = true }); + Assert.True(await kvs.RecordExistsAsync(key)); + Assert.NotNull(await kvs.GetRecordAsync(key)); + await kvs.DeleteRecordAsync(key); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task KeyValueStoreCrudFlow() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-crud")); + try + { + var kvs = client.KeyValueStore(store.Id!); + Assert.NotNull(await kvs.GetAsync()); + await kvs.SetRecordJsonAsync("OUTPUT", new { hello = "world" }); + Assert.True(await kvs.RecordExistsAsync("OUTPUT")); + var record = await kvs.GetRecordAsync("OUTPUT"); + Assert.NotNull(record); + Assert.Contains("world", System.Text.Encoding.UTF8.GetString(record!.Value), StringComparison.Ordinal); + await kvs.GetRecordAsync("OUTPUT", new GetRecordOptions { Attachment = false }); + var keys = await kvs.ListKeysAsync(new ListKeysOptions()); + Assert.NotEmpty(keys.Items); + await kvs.UpdateAsync(new { name = UniqueName("kvs-renamed") }); + await kvs.DeleteRecordAsync("OUTPUT"); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task BinaryRecordRoundTripPreservesBytes() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-binary")); + try + { + var kvs = client.KeyValueStore(store.Id!); + // Bytes that are NOT valid UTF-8 (0xFF, 0xFE, 0x00) — a string-based read would corrupt these. + var payload = new byte[] { 0x00, 0xFF, 0xFE, 0x01, 0x80, 0x7F }; + await kvs.SetRecordAsync("binary", payload, "application/octet-stream"); + + var record = await kvs.GetRecordAsync("binary"); + Assert.NotNull(record); + Assert.Equal(payload, record!.Value); + Assert.Equal((byte)0xFF, record.Value[1]); + + await kvs.DeleteRecordAsync("binary"); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RecordPublicUrlIsFetchable() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-pub")); + try + { + var kvs = client.KeyValueStore(store.Id!); + await kvs.SetRecordJsonAsync("OUTPUT", new { pub = true }); + var url = await kvs.GetRecordPublicUrlAsync("OUTPUT"); + Assert.NotEqual(string.Empty, url); + + using var http = new HttpClient(); + using var response = await http.GetAsync(new Uri(url)); + Assert.True((int)response.StatusCode < 300, "expected success fetching public url"); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs new file mode 100644 index 0000000..3106671 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs @@ -0,0 +1,146 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Apify.Client.Models; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class RequestQueueIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListRequestQueues() + { + var client = RequireClient(); + var page = await client.RequestQueues().ListAsync(new StorageListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetRequestQueue() + { + var client = RequireClient(); + var rq = await client.RequestQueues().GetOrCreateAsync(UniqueName("rq-get")); + try + { + var got = await client.RequestQueue(rq.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(rq.Id, got!.Id); + } + finally + { + await client.RequestQueue(rq.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RequestQueueCrudFlow() + { + var client = RequireClient(); + var rq = await client.RequestQueues().GetOrCreateAsync(UniqueName("rq-crud")); + try + { + var queue = client.RequestQueue(rq.Id!); + Assert.NotNull(await queue.GetAsync()); + + var request = new RequestQueueRequest("https://example.com", "example") { Method = "GET" }; + var info = await queue.AddRequestAsync(request); + Assert.False(string.IsNullOrEmpty(info.RequestId)); + + var got = await queue.GetRequestAsync(info.RequestId!); + Assert.NotNull(got); + Assert.Equal("https://example.com", got!.Url); + + Assert.NotEmpty((await queue.ListHeadAsync(10)).Items); + await queue.UpdateAsync(new { name = UniqueName("rq-renamed") }); + await queue.DeleteRequestAsync(info.RequestId!); + } + finally + { + await client.RequestQueue(rq.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RequestQueuePaginateMultiplePages() + { + var client = RequireClient(); + var rq = await client.RequestQueues().GetOrCreateAsync(UniqueName("rq-page")); + try + { + var queue = client.RequestQueue(rq.Id!); + const int total = 5; + for (var i = 0; i < total; i++) + { + var url = "https://example.com/" + i; + await queue.AddRequestAsync(new RequestQueueRequest(url, url)); + } + + var seen = new HashSet(); + await foreach (var request in queue.PaginateRequestsAsync(new PaginateRequestsOptions { MaxPageLimit = 2 })) + { + seen.Add(request.Url!); + } + + Assert.Equal(total, seen.Count); + } + finally + { + await client.RequestQueue(rq.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RequestQueueBatchAddRequests() + { + var client = RequireClient(); + var rq = await client.RequestQueues().GetOrCreateAsync(UniqueName("rq-batch")); + try + { + var queue = client.RequestQueue(rq.Id!); + const int total = 30; // > 25, so the client must split into multiple chunks + var requests = new List(); + for (var i = 0; i < total; i++) + { + var url = "https://batch.example.com/" + i; + requests.Add(new RequestQueueRequest(url, url)); + } + + var result = await queue.BatchAddRequestsAsync(requests); + Assert.Equal(total, result.ProcessedRequests.Count); + Assert.Empty(result.UnprocessedRequests); + } + finally + { + await client.RequestQueue(rq.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task RequestQueueLockLifecycle() + { + var client = RequireClient(); + var rq = await client.RequestQueues().GetOrCreateAsync(UniqueName("rq-lock")); + try + { + var queue = client.RequestQueue(rq.Id!).WithClientKey("dotnet-test-client-key"); + var info = await queue.AddRequestAsync(new RequestQueueRequest("https://lock.example.com", "lock")); + Assert.True((await queue.ListRequestsAsync(new ListRequestsOptions())).ContainsKey("items")); + await queue.ListRequestsAsync(new ListRequestsOptions + { + Filter = new[] { ListRequestsOptions.FilterLocked, ListRequestsOptions.FilterPending }, + }); + Assert.True((await queue.ListAndLockHeadAsync(60, 10)).ContainsKey("items")); + await queue.ProlongRequestLockAsync(info.RequestId!, 30); + await queue.DeleteRequestLockAsync(info.RequestId!); + await queue.UnlockRequestsAsync(); + } + finally + { + await client.RequestQueue(rq.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/ScheduleIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ScheduleIntegrationTests.cs new file mode 100644 index 0000000..1759940 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/ScheduleIntegrationTests.cs @@ -0,0 +1,65 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class ScheduleIntegrationTests : IntegrationTestBase +{ + private static object ScheduleDef(string name) => new + { + name, + cronExpression = "0 0 * * *", + isEnabled = false, + isExclusive = true, + actions = System.Array.Empty(), + }; + + [SkippableFact] + public async Task ListSchedules() + { + var client = RequireClient(); + var page = await client.Schedules().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetSchedule() + { + var client = RequireClient(); + var sch = await client.Schedules().CreateAsync(ScheduleDef(UniqueName("sch-get"))); + try + { + var got = await client.Schedule(sch.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(sch.Id, got!.Id); + } + finally + { + await client.Schedule(sch.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ScheduleCrudFlow() + { + var client = RequireClient(); + var sch = await client.Schedules().CreateAsync(ScheduleDef(UniqueName("sch-crud"))); + try + { + var schedule = client.Schedule(sch.Id!); + Assert.NotNull(await schedule.GetAsync()); + var updated = await schedule.UpdateAsync(new { cronExpression = "0 12 * * *" }); + Assert.Equal("0 12 * * *", updated.CronExpression); + // A fresh schedule may have no log yet (null), which is a valid result — we only assert the call succeeds. + await schedule.GetLogAsync(); + } + finally + { + await client.Schedule(sch.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs new file mode 100644 index 0000000..e9a2727 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs @@ -0,0 +1,34 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class StoreIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task ListStore() + { + var client = RequireClient(); + var page = await client.Store().ListAsync(new StoreListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + } + + [SkippableFact] + public async Task IterateStore() + { + var client = RequireClient(); + var count = 0; + await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Limit = 5 })) + { + Assert.False(string.IsNullOrEmpty(item.Id)); + if (++count >= 12) + { + break; + } + } + + Assert.True(count >= 12, "expected to iterate at least 12 store actors"); + } +} diff --git a/tests/Apify.Client.Tests/Integration/TaskIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/TaskIntegrationTests.cs new file mode 100644 index 0000000..b632b22 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/TaskIntegrationTests.cs @@ -0,0 +1,64 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class TaskIntegrationTests : IntegrationTestBase +{ + private static object TaskDef(string name) => new + { + actId = "apify/hello-world", + name, + options = new { build = "latest", memoryMbytes = 256, timeoutSecs = 60 }, + input = new { message = "hello" }, + }; + + [SkippableFact] + public async Task ListTasks() + { + var client = RequireClient(); + var page = await client.Tasks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetTask() + { + var client = RequireClient(); + var task = await client.Tasks().CreateAsync(TaskDef(UniqueName("task-get"))); + try + { + var got = await client.Task(task.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(task.Id, got!.Id); + } + finally + { + await client.Task(task.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task TaskCrudFlow() + { + var client = RequireClient(); + var task = await client.Tasks().CreateAsync(TaskDef(UniqueName("task-crud"))); + try + { + var tc = client.Task(task.Id!); + Assert.NotNull(await tc.GetAsync()); + await tc.UpdateInputAsync(new { message = "updated" }); + Assert.NotNull(await tc.GetInputAsync()); + await tc.UpdateAsync(new { name = UniqueName("task-renamed") }); + await tc.Runs().ListAsync(new ListOptions(), new RunListOptions()); + } + finally + { + await client.Task(task.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs new file mode 100644 index 0000000..df32cf6 --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs @@ -0,0 +1,38 @@ +using System.Threading.Tasks; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class UserIntegrationTests : IntegrationTestBase +{ + [SkippableFact] + public async Task GetOwnAccount() + { + var client = RequireClient(); + var user = await client.Me().GetAsync(); + Assert.NotNull(user); + Assert.False(string.IsNullOrEmpty(user!.Id)); + } + + [SkippableFact] + public async Task GetMonthlyUsage() + { + var client = RequireClient(); + Assert.NotEmpty(await client.Me().MonthlyUsageAsync()); + } + + [SkippableFact] + public async Task GetMonthlyUsageForDate() + { + var client = RequireClient(); + Assert.NotEmpty(await client.Me().MonthlyUsageAsync("2026-06-01")); + } + + [SkippableFact] + public async Task GetLimits() + { + var client = RequireClient(); + Assert.NotEmpty(await client.Me().LimitsAsync()); + } +} diff --git a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs new file mode 100644 index 0000000..9eabb2b --- /dev/null +++ b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs @@ -0,0 +1,92 @@ +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Integration; + +[Trait("Category", "Integration")] +public sealed class WebhookIntegrationTests : IntegrationTestBase +{ + private static object WebhookDef(string url) => new + { + isAdHoc = true, + eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, + condition = new { actorRunId = "ZZZZZZZZZZZZZZZZZ" }, + requestUrl = url, + }; + + [SkippableFact] + public async Task ListWebhooks() + { + var client = RequireClient(); + var page = await client.Webhooks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task ListWebhookDispatches() + { + var client = RequireClient(); + var page = await client.WebhookDispatches().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + + [SkippableFact] + public async Task GetWebhook() + { + var client = RequireClient(); + var wh = await client.Webhooks().CreateAsync(WebhookDef("https://example.com/webhook")); + try + { + var got = await client.Webhook(wh.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(wh.Id, got!.Id); + } + finally + { + await client.Webhook(wh.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task GetWebhookDispatch() + { + var client = RequireClient(); + var wh = await client.Webhooks().CreateAsync(WebhookDef("https://example.com/webhook")); + try + { + var dispatch = await client.Webhook(wh.Id!).TestAsync(); + var got = await client.WebhookDispatch(dispatch.Id!).GetAsync(); + Assert.NotNull(got); + Assert.Equal(dispatch.Id, got!.Id); + } + finally + { + await client.Webhook(wh.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task WebhookCrudFlow() + { + var client = RequireClient(); + var wh = await client.Webhooks().CreateAsync(WebhookDef("https://example.com/webhook")); + try + { + var webhook = client.Webhook(wh.Id!); + Assert.NotNull(await webhook.GetAsync()); + var updated = await webhook.UpdateAsync(new { requestUrl = "https://example.com/updated" }); + Assert.Equal("https://example.com/updated", updated.RequestUrl); + await webhook.Dispatches().ListAsync(new ListOptions()); + await webhook.TestAsync(); + } + finally + { + await client.Webhook(wh.Id!).DeleteAsync(); + } + } +} diff --git a/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs b/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs new file mode 100644 index 0000000..5d95a5d --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs @@ -0,0 +1,251 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Models; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Offline behavioral tests for : +/// uniqueKey validation, count/byte chunking, unprocessed-retry from a successful response, and the +/// non-throwing error contract, matching the JS reference. +/// +[Trait("Category", "Unit")] +public sealed class BatchAddRequestsTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + /// + /// No-delay, sequential options so retry/chunking tests that queue ordered responses stay + /// deterministic (parallel dispatch is covered separately). + /// + private static BatchAddRequestsOptions FastOptions(int maxRetries = 3) => + new(maxUnprocessedRequestsRetries: maxRetries, maxParallel: 1, minDelayBetweenUnprocessedRequestsRetriesMillis: 0); + + private static string BatchResponse(IEnumerable uniqueKeys, IEnumerable? unprocessedKeys = null) + { + var processed = new JsonArray(); + foreach (var k in uniqueKeys) + { + processed.Add(new JsonObject + { + ["uniqueKey"] = k, + ["requestId"] = "id-" + k, + ["wasAlreadyPresent"] = false, + ["wasAlreadyHandled"] = false, + }); + } + + var unprocessed = new JsonArray(); + foreach (var k in unprocessedKeys ?? Array.Empty()) + { + unprocessed.Add(new JsonObject { ["uniqueKey"] = k, ["url"] = "https://x/" + k, ["method"] = "GET" }); + } + + return new JsonObject + { + ["data"] = new JsonObject { ["processedRequests"] = processed, ["unprocessedRequests"] = unprocessed }, + }.ToJsonString(); + } + + [Fact] + public async Task MissingUniqueKeyThrowsBeforeAnyCall() + { + var transport = new MockTransport(); + var requests = new List { new("https://a.com") }; + + await Assert.ThrowsAsync(() => Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests)); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task ApiErrorReportedAsUnprocessedNotThrown() + { + var transport = new MockTransport().QueueResponse(403, "{\"error\":{\"type\":\"insufficient-permissions\",\"message\":\"nope\"}}"); + var requests = new List { new("https://a.com", "a") }; + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions()); + + Assert.Empty(result.ProcessedRequests); + Assert.Single(result.UnprocessedRequests); + Assert.Equal("a", result.UnprocessedRequests[0].UniqueKey); + } + + [Fact] + public async Task MultiChunkPreservesEarlierChunksWhenLaterChunkFails() + { + var keys = new List(); + for (var i = 0; i < 30; i++) + { + keys.Add("u" + i); + } + + var transport = new MockTransport() + .QueueResponse(200, BatchResponse(keys.GetRange(0, 25))) + .QueueResponse(403, "{\"error\":{\"type\":\"x\",\"message\":\"boom\"}}"); + var requests = keys.ConvertAll(k => new RequestQueueRequest("https://x/" + k, k)); + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions()); + + Assert.Equal(25, result.ProcessedRequests.Count); + Assert.Equal(5, result.UnprocessedRequests.Count); + } + + [Fact] + public async Task RetriesOnlyUnprocessedFromSuccessfulResponse() + { + var transport = new MockTransport() + .QueueResponse(200, BatchResponse(new[] { "r0" }, new[] { "r1" })) + .QueueResponse(200, BatchResponse(new[] { "r1" })); + var requests = new List { new("https://a.com", "r0"), new("https://b.com", "r1") }; + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions()); + + Assert.Equal(2, transport.CallCount); + Assert.Equal(2, result.ProcessedRequests.Count); + Assert.Empty(result.UnprocessedRequests); + + // The retry must send only the still-unprocessed request (r1), not the whole batch again. + var retryBody = JsonNode.Parse(transport.Received[1].Body)!.AsArray(); + Assert.Single(retryBody); + Assert.Equal("r1", retryBody[0]!["uniqueKey"]!.GetValue()); + } + + [Fact] + public async Task UnprocessedReportedAfterRetriesExhausted() + { + var transport = new MockTransport(); + for (var i = 0; i < 3; i++) + { + transport.QueueResponse(200, BatchResponse(Array.Empty(), new[] { "r0" })); + } + + var requests = new List { new("https://a.com", "r0") }; + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions(2)); + + Assert.Equal(3, transport.CallCount); // 1 + 2 retries + Assert.Empty(result.ProcessedRequests); + Assert.Single(result.UnprocessedRequests); + Assert.Equal("r0", result.UnprocessedRequests[0].UniqueKey); + } + + [Fact] + public async Task ChunksByCountLimit() + { + var keys = new List(); + for (var i = 0; i < 30; i++) + { + keys.Add("u" + i); + } + + var transport = new MockTransport() + .QueueResponse(200, BatchResponse(keys.GetRange(0, 25))) + .QueueResponse(200, BatchResponse(keys.GetRange(25, 5))); + var requests = keys.ConvertAll(k => new RequestQueueRequest("https://x/" + k, k)); + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions()); + + Assert.Equal(2, transport.CallCount); + Assert.Equal(30, result.ProcessedRequests.Count); + Assert.Equal(25, JsonNode.Parse(transport.Received[0].Body)!.AsArray().Count); + } + + [Fact] + public async Task ChunksByPayloadByteSize() + { + var big = new string('x', 4 * 1024 * 1024); + var keys = new[] { "b0", "b1", "b2" }; + var transport = new MockTransport() + .QueueResponse(200, BatchResponse(new[] { "b0", "b1" })) + .QueueResponse(200, BatchResponse(new[] { "b2" })); + var requests = new List(); + foreach (var k in keys) + { + requests.Add(new RequestQueueRequest("https://x/" + k, k) { UserData = new JsonObject { ["blob"] = big } }); + } + + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, FastOptions()); + + Assert.Equal(2, transport.CallCount); + Assert.Equal(3, result.ProcessedRequests.Count); + Assert.Equal(2, JsonNode.Parse(transport.Received[0].Body)!.AsArray().Count); // byte limit, not the count limit + } + + [Fact] + public async Task DispatchesChunksWithBoundedParallelism() + { + var transport = new MockTransport { EchoBatchProcessed = true, ArtificialDelayMillis = 40 }; + var requests = new List(); + for (var i = 0; i < 100; i++) // 100 requests -> 4 chunks of 25 + { + requests.Add(new RequestQueueRequest("https://x/" + i, "k" + i)); + } + + var options = new BatchAddRequestsOptions(maxParallel: 4, minDelayBetweenUnprocessedRequestsRetriesMillis: 0); + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, options); + + Assert.Equal(100, result.ProcessedRequests.Count); + Assert.Empty(result.UnprocessedRequests); + Assert.Equal(4, transport.CallCount); + // With 4 chunks and maxParallel=4 the calls must overlap; strictly sequential dispatch would be 1. + Assert.True(transport.MaxObservedConcurrency > 1, "expected concurrent batch calls"); + } + + [Fact] + public async Task MaxParallelOneKeepsDispatchSequential() + { + var transport = new MockTransport { EchoBatchProcessed = true, ArtificialDelayMillis = 20 }; + var requests = new List(); + for (var i = 0; i < 75; i++) // 75 requests -> 3 chunks of 25 + { + requests.Add(new RequestQueueRequest("https://x/" + i, "k" + i)); + } + + var options = new BatchAddRequestsOptions(maxParallel: 1, minDelayBetweenUnprocessedRequestsRetriesMillis: 0); + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, options); + + Assert.Equal(75, result.ProcessedRequests.Count); + Assert.Equal(3, transport.CallCount); + Assert.Equal(1, transport.MaxObservedConcurrency); + } + + [Fact] + public async Task ParallelResultsMergedInInputOrder() + { + var transport = new MockTransport { EchoBatchProcessed = true, ArtificialDelayMillis = 30 }; + var requests = new List(); + for (var i = 0; i < 60; i++) // 60 requests -> 3 chunks (0..24, 25..49, 50..59) + { + requests.Add(new RequestQueueRequest("https://x/" + i, "k" + i)); + } + + var options = new BatchAddRequestsOptions(maxParallel: 3, minDelayBetweenUnprocessedRequestsRetriesMillis: 0); + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, options); + + Assert.Equal(60, result.ProcessedRequests.Count); + // Merge order must follow input order regardless of which chunk finished first. + for (var i = 0; i < 60; i++) + { + Assert.Equal("k" + i, result.ProcessedRequests[i].UniqueKey); + } + } + + [Fact] + public async Task OversizedSingleRequestThrows() + { + var huge = new string('x', 10 * 1024 * 1024); // > 9 MiB on its own + var requests = new List { new("https://a.com", "big") { UserData = new JsonObject { ["blob"] = huge } } }; + + await Assert.ThrowsAsync(() => Client(new MockTransport()).RequestQueue("q1").BatchAddRequestsAsync(requests)); + } +} diff --git a/tests/Apify.Client.Tests/Unit/ConfigTests.cs b/tests/Apify.Client.Tests/Unit/ConfigTests.cs new file mode 100644 index 0000000..f5acf81 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ConfigTests.cs @@ -0,0 +1,53 @@ +using System.Text.RegularExpressions; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class ConfigTests +{ + [Fact] + public void UserAgentFormat() + { + var client = new ApifyClient(new ApifyClientOptions + { + Token = "test-token", + HttpTransport = new MockTransport(), + IsAtHome = () => false, + }); + + var ua = client.UserAgent; + Assert.StartsWith("ApifyClient/" + ApifyClientVersion.ClientVersion, ua, System.StringComparison.Ordinal); + Assert.Contains("; .NET/", ua, System.StringComparison.Ordinal); + Assert.EndsWith("isAtHome/false", ua, System.StringComparison.Ordinal); + } + + [Fact] + public void UserAgentIsAtHomeTrueAndSuffix() + { + var client = new ApifyClient(new ApifyClientOptions + { + Token = "t", + UserAgentSuffix = "my-suffix", + HttpTransport = new MockTransport(), + IsAtHome = () => true, + }); + + Assert.Contains("isAtHome/true", client.UserAgent, System.StringComparison.Ordinal); + Assert.EndsWith("; my-suffix", client.UserAgent, System.StringComparison.Ordinal); + } + + [Fact] + public void ApiBaseUrlAppendsV2() + { + var client = new ApifyClient(new ApifyClientOptions { Token = "t", BaseUrl = "https://api.example.com/", HttpTransport = new MockTransport() }); + Assert.Equal("https://api.example.com/v2", client.ApiBaseUrl); + } + + [Fact] + public void VersionConstants() + { + Assert.Matches(new Regex(@"^\d+\.\d+\.\d+$"), ApifyClientVersion.ClientVersion); + Assert.StartsWith("v2-", ApifyClientVersion.ApiSpecVersion, System.StringComparison.Ordinal); + } +} diff --git a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs new file mode 100644 index 0000000..41f8aa0 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs @@ -0,0 +1,156 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class HttpClientTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "test-token", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task AuthAndUserAgentHeadersAreSent() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"abc\"}}"); + await Client(transport).Actor("abc").GetAsync(); + + Assert.Equal("Bearer test-token", transport.LastRequest.Header("Authorization")); + Assert.StartsWith("ApifyClient/", transport.LastRequest.Header("User-Agent"), StringComparison.Ordinal); + } + + [Fact] + public async Task DataEnvelopeIsUnwrapped() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"act1\",\"name\":\"my-actor\"}}"); + var actor = await Client(transport).Actor("act1").GetAsync(); + + Assert.NotNull(actor); + Assert.Equal("act1", actor!.Id); + Assert.Equal("my-actor", actor.Name); + } + + [Fact] + public async Task NotFoundReturnsNull() + { + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"not here\"}}"); + Assert.Null(await Client(transport).Actor("missing").GetAsync()); + } + + [Fact] + public async Task ServerErrorsAreRetriedThenSucceed() + { + var transport = new MockTransport() + .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}") + .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + + var actor = await Client(transport).Actor("x").GetAsync(); + Assert.Equal("ok", actor!.Id); + Assert.Equal(2, transport.CallCount); + } + + [Fact] + public async Task ValidationErrorIsNotRetriedAndThrows() + { + var transport = new MockTransport().QueueResponse(400, "{\"error\":{\"type\":\"bad-input\",\"message\":\"invalid\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + Assert.Equal(400, ex.StatusCode); + Assert.Equal("bad-input", ex.Type); + Assert.Contains("invalid", ex.ApiMessage, StringComparison.Ordinal); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task TransportErrorsAreRetried() + { + var transport = new MockTransport() + .QueueError() + .QueueResponse(200, "{\"data\":{\"id\":\"recovered\"}}"); + var actor = await Client(transport).Actor("x").GetAsync(); + Assert.Equal("recovered", actor!.Id); + Assert.Equal(2, transport.CallCount); + } + + [Fact] + public async Task BooleanQueryParamsEncodedAsOneZero() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + await Client(transport).Actors().ListAsync(new ActorListOptions { My = true, Limit = 5 }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("my=1", uri, StringComparison.Ordinal); + Assert.Contains("limit=5", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task ListUnwrapsPaginationEnvelope() + { + const string body = "{\"data\":{\"total\":2,\"offset\":0,\"limit\":10,\"count\":2,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; + var transport = new MockTransport().QueueResponse(200, body); + var page = await Client(transport).Actors().ListAsync(); + + Assert.Equal(2, page.Total); + Assert.Equal(2, page.Items.Count); + Assert.Equal("a", page.Items[0].Id); + } + + [Fact] + public async Task PaginationCountReflectsItemsNotServerMetadata() + { + // Server reports total/count larger than the items actually returned in this page. Count and the + // indexer must operate on the item array so `for (i < Count) page[i]` cannot throw; Total keeps the + // API's reported total. + const string body = "{\"data\":{\"total\":100,\"offset\":0,\"limit\":2,\"count\":100,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; + var transport = new MockTransport().QueueResponse(200, body); + var page = await Client(transport).Actors().ListAsync(); + + Assert.Equal(100, page.Total); + Assert.Equal(2, page.Count); + Assert.Equal(2, page.Items.Count); + for (var i = 0; i < page.Count; i++) + { + Assert.NotNull(page[i]); // must not throw IndexOutOfRange + } + } + + [Fact] + public async Task DatasetItemsUseHeaderPagination() + { + var headers = new System.Collections.Generic.Dictionary + { + ["X-Apify-Pagination-Total"] = "42", + ["X-Apify-Pagination-Offset"] = "0", + ["X-Apify-Pagination-Limit"] = "3", + }; + var transport = new MockTransport().QueueResponse(200, "[{\"n\":1},{\"n\":2},{\"n\":3}]", headers); + var page = await Client(transport).Dataset("ds1").ListItemsAsync(); + + Assert.Equal(42, page.Total); + Assert.Equal(3, page.Count); + Assert.Equal(1, page.Items[0]!["n"]!.GetValue()); + } + + [Fact] + public async Task ValidateInputParsesBareObject() + { + var transport = new MockTransport().QueueResponse(200, "{\"valid\":true}"); + Assert.True(await Client(transport).Actor("apify/hello-world").ValidateInputAsync(new { x = 1 })); + } + + [Fact] + public async Task SafeIdReplacesFirstSlashWithTilde() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + await Client(transport).Actor("apify/hello-world").GetAsync(); + Assert.Contains("/actors/apify~hello-world", transport.LastRequest.Uri, StringComparison.Ordinal); + } +} diff --git a/tests/Apify.Client.Tests/Unit/LogClientTests.cs b/tests/Apify.Client.Tests/Unit/LogClientTests.cs new file mode 100644 index 0000000..5ec0655 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/LogClientTests.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class LogClientTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task GetLogByIdReturnsText() + { + var transport = new MockTransport().QueueResponse(200, "line1\nline2\n"); + var log = await Client(transport).Log("run1").GetAsync(); + + Assert.Equal("line1\nline2\n", log); + Assert.Equal("GET", transport.LastRequest.Method); + Assert.Contains("/logs/run1", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task MissingLogReturnsNull() + { + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"no log\"}}"); + Assert.Null(await Client(transport).Log("missing").GetAsync()); + } + + [Fact] + public async Task RunNestedLogGet() + { + var transport = new MockTransport().QueueResponse(200, "run log"); + var log = await Client(transport).Run("run1").Log().GetAsync(new LogOptions { Raw = true }); + + Assert.Equal("run log", log); + var uri = transport.LastRequest.Uri; + Assert.Contains("/actor-runs/run1/log", uri, StringComparison.Ordinal); + Assert.Contains("raw=1", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StreamedLogUsesStreamQueryAndReturnsReadableStream() + { + var transport = new MockTransport().QueueResponse(200, "streamed log body"); + using var stream = await Client(transport).Run("run1").GetStreamedLogAsync(); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actor-runs/run1/log", uri, StringComparison.Ordinal); + Assert.Contains("stream=1", uri, StringComparison.Ordinal); + Assert.Contains("raw=1", uri, StringComparison.Ordinal); + + using var reader = new StreamReader(stream); + Assert.Equal("streamed log body", await reader.ReadToEndAsync()); + } +} diff --git a/tests/Apify.Client.Tests/Unit/MockTransport.cs b/tests/Apify.Client.Tests/Unit/MockTransport.cs new file mode 100644 index 0000000..851dafc --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/MockTransport.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Http; + +namespace Apify.Client.Tests.Unit; + +/// +/// A snapshot of a request received by , captured before the underlying +/// is disposed by the client. +/// +public sealed class RecordedRequest +{ + private readonly Dictionary _headers; + + internal RecordedRequest(string method, string uri, string body, Dictionary headers) + { + Method = method; + Uri = uri; + Body = body; + _headers = headers; + } + + public string Method { get; } + + public string Uri { get; } + + public string Body { get; } + + public string Header(string name) => _headers.TryGetValue(name, out var value) ? value : string.Empty; +} + +/// +/// A scripted for offline unit tests. Each queued entry is either a response +/// to return or a transport failure to throw, consumed in order. All received requests are recorded for +/// assertions. +/// +public sealed class MockTransport : IHttpTransport +{ + private sealed record QueueEntry(bool IsError, bool Timeout, int Status, string Body, IReadOnlyDictionary? Headers); + + private readonly Queue _queue = new(); + private readonly object _lock = new(); + private int _inFlight; + + public List Received { get; } = new(); + + public List Timeouts { get; } = new(); + + /// + /// When set, every request is answered with a 200 whose processedRequests echoes each + /// uniqueKey in the (JSON array) request body — so batch calls succeed regardless of the order in + /// which concurrent chunks arrive. No queued responses are needed in this mode. + /// + public bool EchoBatchProcessed { get; set; } + + /// Artificial per-call delay (ms) used to force overlap when testing parallel dispatch. + public int ArtificialDelayMillis { get; set; } + + /// The highest number of requests observed in flight at the same time. + public int MaxObservedConcurrency { get; private set; } + + public MockTransport QueueResponse(int status, string body = "", IReadOnlyDictionary? headers = null) + { + _queue.Enqueue(new QueueEntry(false, false, status, body, headers)); + return this; + } + + public MockTransport QueueError(bool timeout = false) + { + _queue.Enqueue(new QueueEntry(true, timeout, 0, string.Empty, null)); + return this; + } + + public RecordedRequest LastRequest => + Received.Count == 0 ? throw new InvalidOperationException("no request was received") : Received[^1]; + + public int CallCount => Received.Count; + + public async Task SendAsync(HttpRequestMessage request, TimeSpan timeout, bool streaming, CancellationToken cancellationToken) + { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in request.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + var body = string.Empty; + if (request.Content is not null) + { + foreach (var header in request.Content.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + body = await request.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + + lock (_lock) + { + Received.Add(new RecordedRequest(request.Method.Method, request.RequestUri?.ToString() ?? string.Empty, body, headers)); + Timeouts.Add(timeout.TotalSeconds); + _inFlight++; + MaxObservedConcurrency = Math.Max(MaxObservedConcurrency, _inFlight); + } + + try + { + if (ArtificialDelayMillis > 0) + { + await Task.Delay(ArtificialDelayMillis, cancellationToken).ConfigureAwait(false); + } + + if (EchoBatchProcessed) + { + return BuildEchoResponse(body); + } + + QueueEntry entry; + lock (_lock) + { + if (_queue.Count == 0) + { + throw new InvalidOperationException("MockTransport queue is empty"); + } + + entry = _queue.Dequeue(); + } + + if (entry.IsError) + { + throw new ApifyTransportException("mock transport failure", null, entry.Timeout); + } + + var response = new HttpResponseMessage((HttpStatusCode)entry.Status) + { + Content = new StringContent(entry.Body), + }; + if (entry.Headers is not null) + { + foreach (var header in entry.Headers) + { + if (!response.Headers.TryAddWithoutValidation(header.Key, header.Value)) + { + response.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + } + + return response; + } + finally + { + lock (_lock) + { + _inFlight--; + } + } + } + + /// Builds a 200 batch response echoing each request-body uniqueKey as processed. + private static HttpResponseMessage BuildEchoResponse(string requestBody) + { + var processed = new System.Text.Json.Nodes.JsonArray(); + if (System.Text.Json.Nodes.JsonNode.Parse(requestBody) is System.Text.Json.Nodes.JsonArray array) + { + foreach (var item in array) + { + var key = item?["uniqueKey"]?.GetValue(); + if (key is not null) + { + processed.Add(new System.Text.Json.Nodes.JsonObject + { + ["uniqueKey"] = key, + ["requestId"] = "id-" + key, + ["wasAlreadyPresent"] = false, + ["wasAlreadyHandled"] = false, + }); + } + } + } + + var payload = new System.Text.Json.Nodes.JsonObject + { + ["data"] = new System.Text.Json.Nodes.JsonObject + { + ["processedRequests"] = processed, + ["unprocessedRequests"] = new System.Text.Json.Nodes.JsonArray(), + }, + }; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(payload.ToJsonString()) }; + } +} diff --git a/tests/Apify.Client.Tests/Unit/ModelSerializationTests.cs b/tests/Apify.Client.Tests/Unit/ModelSerializationTests.cs new file mode 100644 index 0000000..dc8346e --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ModelSerializationTests.cs @@ -0,0 +1,80 @@ +using System.Text.Json.Nodes; +using Apify.Client.Models; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Offline tests for the model "null fields are omitted" serialization contract: setting a property to +/// null must remove the key from the underlying JSON object rather than writing a JSON null +/// node (which the API would treat as an explicit null). +/// +[Trait("Category", "Unit")] +public sealed class ModelSerializationTests +{ + [Fact] + public void RequestQueueRequestUserDataNullRemovesKey() + { + var request = new RequestQueueRequest("https://a.com", "k"); + request.UserData = new JsonObject { ["label"] = "DETAIL" }; + Assert.True(request.ToJsonObject().ContainsKey("userData")); + + request.UserData = null; + Assert.False(request.ToJsonObject().ContainsKey("userData")); + } + + [Fact] + public void RequestQueueRequestUserDataStoresIndependentCopy() + { + var data = new JsonObject { ["label"] = "DETAIL" }; + var request = new RequestQueueRequest("https://a.com", "k") { UserData = data }; + + // Mutating the caller's object must not change the stored request (deep-cloned on set). + data["label"] = "MUTATED"; + Assert.Equal("DETAIL", request.UserData!["label"]!.GetValue()); + } + + [Fact] + public void ActorEnvVarNullSettersRemoveKeys() + { + var envVar = new ActorEnvVar("NAME", "value", isSecret: true); + var json = envVar.ToJsonObject(); + Assert.True(json.ContainsKey("name")); + Assert.True(json.ContainsKey("value")); + Assert.True(json.ContainsKey("isSecret")); + + envVar.Name = null; + envVar.Value = null; + envVar.IsSecret = null; + + Assert.False(json.ContainsKey("name")); + Assert.False(json.ContainsKey("value")); + Assert.False(json.ContainsKey("isSecret")); + } + + [Fact] + public void ActorEnvVarConstructorOmitsUnsetFields() + { + // Unset (null) constructor args must not appear as null nodes in the payload. + var envVar = new ActorEnvVar(name: "ONLY_NAME"); + var json = envVar.ToJsonObject(); + + Assert.True(json.ContainsKey("name")); + Assert.False(json.ContainsKey("value")); + Assert.False(json.ContainsKey("isSecret")); + } + + [Fact] + public void ActorEnvVarSettersWriteTypedValues() + { + var envVar = new ActorEnvVar(); + envVar.Name = "K"; + envVar.Value = "V"; + envVar.IsSecret = true; + + var json = envVar.ToJsonObject(); + Assert.Equal("K", json["name"]!.GetValue()); + Assert.Equal("V", json["value"]!.GetValue()); + Assert.True(json["isSecret"]!.GetValue()); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs new file mode 100644 index 0000000..4acf792 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -0,0 +1,134 @@ +using System; +using System.Threading.Tasks; +using System.Text.Json.Nodes; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Offline request-shape tests for mutating/convenience endpoints that are risky to exercise live on the +/// shared account. Each asserts the HTTP method, path, query and body the client actually sends. +/// +[Trait("Category", "Unit")] +public sealed class RequestShapeTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task RunChargeSendsBodyAndIdempotencyKey() + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Run("run1").ChargeAsync(new RunChargeOptions("result", count: 3)); + + var request = transport.LastRequest; + Assert.Equal("POST", request.Method); + Assert.Contains("/actor-runs/run1/charge", request.Uri, StringComparison.Ordinal); + Assert.NotEqual(string.Empty, request.Header("idempotency-key")); + var body = JsonNode.Parse(request.Body)!; + Assert.Equal("result", body["eventName"]!.GetValue()); + Assert.Equal(3, body["count"]!.GetValue()); + } + + [Fact] + public async Task RunChargeUsesProvidedIdempotencyKey() + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Run("run1").ChargeAsync(new RunChargeOptions("e", idempotencyKey: "fixed-key")); + Assert.Equal("fixed-key", transport.LastRequest.Header("idempotency-key")); + } + + [Fact] + public async Task MetamorphSendsTargetActorIdAndInput() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + await Client(transport).Run("run1").MetamorphAsync("apify/other", new { x = 1 }, new MetamorphOptions { Build = "latest" }); + + var request = transport.LastRequest; + Assert.Equal("POST", request.Method); + Assert.Contains("/actor-runs/run1/metamorph", request.Uri, StringComparison.Ordinal); + Assert.Contains("targetActorId=apify%2Fother", request.Uri, StringComparison.Ordinal); + Assert.Contains("build=latest", request.Uri, StringComparison.Ordinal); + Assert.Equal(1, JsonNode.Parse(request.Body)!["x"]!.GetValue()); + } + + [Fact] + public async Task ResurrectSendsOptions() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + await Client(transport).Run("run1").ResurrectAsync(new RunResurrectOptions { Build = "beta", MemoryMbytes = 1024 }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actor-runs/run1/resurrect", uri, StringComparison.Ordinal); + Assert.Contains("build=beta", uri, StringComparison.Ordinal); + Assert.Contains("memory=1024", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RebootPostsToRebootPath() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + await Client(transport).Run("run1").RebootAsync(); + + var request = transport.LastRequest; + Assert.Equal("POST", request.Method); + Assert.Contains("/actor-runs/run1/reboot", request.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task AbortSendsGracefullyFlag() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + await Client(transport).Run("run1").AbortAsync(true); + Assert.Contains("gracefully=1", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task DefaultBuildFetchesBuildsDefault() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"build1\"}}"); + await Client(transport).Actor("me~a").DefaultBuildAsync(10); + + var request = transport.LastRequest; + Assert.Equal("GET", request.Method); + Assert.Contains("/actors/me~a/builds/default", request.Uri, StringComparison.Ordinal); + Assert.Contains("waitForFinish=10", request.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RequestQueueOptionsApplyClientKey() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { ClientKey = "ck-123" }).ListHeadAsync(5); + + Assert.Contains("clientKey=ck-123", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RequestQueueOptionsApplyTimeout() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { TimeoutSecs = 2.0 }).ListHeadAsync(5); + + // The per-queue timeout must be threaded down to the transport (first attempt uses it directly). + Assert.Equal(2.0, transport.Timeouts[0]); + } + + [Fact] + public async Task UpdateLimitsPutsToMeLimits() + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Me().UpdateLimitsAsync(new { maxMonthlyUsageUsd = 100 }); + + var request = transport.LastRequest; + Assert.Equal("PUT", request.Method); + Assert.Contains("/users/me/limits", request.Uri, StringComparison.Ordinal); + Assert.Equal(100, JsonNode.Parse(request.Body)!["maxMonthlyUsageUsd"]!.GetValue()); + } +} diff --git a/tests/Apify.Client.Tests/Unit/SignatureTests.cs b/tests/Apify.Client.Tests/Unit/SignatureTests.cs new file mode 100644 index 0000000..a7497c5 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/SignatureTests.cs @@ -0,0 +1,59 @@ +using System; +using System.Text.RegularExpressions; +using Apify.Client.Internal; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class SignatureTests +{ + [Fact] + public void HmacSignatureIsDeterministicAndBase62() + { + var sig = Signatures.CreateHmacSignature("secret-key", "my-message"); + Assert.Equal(sig, Signatures.CreateHmacSignature("secret-key", "my-message")); + Assert.Matches(new Regex("^[0-9a-zA-Z]+$"), sig); + Assert.NotEqual(sig, Signatures.CreateHmacSignature("secret-key", "other-message")); + } + + /// + /// Known-answer vectors pinned to values independently computed with a bignum oracle (matching the + /// upstream @apify/utilities algorithm). Guards the byte-wise base62 long division and the base64url + /// envelope against regressions. + /// + [Fact] + public void KnownAnswerVectors() + { + Assert.Equal("G5BYW8zvRuVZrdxLfboF", Signatures.CreateHmacSignature("secret-key", "my-message")); + Assert.Equal("Oj9uljsqvVPaH2iLmW4i", Signatures.CreateHmacSignature("secret", "0.0.resource-id")); + Assert.Equal("MC4wLk9qOXVsanNxdlZQYUgyaUxtVzRp", Signatures.SignStorageContent("secret", "resource-id", null)); + } + + [Fact] + public void StorageContentSignatureIsBase64UrlWithoutPadding() + { + var sig = Signatures.SignStorageContent("secret", "resource-id", null); + Assert.DoesNotMatch(new Regex("[+/=]"), sig); + + var decoded = DecodeBase64Url(sig); + // Envelope form: "{version}.{expiresAtMillis}.{hmac}"; non-expiring uses expiry 0. + Assert.StartsWith("0.0.", decoded, StringComparison.Ordinal); + } + + [Fact] + public void ExpiringSignatureEncodesFutureExpiry() + { + var sig = Signatures.SignStorageContent("secret", "rid", 3600); + var decoded = DecodeBase64Url(sig); + var parts = decoded.Split('.'); + Assert.True(long.Parse(parts[1], System.Globalization.CultureInfo.InvariantCulture) > 0); + } + + private static string DecodeBase64Url(string value) + { + var s = value.Replace('-', '+').Replace('_', '/'); + s = s.PadRight(s.Length + ((4 - (s.Length % 4)) % 4), '='); + return System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(s)); + } +} From 4606459df7eca6dc01be90d32b73a03a6a19a4f4 Mon Sep 17 00:00:00 2001 From: apify-bot Date: Sat, 4 Jul 2026 00:31:01 +0000 Subject: [PATCH 2/5] chore: address review compliance for .NET client (spec v2-2026-07-02T131926Z) Client is already on the latest spec; this is a compliance/review pass: - Publishing workflow: use NuGet Trusted Publishing (OIDC via NuGet/login) with a short-lived key instead of a long-lived NUGET_API_KEY. - Forward last-run status/origin filters to nested dataset/key-value-store/ request-queue/log accessors (and route Dataset list/download/push through MergedParams) so a filtered last-run resolves the correct run. - Add lazy auto-paging IterateAsync to collection clients and DatasetClient.IterateItemsAsync (JS PaginatedIterator parity). - Add run-log redirection: public StreamedLog, RunClient.GetStreamedLog(sink), and a log sink parameter on Actor/Task CallAsync. - Add DefaultBuild integration test; docs/CHANGELOG/comment cleanups. --- .github/workflows/dotnet-publish.yml | 19 +- CHANGELOG.md | 14 +- docs/actors.md | 9 +- docs/builds.md | 1 + docs/examples.md | 15 +- docs/misc.md | 7 +- docs/runs.md | 5 + docs/schedules.md | 3 +- docs/storages.md | 8 +- docs/tasks.md | 6 +- docs/webhooks.md | 3 +- src/Apify.Client/Internal/QueryParams.cs | 12 ++ src/Apify.Client/Internal/ResourceContext.cs | 55 ++++- .../AbstractWebhookCollectionClient.cs | 14 ++ src/Apify.Client/Resources/ActorClient.cs | 9 +- .../Resources/ActorCollectionClient.cs | 15 ++ .../Resources/ActorEnvVarCollectionClient.cs | 18 ++ .../Resources/ActorVersionCollectionClient.cs | 14 ++ .../Resources/BuildCollectionClient.cs | 14 ++ src/Apify.Client/Resources/DatasetClient.cs | 67 +++++- .../Resources/DatasetCollectionClient.cs | 14 ++ .../Resources/KeyValueStoreClient.cs | 4 +- .../KeyValueStoreCollectionClient.cs | 14 ++ src/Apify.Client/Resources/LogClient.cs | 4 +- .../Resources/RequestQueueClient.cs | 4 +- .../Resources/RequestQueueCollectionClient.cs | 14 ++ src/Apify.Client/Resources/RunClient.cs | 53 ++++- .../Resources/RunCollectionClient.cs | 19 ++ .../Resources/ScheduleCollectionClient.cs | 14 ++ src/Apify.Client/Resources/TaskClient.cs | 9 +- .../Resources/TaskCollectionClient.cs | 14 ++ .../WebhookDispatchCollectionClient.cs | 14 ++ src/Apify.Client/StreamedLog.cs | 191 ++++++++++++++++++ .../Examples/LogRedirectionExample.cs | 12 +- .../Integration/BuildIntegrationTests.cs | 11 + .../Unit/AutoPagingTests.cs | 95 +++++++++ .../Unit/RequestShapeTests.cs | 76 +++++++ .../Unit/StreamedLogTests.cs | 107 ++++++++++ 38 files changed, 922 insertions(+), 55 deletions(-) create mode 100644 src/Apify.Client/StreamedLog.cs create mode 100644 tests/Apify.Client.Tests/Unit/AutoPagingTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/StreamedLogTests.cs diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml index 34eeda5..6b9732c 100644 --- a/.github/workflows/dotnet-publish.yml +++ b/.github/workflows/dotnet-publish.yml @@ -5,8 +5,10 @@ name: Publish .NET client # of truth in src/Apify.Client/Apify.Client.csproj (). This workflow packs the library, # pushes it to NuGet.org, tags the release, and creates the GitHub release. # -# The NuGet API key is read from a repository secret (nothing is stored in the repo). If the account -# has NuGet Trusted Publishing (OIDC) configured, the id-token permission below allows switching to it. +# Publishing uses NuGet.org Trusted Publishing (OIDC): the NuGet/login action exchanges a short-lived +# GitHub OIDC token for a temporary NuGet API key just before the push, so no long-lived API key is +# stored in the repo. The only repository secret needed is NUGET_USER (the nuget.org account/profile +# name that owns the trusted-publishing policy). on: workflow_dispatch: inputs: @@ -22,7 +24,7 @@ concurrency: permissions: contents: write # create the tagged GitHub release - id-token: write # allow NuGet Trusted Publishing (OIDC) if configured + id-token: write # NuGet Trusted Publishing exchanges this OIDC token for a temporary API key jobs: publish: @@ -82,10 +84,19 @@ jobs: - name: Pack run: dotnet pack src/Apify.Client/Apify.Client.csproj --configuration Release --no-build --output ./artifacts + # Exchange the GitHub OIDC token for a short-lived NuGet API key (Trusted Publishing). Runs + # immediately before the push because the temporary key is valid for only ~1 hour. + - name: NuGet login (Trusted Publishing OIDC) + if: ${{ github.event.inputs.dry_run != 'true' }} + id: nuget_login + uses: NuGet/login@v1 + with: + user: ${{ secrets.NUGET_USER }} + - name: Push to NuGet if: ${{ github.event.inputs.dry_run != 'true' }} env: - NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + NUGET_API_KEY: ${{ steps.nuget_login.outputs.NUGET_API_KEY }} run: | dotnet nuget push "./artifacts/*.nupkg" \ --api-key "${NUGET_API_KEY}" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 38269ac..5fd4a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,16 @@ with the JS reference client: `Actor().CallAsync()`/`StartAsync()`, `ValidateInputAsync()`, `DefaultBuildAsync()`, `LastRun()`, run `AbortAsync`/`MetamorphAsync`/`RebootAsync`/`ResurrectAsync`/ `ChargeAsync`/`WaitForFinishAsync`, dataset `ListItemsAsync`/`DownloadItemsAsync`/`PushItemsAsync`/ - public URLs, key-value store records and public URLs, request queue batch add with retries, lazy - request/store iteration (`IAsyncEnumerable`), and log streaming. + public URLs, key-value store records and public URLs, request queue batch add with retries, and log + streaming. +- Auto-paging lazy iteration (`IAsyncEnumerable`) across all collection clients (`IterateAsync`) and + dataset items (`IterateItemsAsync`), plus request-queue and Store iteration, matching the reference + client's paginated iterators. +- Run log redirection: `RunClient.GetStreamedLog(toLog, fromStart)` returns a `StreamedLog` that forwards + a run's live log to a sink one message at a time, and `Actor`/`Task` `CallAsync` accept a `log` sink that + redirects the run's log for the duration of the wait. +- Last-run accessors forward their `status`/`origin` filters to the run's nested dataset, key-value store, + request queue, and log clients. - Binary-safe storage payloads: `KeyValueStoreRecord.Value` and `DownloadItemsAsync` return `byte[]` (raw bytes), and `SetRecordAsync` accepts `byte[]`, so binary records and exports (e.g. XLSX) are not corrupted; `SetRecordJsonAsync` serializes to JSON bytes. @@ -34,4 +42,4 @@ HMAC-SHA256 storage URL signing. - Public `ApifyClientVersion.ClientVersion` and `ApifyClientVersion.ApiSpecVersion` constants. - Integration test suite, documentation with runnable examples, and CI workflows for integration - tests and publishing. + tests and publishing (NuGet.org Trusted Publishing via OIDC). diff --git a/docs/actors.md b/docs/actors.md index 281ca6b..a54de5b 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -5,8 +5,10 @@ Access the Actor collection with `client.Actors()` and a specific Actor with `cl ## Collection — `client.Actors()` -- `ListAsync(ActorListOptions? options = null)` — list the account's Actors. Returns +- `ListAsync(ActorListOptions? options = null)` — list the account's Actors (one page). Returns `PaginationList`. Options: `Offset`, `Limit`, `Desc`, `My`, `SortBy`. +- `IterateAsync(ActorListOptions? options = null)` → `IAsyncEnumerable` — lazily iterate every + Actor across pages, fetching each page on demand. - `CreateAsync(object actor)` — create an Actor from any JSON-serializable definition. Returns `Actor`. ```csharp @@ -27,8 +29,9 @@ foreach (var actor in page.Items) - `UpdateAsync(object newFields)` → `Actor`. - `DeleteAsync()`. - `StartAsync(object? input = null, ActorStartOptions? options = null)` → `ActorRun` (returns immediately). -- `CallAsync(object? input = null, ActorStartOptions? options = null, int? waitSecs = null)` → `ActorRun` - (starts then waits; `waitSecs` bounds the wait, `null` waits indefinitely). +- `CallAsync(object? input = null, ActorStartOptions? options = null, int? waitSecs = null, Action? log = null)` + → `ActorRun` (starts then waits; `waitSecs` bounds the wait, `null` waits indefinitely; `log`, if set, + redirects the run's live log to that sink for the duration of the wait). - `ValidateInputAsync(object? input = null, ValidateInputOptions? options = null)` → `bool`. - `BuildAsync(string versionNumber, ActorBuildOptions? options = null)` → `Build`. - `DefaultBuildAsync(int? waitForFinish = null)` → `BuildClient`. diff --git a/docs/builds.md b/docs/builds.md index f6e8afb..cf01e06 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -6,6 +6,7 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil ## Collection - `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`). +- `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). ## Single build — `client.Build(buildId)` diff --git a/docs/examples.md b/docs/examples.md index f1a10c6..22dcfba 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -23,6 +23,7 @@ var client = new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN")); ## Run a store Actor and read its dataset ```csharp +// The third argument bounds the wait in seconds (120 here); pass null to wait indefinitely. var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); Console.WriteLine("Item count: " + items.Count); @@ -121,10 +122,18 @@ await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Li ## Run an Actor with log redirection (streaming) +```csharp +// The `log` argument redirects the run's live log to the given sink (here stdout) while it runs; the +// client streams the log and forwards each complete message as it arrives. +await client.Actor("apify/hello-world").CallAsync(null, null, 120, log: Console.WriteLine); +``` + +You can also redirect a specific run's log yourself with `GetStreamedLog`: + ```csharp var run = await client.Actor("apify/hello-world").StartAsync(); +await using var streamedLog = client.Run(run.Id!).GetStreamedLog(Console.WriteLine); +streamedLog.Start(); await client.Run(run.Id!).WaitForFinishAsync(120); -using var stream = await client.Run(run.Id!).GetStreamedLogAsync(); -using var reader = new StreamReader(stream); -Console.WriteLine(await reader.ReadToEndAsync()); +await streamedLog.StopAsync(); ``` diff --git a/docs/misc.md b/docs/misc.md index 0db370e..5e3b140 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -52,11 +52,10 @@ var usage = await client.Me().MonthlyUsageAsync(); ```csharp using System; -using System.IO; using Apify.Client; +using Apify.Client.Options; var client = new ApifyClient("my-api-token"); -using var stream = await client.Run("some-run-id").GetStreamedLogAsync(); -using var reader = new StreamReader(stream); -Console.WriteLine(await reader.ReadToEndAsync()); +var log = await client.Log("some-run-id").GetAsync(new LogOptions { Raw = true }); +Console.WriteLine(log); ``` diff --git a/docs/runs.md b/docs/runs.md index 489ee0c..d088ea0 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -6,6 +6,8 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' ## Collection - `ListAsync(ListOptions? options = null, RunListOptions? filter = null)` → `PaginationList`. +- `IterateAsync(ListOptions? options = null, RunListOptions? filter = null)` → `IAsyncEnumerable` + (lazy, all pages). `RunListOptions`: `Status` (list), `StartedAfter`, `StartedBefore`. ## Single run — `client.Run(runId)` @@ -21,6 +23,9 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' - `WaitForFinishAsync(int? waitSecs = null)` → `ActorRun`. - `Dataset()`, `KeyValueStore()`, `RequestQueue()` — the run's default storages. - `Log()` → `LogClient`; `GetStreamedLogAsync()` → `Stream` (live raw log). +- `GetStreamedLog(Action toLog, bool fromStart = true)` → `StreamedLog` — redirects the run's live + log to `toLog` one complete message at a time. Call `Start()` to begin and `StopAsync()` (or dispose) to + end. `fromStart: false` skips messages older than the helper's creation. ```csharp using Apify.Client; diff --git a/docs/schedules.md b/docs/schedules.md index 58d9bf0..ec620a7 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -5,7 +5,8 @@ Schedules automatically start Actor or task runs at specified times. Access the ## Collection -- `ListAsync(ListOptions?)` → `PaginationList`. +- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → + `IAsyncEnumerable` (lazy, all pages). - `CreateAsync(object schedule)` → `Schedule`. ## Single schedule — `client.Schedule(id)` diff --git a/docs/storages.md b/docs/storages.md index db0218d..6aab6c7 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -1,7 +1,8 @@ # Storages The three storage types — datasets, key-value stores and request queues — share the same collection -shape: `ListAsync(StorageListOptions?)` and `GetOrCreateAsync(name?)`. Storages can also be reached +shape: `ListAsync(StorageListOptions?)` (one page), `IterateAsync(StorageListOptions?)` → +`IAsyncEnumerable` (lazy, all pages), and `GetOrCreateAsync(name?)`. Storages can also be reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). > Snippets below run inside an `async` context. `ImplicitUsings` is disabled in this repository, so all @@ -14,7 +15,10 @@ from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). `client.Datasets()` / `client.Dataset(id)`. - `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. -- `ListItemsAsync(DatasetListItemsOptions?)` → `PaginationList` (pagination via response headers). +- `ListItemsAsync(DatasetListItemsOptions?)` → `PaginationList` (one page; pagination via + response headers). +- `IterateItemsAsync(DatasetListItemsOptions?)` → `IAsyncEnumerable` — lazily iterate every + item across pages, fetching each page on demand. - `DownloadItemsAsync(DownloadItemsFormat, DatasetDownloadOptions?)` → serialized items as `byte[]` (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). - `PushItemsAsync(object items)` — push one object or an array of objects. diff --git a/docs/tasks.md b/docs/tasks.md index d5e6e33..f29b808 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -5,14 +5,16 @@ a specific task with `client.Task(id)`. ## Collection -- `ListAsync(ListOptions?)` → `PaginationList`. +- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → + `IAsyncEnumerable` (lazy, all pages). - `CreateAsync(object task)` → `ActorTask`. ## Single task — `client.Task(id)` - `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. - `StartAsync(object? input = null, TaskStartOptions? options = null)` → `ActorRun`. -- `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null)` → `ActorRun`. +- `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null, Action? log = null)` + → `ActorRun` (`log`, if set, redirects the run's live log to that sink for the duration of the wait). - `GetInputAsync()` / `UpdateInputAsync(object input)`. - `LastRun(LastRunOptions?)` → `RunClient`; `Runs()` → `RunCollectionClient`. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. diff --git a/docs/webhooks.md b/docs/webhooks.md index 6a5a611..70f7b56 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -6,7 +6,8 @@ Webhooks notify an external service when specific events occur. Access the accou ## Webhook collection — `client.Webhooks()` -- `ListAsync(ListOptions?)` → `PaginationList`. +- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → + `IAsyncEnumerable` (lazy, all pages). Webhook dispatches expose the same pair. - `CreateAsync(object webhook)` → `Webhook`. Webhooks nested under an Actor or task (`client.Actor(id).Webhooks()`, `client.Task(id).Webhooks()`) diff --git a/src/Apify.Client/Internal/QueryParams.cs b/src/Apify.Client/Internal/QueryParams.cs index a42e2f6..796a529 100644 --- a/src/Apify.Client/Internal/QueryParams.cs +++ b/src/Apify.Client/Internal/QueryParams.cs @@ -80,6 +80,18 @@ public QueryParams AddRaw(string key, string value) return this; } + /// + /// Sets an integer parameter, replacing any existing occurrences of . Used by + /// auto-paging iteration to overwrite the per-page offset/limit without emitting the + /// parameter twice. + /// + public QueryParams Set(string key, long value) + { + _pairs.RemoveAll(pair => string.Equals(pair.Key, key, StringComparison.Ordinal)); + _pairs.Add(new KeyValuePair(key, value.ToString(CultureInfo.InvariantCulture))); + return this; + } + /// Whether no parameters have been added. public bool IsEmpty => _pairs.Count == 0; diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index 097c3d0..057e856 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Net.Http; using System.Threading; @@ -62,9 +63,18 @@ private ResourceContext(HttpClientCore http, string url, string baseUrl) /// The per-context request timeout, or null to use the client-wide default. public TimeSpan? RequestTimeout => _requestTimeout; - /// Creates a context for a collection endpoint: {base}/{resourcePath}. - public static ResourceContext Collection(HttpClientCore http, string baseUrl, string resourcePath) - => new(http, baseUrl + "/" + resourcePath, baseUrl); + /// + /// Creates a context for a collection endpoint: {base}/{resourcePath}. Any + /// are copied into so they are applied to + /// every call on the nested resource (used to forward the last-run status/origin filters + /// to a run's nested storage and log clients, matching the reference client). + /// + public static ResourceContext Collection(HttpClientCore http, string baseUrl, string resourcePath, QueryParams? inheritedParams = null) + { + var ctx = new ResourceContext(http, baseUrl + "/" + resourcePath, baseUrl); + ctx.BaseParams.Extend(inheritedParams); + return ctx; + } /// Creates a context for a single resource: {base}/{resourcePath}/{safeId}. public static ResourceContext Single(HttpClientCore http, string baseUrl, string resourcePath, string id) @@ -157,6 +167,45 @@ public async Task> ListResourceAsync(string subPath, QueryP return PaginationList.FromData(data, hydrate); } + /// + /// Lazily iterates every item of an offset/limit-paginated listing, fetching pages on demand. Mirrors + /// the reference client's auto-paging list iterators: it starts at and, + /// if is set, yields at most that many items in total. + /// carries the caller's filters; its offset/limit are overwritten per page. + /// + public async IAsyncEnumerable IterateListAsync( + string subPath, + QueryParams baseQuery, + int startOffset, + int? limit, + Func hydrate, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + { + var offset = Math.Max(startOffset, 0); + var yielded = 0; + while (true) + { + var q = baseQuery.Copy().Set("offset", offset); + if (limit is not null) + { + q.Set("limit", Math.Max(limit.Value - yielded, 0)); + } + + var page = await ListResourceAsync(subPath, q, hydrate, ct).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + yielded++; + } + + offset += (int)page.Count; + if (page.Count == 0 || offset >= page.Total || (limit is not null && yielded >= limit.Value)) + { + yield break; + } + } + } + /// POST to create a resource with a JSON-serializable body, returning the decoded data. public async Task CreateResourceAsync(QueryParams p, object? body, CancellationToken ct) { diff --git a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs index 7617f62..fa5c609 100644 --- a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs +++ b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -30,4 +31,17 @@ public Task> ListAsync(ListOptions? options = null, Canc (options ?? new ListOptions()).AppendTo(q); return Ctx.ListResourceAsync("", q, static d => new Webhook(d), cancellationToken); } + + /// Lazily iterates over all webhooks across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return Ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Webhook(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/ActorClient.cs b/src/Apify.Client/Resources/ActorClient.cs index 9374f30..8d05426 100644 --- a/src/Apify.Client/Resources/ActorClient.cs +++ b/src/Apify.Client/Resources/ActorClient.cs @@ -1,3 +1,4 @@ +using System; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -70,15 +71,21 @@ public async Task StartAsync(object? input = null, ActorStartOptions? /// Any JSON-serializable value (or null for no input). /// Optional run-start options. /// Bounds the wait; null waits indefinitely. + /// + /// If provided, the run's live log is redirected to this sink (one complete message per call) for the + /// duration of the wait, matching the reference client's log call option. null disables + /// redirection. + /// /// A token to cancel the request. public async Task CallAsync( object? input = null, ActorStartOptions? options = null, int? waitSecs = null, + Action? log = null, CancellationToken cancellationToken = default) { var run = await StartAsync(input, options, cancellationToken).ConfigureAwait(false); - return await _root.Run(run.Id ?? string.Empty).WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + return await _root.Run(run.Id ?? string.Empty).WaitForFinishWithLogAsync(waitSecs, log, cancellationToken).ConfigureAwait(false); } /// Validates against the Actor's input schema and returns whether it is valid. diff --git a/src/Apify.Client/Resources/ActorCollectionClient.cs b/src/Apify.Client/Resources/ActorCollectionClient.cs index a7b8293..d6ce616 100644 --- a/src/Apify.Client/Resources/ActorCollectionClient.cs +++ b/src/Apify.Client/Resources/ActorCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -26,6 +27,20 @@ public Task> ListAsync(ActorListOptions? options = null, C return _ctx.ListResourceAsync("", q, static d => new Actor(d), cancellationToken); } + /// + /// Lazily iterates over all of the account's Actors across pages, fetching each page on demand. + /// + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of Actors yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ActorListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ActorListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Actor(d), cancellationToken); + } + /// Creates a new Actor. /// Any JSON-serializable Actor definition. /// A token to cancel the request. diff --git a/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs b/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs index 34fcef9..d073ad4 100644 --- a/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs +++ b/src/Apify.Client/Resources/ActorEnvVarCollectionClient.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -33,4 +35,20 @@ public async Task CreateAsync(ActorEnvVar envVar, CancellationToken return ActorEnvVar.FromJsonObject( await _ctx.CreateResourceAsync(new QueryParams(), envVar.ToJsonObject(), cancellationToken).ConfigureAwait(false)); } + + /// + /// Lazily iterates over the version's environment variables. The env-var endpoint returns the whole + /// list in a single page, so this yields that page's items; it exists for API parity with the other + /// collection iterators and the reference client. + /// + /// A token to cancel the iteration. + public async IAsyncEnumerable IterateAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var page = await ListAsync(cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + } + } diff --git a/src/Apify.Client/Resources/ActorVersionCollectionClient.cs b/src/Apify.Client/Resources/ActorVersionCollectionClient.cs index 771b156..d681df7 100644 --- a/src/Apify.Client/Resources/ActorVersionCollectionClient.cs +++ b/src/Apify.Client/Resources/ActorVersionCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -33,4 +34,17 @@ public async Task CreateAsync(object version, CancellationToken ca { return new ActorVersion(await _ctx.CreateResourceAsync(new QueryParams(), version, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all Actor versions across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorVersion(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/BuildCollectionClient.cs b/src/Apify.Client/Resources/BuildCollectionClient.cs index ce4d187..3960e55 100644 --- a/src/Apify.Client/Resources/BuildCollectionClient.cs +++ b/src/Apify.Client/Resources/BuildCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -28,4 +29,17 @@ public Task> ListAsync(ListOptions? options = null, Cancel (options ?? new ListOptions()).AppendTo(q); return _ctx.ListResourceAsync("", q, static d => new Build(d), cancellationToken); } + + /// Lazily iterates over all builds across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Build(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index 6476d98..477e657 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -1,6 +1,8 @@ +using System; using System.Collections.Generic; using System.Globalization; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -25,8 +27,8 @@ private DatasetClient(HttpClientCore http, ResourceContext ctx) internal static DatasetClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "datasets", id)); - internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath) - => new(http, ResourceContext.Collection(http, baseUrl, subPath)); + internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) + => new(http, ResourceContext.Collection(http, baseUrl, subPath, inheritedParams)); internal DatasetClient WithPublicBase(string publicBaseUrl) { @@ -64,12 +66,62 @@ public async Task UpdateAsync(object newFields, CancellationToken cance /// /// Optional item filtering/projection and pagination. /// A token to cancel the request. - public async Task> ListItemsAsync(DatasetListItemsOptions? options = null, CancellationToken cancellationToken = default) + public Task> ListItemsAsync(DatasetListItemsOptions? options = null, CancellationToken cancellationToken = default) { options ??= new DatasetListItemsOptions(); var q = new QueryParams(); options.AppendTo(q); - var url = q.ApplyToUrl(_ctx.SubUrl("items")); + return FetchItemsPageAsync(q, options.Desc ?? false, cancellationToken); + } + + /// + /// Lazily iterates over all items of the dataset across pages, fetching each page on demand. Mirrors + /// the reference client's auto-paging listItems iterator. + /// + /// Optional item filtering/projection; Offset/Limit bound where + /// iteration starts and the total number of items yielded. + /// A token to cancel the iteration. + public async IAsyncEnumerable IterateItemsAsync( + DatasetListItemsOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new DatasetListItemsOptions(); + var desc = options.Desc ?? false; + var baseQuery = new QueryParams(); + options.AppendTo(baseQuery); + var offset = Math.Max(options.Offset ?? 0, 0); + var limit = options.Limit; + var yielded = 0; + while (true) + { + var q = baseQuery.Copy().Set("offset", offset); + if (limit is not null) + { + q.Set("limit", Math.Max(limit.Value - yielded, 0)); + } + + var page = await FetchItemsPageAsync(q, desc, cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + yielded++; + } + + offset += (int)page.Count; + if (page.Count == 0 || offset >= page.Total || (limit is not null && yielded >= limit.Value)) + { + yield break; + } + } + } + + /// + /// Fetches a single page of dataset items. The endpoint returns a bare JSON array with pagination in + /// X-Apify-Pagination-* response headers. + /// + private async Task> FetchItemsPageAsync(QueryParams q, bool desc, CancellationToken cancellationToken) + { + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("items")); using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); @@ -88,7 +140,7 @@ public async Task UpdateAsync(object newFields, CancellationToken cance HeaderInt(response, "X-Apify-Pagination-Total", count), HeaderInt(response, "X-Apify-Pagination-Offset", 0), HeaderInt(response, "X-Apify-Pagination-Limit", count), - options.Desc ?? false); + desc); } /// @@ -106,7 +158,7 @@ public async Task DownloadItemsAsync(DownloadItemsFormat format, Dataset var q = new QueryParams(); q.AddString("format", format.ToWireValue()); (options ?? new DatasetDownloadOptions()).AppendTo(q); - var url = q.ApplyToUrl(_ctx.SubUrl("items")); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("items")); using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); } @@ -116,9 +168,10 @@ public async Task DownloadItemsAsync(DownloadItemsFormat format, Dataset /// A token to cancel the request. public async Task PushItemsAsync(object items, CancellationToken cancellationToken = default) { + var url = _ctx.MergedParams(new QueryParams()).ApplyToUrl(_ctx.SubUrl("items")); using var response = await _http.CallAsync( HttpMethod.Post, - _ctx.SubUrl("items"), + url, Json.Encode(items), ResourceContext.ContentTypeJsonCharset, timeout: _ctx.RequestTimeout, diff --git a/src/Apify.Client/Resources/DatasetCollectionClient.cs b/src/Apify.Client/Resources/DatasetCollectionClient.cs index cecf4e7..6eb6538 100644 --- a/src/Apify.Client/Resources/DatasetCollectionClient.cs +++ b/src/Apify.Client/Resources/DatasetCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -39,4 +40,17 @@ public async Task GetOrCreateAsync(string? name = null, JsonNode? schem { return new Dataset(await _ctx.GetOrCreateNamedAsync(name, schema, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all datasets across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new StorageListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Dataset(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/KeyValueStoreClient.cs b/src/Apify.Client/Resources/KeyValueStoreClient.cs index ff09446..e38a0b5 100644 --- a/src/Apify.Client/Resources/KeyValueStoreClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreClient.cs @@ -24,8 +24,8 @@ private KeyValueStoreClient(HttpClientCore http, ResourceContext ctx) internal static KeyValueStoreClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "key-value-stores", id)); - internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath) - => new(http, ResourceContext.Collection(http, baseUrl, subPath)); + internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) + => new(http, ResourceContext.Collection(http, baseUrl, subPath, inheritedParams)); internal KeyValueStoreClient WithPublicBase(string publicBaseUrl) { diff --git a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs index 065d18f..56097c3 100644 --- a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -38,4 +39,17 @@ public async Task GetOrCreateAsync(string? name = null, JsonNode? { return new KeyValueStore(await _ctx.GetOrCreateNamedAsync(name, schema, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all key-value stores across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new StorageListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new KeyValueStore(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/LogClient.cs b/src/Apify.Client/Resources/LogClient.cs index bac7b31..afdcd06 100644 --- a/src/Apify.Client/Resources/LogClient.cs +++ b/src/Apify.Client/Resources/LogClient.cs @@ -25,8 +25,8 @@ private LogClient(HttpClientCore http, ResourceContext ctx) internal static LogClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "logs", id)); - internal static LogClient Nested(HttpClientCore http, string baseUrl) - => new(http, ResourceContext.Collection(http, baseUrl, "log")); + internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParams? inheritedParams = null) + => new(http, ResourceContext.Collection(http, baseUrl, "log", inheritedParams)); /// Fetches the log as text, or null if the log does not exist. /// Optional log-content options. diff --git a/src/Apify.Client/Resources/RequestQueueClient.cs b/src/Apify.Client/Resources/RequestQueueClient.cs index 275f291..6277d76 100644 --- a/src/Apify.Client/Resources/RequestQueueClient.cs +++ b/src/Apify.Client/Resources/RequestQueueClient.cs @@ -49,8 +49,8 @@ internal static RequestQueueClient ForId(HttpClientCore http, string baseUrl, st return new RequestQueueClient(http, ctx, options?.ClientKey, timeout); } - internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath) - => new(http, ResourceContext.Collection(http, baseUrl, subPath), null, null); + internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) + => new(http, ResourceContext.Collection(http, baseUrl, subPath, inheritedParams), null, null); /// /// Returns a copy of the client that identifies its requests with . A diff --git a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs index ffc3cf9..c9d3713 100644 --- a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs +++ b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -36,4 +37,17 @@ public async Task GetOrCreateAsync(string? name = null, Cancellati { return new RequestQueue(await _ctx.GetOrCreateNamedAsync(name, null, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all request queues across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(StorageListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new StorageListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new RequestQueue(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs index f3ced69..d6bfdca 100644 --- a/src/Apify.Client/Resources/RunClient.cs +++ b/src/Apify.Client/Resources/RunClient.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; +using Apify.Client; using Apify.Client.Internal; using Apify.Client.Models; using Apify.Client.Options; @@ -196,23 +197,63 @@ public async Task WaitForFinishAsync(int? waitSecs = null, Cancellatio return new ActorRun(data); } + /// + /// Waits for the run to finish, optionally redirecting its live log to for the + /// duration of the wait. Shared by Actor.Call/Task.Call's log option. + /// + internal async Task WaitForFinishWithLogAsync(int? waitSecs, Action? log, CancellationToken cancellationToken) + { + if (log is null) + { + return await WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + } + + var streamedLog = GetStreamedLog(log); + streamedLog.Start(); + try + { + return await WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + } + finally + { + await streamedLog.StopAsync().ConfigureAwait(false); + } + } + + // Nested accessors inherit this client's params so last-run status/origin filters (see + // SetLastRunParams) resolve the intended run's storage/log rather than the latest run's. + /// A client for this run's default dataset. - public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset"); + public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset", _ctx.BaseParams); /// A client for this run's default key-value store. - public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store"); + public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store", _ctx.BaseParams); /// A client for this run's default request queue. - public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue"); + public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue", _ctx.BaseParams); /// A client for accessing this run's log. - public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl("")); + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), _ctx.BaseParams); /// - /// Opens a live stream of this run's raw log, for convenient log redirection. The caller reads (and - /// disposes) the returned stream. + /// Opens a live stream of this run's raw log bytes. The caller reads (and disposes) the returned + /// stream. For automatic redirection into a sink, prefer . /// /// A token to cancel the request. public Task GetStreamedLogAsync(CancellationToken cancellationToken = default) => Log().StreamAsync(new LogOptions { Raw = true }, cancellationToken); + + /// + /// Creates a that redirects this run's live log to , + /// one complete message at a time. Call to begin and + /// (or dispose it) to end. Consistent with the reference client's + /// run-log redirection convenience. + /// + /// The sink each complete log message is written to. + /// + /// If true (default), redirect the whole log including messages from before this call; if + /// false, skip messages older than the moment the helper is created. + /// + public StreamedLog GetStreamedLog(Action toLog, bool fromStart = true) + => new(Log(), toLog, fromStart); } diff --git a/src/Apify.Client/Resources/RunCollectionClient.cs b/src/Apify.Client/Resources/RunCollectionClient.cs index 28cc84a..586234c 100644 --- a/src/Apify.Client/Resources/RunCollectionClient.cs +++ b/src/Apify.Client/Resources/RunCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -33,4 +34,22 @@ public Task> ListAsync( (filter ?? new RunListOptions()).AppendTo(q); return _ctx.ListResourceAsync("", q, static d => new ActorRun(d), cancellationToken); } + + /// Lazily iterates over all runs across pages, fetching each page on demand. + /// Optional pagination; Offset/Limit bound where iteration starts + /// and the total number of runs yielded. + /// Optional run-specific filters. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync( + ListOptions? options = null, + RunListOptions? filter = null, + CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + (filter ?? new RunListOptions()).AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorRun(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/ScheduleCollectionClient.cs b/src/Apify.Client/Resources/ScheduleCollectionClient.cs index 3b35052..3664add 100644 --- a/src/Apify.Client/Resources/ScheduleCollectionClient.cs +++ b/src/Apify.Client/Resources/ScheduleCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -33,4 +34,17 @@ public async Task CreateAsync(object schedule, CancellationToken cance { return new Schedule(await _ctx.CreateResourceAsync(new QueryParams(), schedule, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all schedules across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Schedule(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs index b1f45fe..411eefb 100644 --- a/src/Apify.Client/Resources/TaskClient.cs +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -1,3 +1,4 @@ +using System; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -64,15 +65,21 @@ public async Task StartAsync(object? input = null, TaskStartOptions? o /// Optionally overrides the task's stored input. /// Optional run-start options. /// Bounds the wait; null waits indefinitely. + /// + /// If provided, the run's live log is redirected to this sink (one complete message per call) for the + /// duration of the wait, matching the reference client's log call option. null disables + /// redirection. + /// /// A token to cancel the request. public async Task CallAsync( object? input = null, TaskStartOptions? options = null, int? waitSecs = null, + Action? log = null, CancellationToken cancellationToken = default) { var run = await StartAsync(input, options, cancellationToken).ConfigureAwait(false); - return await _root.Run(run.Id ?? string.Empty).WaitForFinishAsync(waitSecs, cancellationToken).ConfigureAwait(false); + return await _root.Run(run.Id ?? string.Empty).WaitForFinishWithLogAsync(waitSecs, log, cancellationToken).ConfigureAwait(false); } /// Fetches the task's stored input, or null if none is set. diff --git a/src/Apify.Client/Resources/TaskCollectionClient.cs b/src/Apify.Client/Resources/TaskCollectionClient.cs index 0800d14..6bb3ad5 100644 --- a/src/Apify.Client/Resources/TaskCollectionClient.cs +++ b/src/Apify.Client/Resources/TaskCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -33,4 +34,17 @@ public async Task CreateAsync(object task, CancellationToken cancella { return new ActorTask(await _ctx.CreateResourceAsync(new QueryParams(), task, cancellationToken).ConfigureAwait(false)); } + + /// Lazily iterates over all tasks across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorTask(d), cancellationToken); + } + } diff --git a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs index 0563698..d12da5e 100644 --- a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs +++ b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -28,4 +29,17 @@ public Task> ListAsync(ListOptions? options = nu (options ?? new ListOptions()).AppendTo(q); return _ctx.ListResourceAsync("", q, static d => new WebhookDispatch(d), cancellationToken); } + + /// Lazily iterates over all webhook dispatches across pages, fetching each page on demand. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of items yielded. + /// A token to cancel the iteration. + public IAsyncEnumerable IterateAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new ListOptions(); + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new WebhookDispatch(d), cancellationToken); + } + } diff --git a/src/Apify.Client/StreamedLog.cs b/src/Apify.Client/StreamedLog.cs new file mode 100644 index 0000000..5e463b0 --- /dev/null +++ b/src/Apify.Client/StreamedLog.cs @@ -0,0 +1,191 @@ +using System; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Options; +using Apify.Client.Resources; + +namespace Apify.Client; + +/// +/// Redirects a run's (or build's) live log to a destination sink, one complete message at a time. Mirrors +/// the reference client's StreamedLog helper: it opens the raw log stream, splits it on Apify's +/// ISO-8601 timestamp line markers, and forwards each complete message to the toLog callback. +/// +/// +/// The destination is modelled as an (the idiomatic .NET equivalent of the +/// reference client's logger): each argument is one complete, trimmed log message. Redirection runs on a +/// background task started by and drained by (or by disposal). +/// +public sealed class StreamedLog : IAsyncDisposable +{ + /// + /// Apify log lines are prefixed with an ISO-8601 UTC timestamp (e.g. 2024-01-02T03:04:05.678Z). + /// A timestamp at the start of a line marks the beginning of a new (possibly multi-line) message. + /// + private static readonly Regex MessageMarker = new( + @"(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)", + RegexOptions.Compiled); + + /// Size of the read buffer used while draining the log stream. + private const int ReadChunkChars = 4096; + + private readonly LogClient _logClient; + private readonly Action _toLog; + private readonly DateTimeOffset? _relevancyTimeLimit; + private readonly object _lock = new(); + + private CancellationTokenSource? _cts; + private Task? _streamingTask; + + internal StreamedLog(LogClient logClient, Action toLog, bool fromStart) + { + _logClient = logClient; + _toLog = toLog; + // When fromStart is false, ignore messages timestamped before this helper was created. + _relevancyTimeLimit = fromStart ? null : DateTimeOffset.UtcNow; + } + + /// Starts redirecting the log on a background task. Throws if already started. + public void Start() + { + lock (_lock) + { + if (_streamingTask is not null) + { + throw new InvalidOperationException("Log streaming is already active."); + } + + _cts = new CancellationTokenSource(); + _streamingTask = StreamLogAsync(_cts.Token); + } + } + + /// + /// Stops redirecting the log and waits for the background task to drain. A no-op if not started. + /// + public async Task StopAsync() + { + Task? task; + CancellationTokenSource? cts; + lock (_lock) + { + task = _streamingTask; + cts = _cts; + _streamingTask = null; + _cts = null; + } + + if (task is null || cts is null) + { + return; + } + + cts.Cancel(); + try + { + await task.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected: cancellation is how redirection is stopped. + } + finally + { + cts.Dispose(); + } + } + + /// + public async ValueTask DisposeAsync() => await StopAsync().ConfigureAwait(false); + + private async Task StreamLogAsync(CancellationToken cancellationToken) + { + using var stream = await _logClient.StreamAsync(new LogOptions { Raw = true }, cancellationToken).ConfigureAwait(false); + using var reader = new StreamReader(stream, Encoding.UTF8); + + var buffer = new StringBuilder(); + var chunk = new char[ReadChunkChars]; + int read; + while ((read = await reader.ReadAsync(chunk.AsMemory(0, chunk.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + buffer.Append(chunk, 0, read); + FlushMessages(buffer, flushRemainder: false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + } + + // Emit whatever is left when the stream ends or is stopped (possibly a message without a trailing newline). + FlushMessages(buffer, flushRemainder: true); + } + + /// + /// Emits every complete message currently in . A message runs from one + /// timestamp marker to the next; the final one is complete only when + /// is true (stream ended), otherwise it is kept in the buffer as a possibly-incomplete tail. + /// + private void FlushMessages(StringBuilder buffer, bool flushRemainder) + { + var text = buffer.ToString(); + var matches = MessageMarker.Matches(text); + if (matches.Count == 0) + { + if (flushRemainder) + { + EmitMessage(text, timestamp: null); + buffer.Clear(); + } + + return; + } + + var completeCount = flushRemainder ? matches.Count : matches.Count - 1; + for (var i = 0; i < completeCount; i++) + { + var start = matches[i].Groups[1].Index; + var end = i + 1 < matches.Count ? matches[i + 1].Groups[1].Index : text.Length; + EmitMessage(text.Substring(start, end - start), matches[i].Groups[1].Value); + } + + if (flushRemainder) + { + buffer.Clear(); + } + else + { + var lastStart = matches[matches.Count - 1].Groups[1].Index; + buffer.Clear(); + buffer.Append(text, lastStart, text.Length - lastStart); + } + } + + /// + /// Writes one message to the sink, skipping it when fromStart is disabled and the message's + /// timestamp predates this helper's creation. Blank messages are dropped. + /// + private void EmitMessage(string message, string? timestamp) + { + if (_relevancyTimeLimit is not null + && timestamp is not null + && DateTimeOffset.TryParse( + timestamp, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var logTime) + && logTime < _relevancyTimeLimit.Value) + { + return; + } + + var trimmed = message.Trim(); + if (trimmed.Length > 0) + { + _toLog(trimmed); + } + } +} diff --git a/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs b/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs index bcf359e..b12b7a4 100644 --- a/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs +++ b/tests/Apify.Client.Tests/Examples/LogRedirectionExample.cs @@ -1,20 +1,16 @@ using System; -using System.IO; using System.Threading.Tasks; using Apify.Client; namespace Apify.Client.Tests.Examples; -/// Run an Actor with log redirection turned on (stream the run's log). +/// Run an Actor with log redirection turned on: the run's live log is forwarded to a sink. public static class LogRedirectionExample { public static async Task RunAsync(ApifyClient client) { - var run = await client.Actor("apify/hello-world").StartAsync(); - // Wait for the run to finish so the full log is available, then stream it to stdout. - await client.Run(run.Id!).WaitForFinishAsync(120); - using var stream = await client.Run(run.Id!).GetStreamedLogAsync(); - using var reader = new StreamReader(stream); - Console.WriteLine(await reader.ReadToEndAsync()); + // The `log` argument redirects the run's live log to the given sink (here, stdout) for the + // duration of the wait — the client streams and forwards each complete log message as it arrives. + await client.Actor("apify/hello-world").CallAsync(null, null, 120, log: Console.WriteLine); } } diff --git a/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs index aa8e2ae..1bf05cd 100644 --- a/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/BuildIntegrationTests.cs @@ -17,6 +17,17 @@ public async Task ListBuilds() Assert.True(page.Total >= page.Items.Count); } + [SkippableFact] + public async Task DefaultBuild() + { + var client = RequireClient(); + // A public Store Actor always has a default build; resolve it and confirm the build is fetchable. + var buildClient = await client.Actor("apify/hello-world").DefaultBuildAsync(); + var build = await buildClient.GetAsync(); + Assert.NotNull(build); + Assert.NotNull(build!.Id); + } + [SkippableFact] public async Task BuildActorFlow() { diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs new file mode 100644 index 0000000..6e7e23e --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Offline tests for the auto-paging iterators: they must walk pages by offset until the reported total is +/// reached, sending the right per-page offset. +/// +[Trait("Category", "Unit")] +public sealed class AutoPagingTests +{ + private static readonly IReadOnlyDictionary Page1Headers = new Dictionary + { + ["X-Apify-Pagination-Total"] = "3", + ["X-Apify-Pagination-Offset"] = "0", + ["X-Apify-Pagination-Limit"] = "2", + }; + + private static readonly IReadOnlyDictionary Page2Headers = new Dictionary + { + ["X-Apify-Pagination-Total"] = "3", + ["X-Apify-Pagination-Offset"] = "2", + ["X-Apify-Pagination-Limit"] = "1", + }; + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task CollectionIterateWalksAllPagesByOffset() + { + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}") + .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"c\"}]}}"); + + var ids = new List(); + await foreach (var actor in Client(transport).Actors().IterateAsync()) + { + ids.Add(actor.Id); + } + + Assert.Equal(new[] { "a", "b", "c" }, ids); + Assert.Equal(2, transport.CallCount); + Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task CollectionIterateStopsAtLimit() + { + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"total\":10,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"); + + var ids = new List(); + await foreach (var actor in Client(transport).Actors().IterateAsync(new ActorListOptions { Limit = 2 })) + { + ids.Add(actor.Id); + } + + // Limit=2 is satisfied by the first page, so no second page is fetched even though total is 10. + Assert.Equal(new[] { "a", "b" }, ids); + Assert.Equal(1, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task DatasetIterateItemsWalksAllPagesByOffset() + { + var transport = new MockTransport() + .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", Page1Headers) + .QueueResponse(200, "[{\"i\":3}]", Page2Headers); + + var values = new List(); + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + { + values.Add(item!["i"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, values); + Assert.Equal(2, transport.CallCount); + Assert.Contains("/datasets/ds1/items", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index 4acf792..88e34f8 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -21,6 +21,82 @@ public sealed class RequestShapeTests HttpTransport = transport, }); + [Fact] + public async Task LastRunDatasetForwardsStatusAndOrigin() + { + var transport = new MockTransport().QueueResponse(200, "[]"); + await Client(transport).Actor("me/act") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .Dataset() + .ListItemsAsync(new DatasetListItemsOptions()); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actors/me~act/runs/last/dataset/items", uri, StringComparison.Ordinal); + Assert.Contains("status=SUCCEEDED", uri, StringComparison.Ordinal); + Assert.Contains("origin=API", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunKeyValueStoreForwardsStatusAndOrigin() + { + var transport = new MockTransport().QueueResponse(200, "value"); + await Client(transport).Actor("me/act") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .KeyValueStore() + .GetRecordAsync("OUTPUT"); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actors/me~act/runs/last/key-value-store/records/OUTPUT", uri, StringComparison.Ordinal); + Assert.Contains("status=SUCCEEDED", uri, StringComparison.Ordinal); + Assert.Contains("origin=API", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunRequestQueueForwardsStatusAndOrigin() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + await Client(transport).Actor("me/act") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .RequestQueue() + .ListHeadAsync(); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actors/me~act/runs/last/request-queue/head", uri, StringComparison.Ordinal); + Assert.Contains("status=SUCCEEDED", uri, StringComparison.Ordinal); + Assert.Contains("origin=API", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunLogForwardsStatusAndOrigin() + { + var transport = new MockTransport().QueueResponse(200, "log output"); + await Client(transport).Actor("me/act") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .Log() + .GetAsync(); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actors/me~act/runs/last/log", uri, StringComparison.Ordinal); + Assert.Contains("status=SUCCEEDED", uri, StringComparison.Ordinal); + Assert.Contains("origin=API", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunDatasetPushItemsForwardsStatusAndOrigin() + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Actor("me/act") + .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) + .Dataset() + .PushItemsAsync(new { hello = "world" }); + + var request = transport.LastRequest; + Assert.Equal("POST", request.Method); + Assert.Contains("/actors/me~act/runs/last/dataset/items", request.Uri, StringComparison.Ordinal); + Assert.Contains("status=SUCCEEDED", request.Uri, StringComparison.Ordinal); + Assert.Contains("origin=API", request.Uri, StringComparison.Ordinal); + } + [Fact] public async Task RunChargeSendsBodyAndIdempotencyKey() { diff --git a/tests/Apify.Client.Tests/Unit/StreamedLogTests.cs b/tests/Apify.Client.Tests/Unit/StreamedLogTests.cs new file mode 100644 index 0000000..8a12699 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/StreamedLogTests.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class StreamedLogTests +{ + private const string LogBody = + "2024-01-02T03:04:05.678Z first message\n" + + "2024-01-02T03:04:06.789Z second message\n" + + "2024-01-02T03:04:07.890Z third message\n"; + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static async Task> CollectAsync(ApifyClient client, bool fromStart) + { + var messages = new List(); + var streamedLog = client.Run("run1").GetStreamedLog( + m => + { + lock (messages) + { + messages.Add(m); + } + }, + fromStart); + + streamedLog.Start(); + // The mock returns a finite stream, so the background task drains and completes on its own; poll + // until it has, then stop (which awaits the already-finished task). + for (var i = 0; i < 200; i++) + { + await Task.Delay(5); + lock (messages) + { + if (messages.Count >= 3) + { + break; + } + } + } + + await streamedLog.StopAsync(); + return messages; + } + + [Fact] + public async Task RedirectsEachCompleteMessageToSink() + { + var transport = new MockTransport().QueueResponse(200, LogBody); + var messages = await CollectAsync(Client(transport), fromStart: true); + + Assert.Equal(3, messages.Count); + Assert.StartsWith("2024-01-02T03:04:05.678Z", messages[0], StringComparison.Ordinal); + Assert.Contains("first message", messages[0], StringComparison.Ordinal); + Assert.Contains("third message", messages[2], StringComparison.Ordinal); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/actor-runs/run1/log", uri, StringComparison.Ordinal); + Assert.Contains("stream=1", uri, StringComparison.Ordinal); + Assert.Contains("raw=1", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task FromStartFalseSkipsMessagesOlderThanCreation() + { + // All log lines are timestamped in the past, so with fromStart=false they are all filtered out. + var transport = new MockTransport().QueueResponse(200, LogBody); + var messages = new List(); + var streamedLog = Client(transport).Run("run1").GetStreamedLog( + m => + { + lock (messages) + { + messages.Add(m); + } + }, + fromStart: false); + + streamedLog.Start(); + await streamedLog.StopAsync(); + + lock (messages) + { + Assert.Empty(messages); + } + } + + [Fact] + public async Task StartTwiceThrows() + { + var transport = new MockTransport().QueueResponse(200, LogBody); + var streamedLog = Client(transport).Run("run1").GetStreamedLog(_ => { }); + streamedLog.Start(); + Assert.Throws(() => streamedLog.Start()); + await streamedLog.StopAsync(); + } +} From 67aa1c7607e70d34e6dde35b389e448e9ffa7c15 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:19:13 +0000 Subject: [PATCH 3/5] test/docs: close verification-review gaps in .NET client Add offline unit tests for 429 retry, doNotRetryTimeouts, per-attempt timeout doubling/cap, retry exhaustion, bounded-parallelism over the limit, CSV/double query encoding, KVS binary write shape, and IsNotFound/null-omit branches (unit tests 50 -> 65). Align ApiSpecVersion + CHANGELOG to v2-2026-07-01T115402Z (match the sibling clients). Document the public namespaces (incl. Apify.Client.Resources) and add using System; to Console snippets; drop the unused id-token permission from the publish workflow. --- CHANGELOG.md | 2 +- README.md | 3 + docs/README.md | 33 +++++ docs/actors.md | 2 + docs/builds.md | 1 + docs/examples.md | 2 +- docs/storages.md | 8 +- docs/tasks.md | 1 + src/Apify.Client/ApifyClientVersion.cs | 2 +- .../Options/StorageListOptions.cs | 2 +- .../Unit/BatchAddRequestsTests.cs | 23 +++ .../Unit/HttpClientTests.cs | 137 ++++++++++++++++++ .../Apify.Client.Tests/Unit/MockTransport.cs | 14 +- .../Unit/RequestShapeTests.cs | 78 ++++++++++ 14 files changed, 297 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd4a7c..fa3b4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.1.0 -- Initial .NET client for the Apify API (spec `v2-2026-07-02T131926Z`). +- Initial .NET client for the Apify API (spec `v2-2026-07-01T115402Z`). - Resource clients for Actors, Actor versions and environment variables, builds, runs, datasets, key-value stores, request queues, tasks, schedules, webhooks, webhook dispatches, the Apify Store, users, and logs. diff --git a/README.md b/README.md index 668064c..6613541 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,10 @@ dotnet add package Apify.Client ## Quick start +`ImplicitUsings` is disabled in this repository, so every `using` (even `System`) is listed explicitly. + ```csharp +using System; using Apify.Client; var client = new ApifyClient("my-api-token"); diff --git a/docs/README.md b/docs/README.md index 9ddc4ac..466ea01 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,7 +36,11 @@ dotnet add package Apify.Client ## Quick start +All snippets in this documentation assume `ImplicitUsings` is disabled (the repository's convention), +so every `using` — even `System` — is listed explicitly and appears before any top-level statement. + ```csharp +using System; using Apify.Client; var client = new ApifyClient("my-api-token"); @@ -57,6 +61,33 @@ e.g. `new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN"))`. Get your API token from the [Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). +## Namespaces + +The public API is spread across a small set of namespaces. Because `ImplicitUsings` is disabled, add +the `using` directives for whichever ones a file references: + +| Namespace | What lives here | +|---|---| +| `Apify.Client` | The entry point (`ApifyClient`), `ApifyClientOptions`, and `ApifyClientVersion`. | +| `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, the `…CollectionClient` types, etc. | +| `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | +| `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `DatasetListItemsOptions`, `ListOptions`, `SetRecordOptions`, `StorageListOptions`, etc. | +| `Apify.Client.Exceptions` | `ApifyApiException` and `ApifyTransportException`. | +| `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | + +Fluent chains such as `client.Actor("id").Builds()` compile with only `using Apify.Client;` because the +intermediate types are inferred. You only need `using Apify.Client.Resources;` when you name a resource +client type explicitly — e.g. storing one in a variable or field: + +```csharp +using Apify.Client; +using Apify.Client.Resources; + +var client = new ApifyClient("my-api-token"); +BuildClient build = await client.Actor("apify/hello-world").DefaultBuildAsync(); +RunClient lastRun = client.Actor("apify/hello-world").LastRun(); +``` + ## Configuration Pass an `ApifyClientOptions` to configure non-default settings: @@ -115,6 +146,7 @@ throwing). Other API failures are thrown as `Apify.Client.Exceptions.ApifyApiExc the HTTP status, API error `Type`, message, attempt count, and request method/path: ```csharp +using System; using Apify.Client; using Apify.Client.Exceptions; @@ -136,6 +168,7 @@ catch (ApifyApiException e) built against. ```csharp +using System; using Apify.Client; Console.WriteLine($"{ApifyClientVersion.ClientVersion} / {ApifyClientVersion.ApiSpecVersion}"); diff --git a/docs/actors.md b/docs/actors.md index a54de5b..25a4e1b 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -12,6 +12,7 @@ Access the Actor collection with `client.Actors()` and a specific Actor with `cl - `CreateAsync(object actor)` — create an Actor from any JSON-serializable definition. Returns `Actor`. ```csharp +using System; using Apify.Client; using Apify.Client.Options; @@ -44,6 +45,7 @@ foreach (var actor in page.Items) `MaxTotalChargeUsd`, `ContentType`, `RestartOnError`, `ForcePermissionLevel`, `Webhooks`. ```csharp +using System; using Apify.Client; using Apify.Client.Options; diff --git a/docs/builds.md b/docs/builds.md index cf01e06..f2857c3 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -20,6 +20,7 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil - `Log()` → `LogClient`. ```csharp +using System; using Apify.Client; using Apify.Client.Options; diff --git a/docs/examples.md b/docs/examples.md index 22dcfba..23c7a52 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -42,7 +42,7 @@ Console.WriteLine("Dataset items: " + items.Count); var store = await client.KeyValueStores().GetOrCreateAsync("example-kvs"); await client.KeyValueStore(store.Id!).SetRecordJsonAsync("OUTPUT", new { answer = 42 }); var record = await client.KeyValueStore(store.Id!).GetRecordAsync("OUTPUT"); -// GetRecordAsync returns the raw bytes; decode JSON/text records with UTF-8. +// GetRecordAsync returns a record whose .Value is the raw bytes; decode JSON/text with UTF-8. var recordText = record is null ? string.Empty : Encoding.UTF8.GetString(record.Value); Console.WriteLine("KVS record: " + recordText); diff --git a/docs/storages.md b/docs/storages.md index 6aab6c7..01e7d04 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -15,11 +15,11 @@ from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). `client.Datasets()` / `client.Dataset(id)`. - `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. -- `ListItemsAsync(DatasetListItemsOptions?)` → `PaginationList` (one page; pagination via +- `ListItemsAsync(DatasetListItemsOptions? = null)` → `PaginationList` (one page; pagination via response headers). -- `IterateItemsAsync(DatasetListItemsOptions?)` → `IAsyncEnumerable` — lazily iterate every +- `IterateItemsAsync(DatasetListItemsOptions? = null)` → `IAsyncEnumerable` — lazily iterate every item across pages, fetching each page on demand. -- `DownloadItemsAsync(DownloadItemsFormat, DatasetDownloadOptions?)` → serialized items as `byte[]` +- `DownloadItemsAsync(DownloadItemsFormat, DatasetDownloadOptions? = null)` → serialized items as `byte[]` (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). - `PushItemsAsync(object items)` — push one object or an array of objects. - `GetStatisticsAsync()` → `JsonObject?`. @@ -47,7 +47,7 @@ Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the - `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. - `ListKeysAsync(ListKeysOptions?)` → `KeyValueStoreKeysPage`. - `RecordExistsAsync(key)` → `bool`. -- `GetRecordAsync(key, GetRecordOptions?)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a +- `GetRecordAsync(key, GetRecordOptions? = null)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a `byte[]` of the record's raw bytes (so binary records survive intact); decode it according to `KeyValueStoreRecord.ContentType` — e.g. `Encoding.UTF8.GetString(record.Value)` for text, or `JsonSerializer.Deserialize(record.Value)` for JSON. diff --git a/docs/tasks.md b/docs/tasks.md index f29b808..e9ee3c4 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -22,6 +22,7 @@ a specific task with `client.Task(id)`. The model is named `ActorTask` (not `Task`) to avoid colliding with `System.Threading.Tasks.Task`. ```csharp +using System; using Apify.Client; var client = new ApifyClient("my-api-token"); diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index 7d5e43f..d636ae8 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -20,5 +20,5 @@ public static class ApifyClientVersion /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-07-02T131926Z"; + public const string ApiSpecVersion = "v2-2026-07-01T115402Z"; } diff --git a/src/Apify.Client/Options/StorageListOptions.cs b/src/Apify.Client/Options/StorageListOptions.cs index caf3043..1428a11 100644 --- a/src/Apify.Client/Options/StorageListOptions.cs +++ b/src/Apify.Client/Options/StorageListOptions.cs @@ -21,7 +21,7 @@ public sealed class StorageListOptions /// If true, include unnamed storages in the result. public bool? Unnamed { get; init; } - /// Filter by ownership (e.g. OWNED / ACCESSIBLE). + /// Filter by ownership: ownedByMe or sharedWithMe. public string? Ownership { get; init; } internal void AppendTo(QueryParams q) diff --git a/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs b/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs index 5d95a5d..6cf5d8d 100644 --- a/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs +++ b/tests/Apify.Client.Tests/Unit/BatchAddRequestsTests.cs @@ -240,6 +240,29 @@ public async Task ParallelResultsMergedInInputOrder() } } + [Fact] + public async Task BoundedParallelismCapsConcurrencyBelowChunkCount() + { + var transport = new MockTransport { EchoBatchProcessed = true, ArtificialDelayMillis = 40 }; + var requests = new List(); + for (var i = 0; i < 100; i++) // 100 requests -> 4 chunks of 25 + { + requests.Add(new RequestQueueRequest("https://x/" + i, "k" + i)); + } + + // Fewer permits than chunks, so the SemaphoreSlim bound must actually constrain concurrency. + const int maxParallel = 2; + var options = new BatchAddRequestsOptions(maxParallel: maxParallel, minDelayBetweenUnprocessedRequestsRetriesMillis: 0); + var result = await Client(transport).RequestQueue("q1").BatchAddRequestsAsync(requests, false, options); + + Assert.Equal(100, result.ProcessedRequests.Count); + Assert.Equal(4, transport.CallCount); + Assert.True(transport.MaxObservedConcurrency > 1, "expected concurrent dispatch"); + Assert.True( + transport.MaxObservedConcurrency <= maxParallel, + $"observed concurrency {transport.MaxObservedConcurrency} exceeded the bound {maxParallel}"); + } + [Fact] public async Task OversizedSingleRequestThrows() { diff --git a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs index 41f8aa0..f575aa5 100644 --- a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs +++ b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs @@ -153,4 +153,141 @@ public async Task SafeIdReplacesFirstSlashWithTilde() await Client(transport).Actor("apify/hello-world").GetAsync(); Assert.Contains("/actors/apify~hello-world", transport.LastRequest.Uri, StringComparison.Ordinal); } + + [Fact] + public async Task RateLimitIsRetriedThenSucceeds() + { + // 429 (rate limit) must be retried just like a 5xx, then succeed on the next attempt. + var transport = new MockTransport() + .QueueResponse(429, "{\"error\":{\"type\":\"rate-limit-exceeded\",\"message\":\"slow down\"}}") + .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + + var actor = await Client(transport).Actor("x").GetAsync(); + Assert.Equal("ok", actor!.Id); + Assert.Equal(2, transport.CallCount); + } + + [Fact] + public async Task ServerErrorsThrowAfterRetriesAreExhausted() + { + // maxRetries=2 => 3 attempts; every attempt is a 5xx, so the last error is thrown. + var options = new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + MaxRetries = 2, + HttpTransport = new MockTransport() + .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}") + .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}") + .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}"), + }; + var transport = (MockTransport)options.HttpTransport; + + var ex = await Assert.ThrowsAsync(() => new ApifyClient(options).Actor("x").GetAsync()); + Assert.Equal(500, ex.StatusCode); + Assert.Equal(3, ex.Attempt); + Assert.Equal(3, transport.CallCount); + } + + [Fact] + public async Task TransportErrorsThrowAfterRetriesAreExhausted() + { + // maxRetries=2 => 3 attempts; every attempt is a transport failure, so it is finally rethrown. + var options = new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + MaxRetries = 2, + HttpTransport = new MockTransport().QueueError().QueueError().QueueError(), + }; + var transport = (MockTransport)options.HttpTransport; + + await Assert.ThrowsAsync(() => new ApifyClient(options).Actor("x").GetAsync()); + Assert.Equal(3, transport.CallCount); + } + + [Fact] + public async Task AttemptTimeoutDoublesPerRetryAndCapsAtOverallBudget() + { + // Per-call base timeout (5s) is below the overall budget (100s), so each retry doubles the + // per-attempt timeout until it would exceed the overall budget, at which point it is capped. + var transport = new MockTransport(); + for (var i = 0; i < 6; i++) + { + transport.QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}"); + } + + var client = new ApifyClient(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 100, // overall budget + MaxRetries = 5, // 6 attempts total + HttpTransport = transport, + }); + + // A per-queue timeout of 5s becomes the base per-attempt timeout that then doubles. + await Assert.ThrowsAsync( + () => client.RequestQueue("q1", new Options.RequestQueueClientOptions { TimeoutSecs = 5 }).ListHeadAsync(5)); + + Assert.Equal(new[] { 5.0, 10.0, 20.0, 40.0, 80.0, 100.0 }, transport.Timeouts); + } + + [Fact] + public async Task TimeoutIsNotRetriedWhenDoNotRetryTimeoutsIsSet() + { + // A single timeout, then a success that must never be reached because retrying is opted out. + var transport = new MockTransport().QueueError(timeout: true).QueueResponse(200, string.Empty); + + await Assert.ThrowsAsync(() => Client(transport) + .KeyValueStore("s1") + .SetRecordAsync("k", new byte[] { 1, 2, 3 }, "application/octet-stream", new SetRecordOptions { DoNotRetryTimeouts = true })); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task TimeoutIsRetriedWhenDoNotRetryTimeoutsIsNotSet() + { + // With the default (DoNotRetryTimeouts=false) a timeout is retryable, so the retry succeeds. + var transport = new MockTransport().QueueError(timeout: true).QueueResponse(200, string.Empty); + + await Client(transport) + .KeyValueStore("s1") + .SetRecordAsync("k", new byte[] { 1, 2, 3 }, "application/octet-stream", new SetRecordOptions()); + Assert.Equal(2, transport.CallCount); + } + + [Fact] + public async Task NullQueryParamsAreOmitted() + { + // Only Limit is set; every other (null) option must be absent from the query string entirely. + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + await Client(transport).Actors().ListAsync(new ActorListOptions { Limit = 5 }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("limit=5", uri, StringComparison.Ordinal); + Assert.DoesNotContain("offset=", uri, StringComparison.Ordinal); + Assert.DoesNotContain("desc=", uri, StringComparison.Ordinal); + Assert.DoesNotContain("my=", uri, StringComparison.Ordinal); + Assert.DoesNotContain("sortBy=", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RecordExistsReturnsFalseOnHeadNotFound() + { + // IsNotFound treats any 404 to a HEAD request as "not found" even without an error type. + var transport = new MockTransport().QueueResponse(404, string.Empty); + Assert.False(await Client(transport).KeyValueStore("s1").RecordExistsAsync("missing")); + Assert.Equal("HEAD", transport.LastRequest.Method); + } + + [Fact] + public async Task GetRecordReturnsNullOnRecordOrTokenNotFound() + { + // The "record-or-token-not-found" error type is one of the IsNotFound branches -> null, not throw. + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-or-token-not-found\",\"message\":\"nope\"}}"); + Assert.Null(await Client(transport).KeyValueStore("s1").GetRecordAsync("missing")); + } } diff --git a/tests/Apify.Client.Tests/Unit/MockTransport.cs b/tests/Apify.Client.Tests/Unit/MockTransport.cs index 851dafc..7c1d47d 100644 --- a/tests/Apify.Client.Tests/Unit/MockTransport.cs +++ b/tests/Apify.Client.Tests/Unit/MockTransport.cs @@ -17,11 +17,12 @@ public sealed class RecordedRequest { private readonly Dictionary _headers; - internal RecordedRequest(string method, string uri, string body, Dictionary headers) + internal RecordedRequest(string method, string uri, string body, byte[] bodyBytes, Dictionary headers) { Method = method; Uri = uri; Body = body; + BodyBytes = bodyBytes; _headers = headers; } @@ -31,6 +32,9 @@ internal RecordedRequest(string method, string uri, string body, DictionaryThe raw (un-decoded) request body bytes, so binary write paths can be asserted verbatim. + public byte[] BodyBytes { get; } + public string Header(string name) => _headers.TryGetValue(name, out var value) ? value : string.Empty; } @@ -90,6 +94,7 @@ public async Task SendAsync(HttpRequestMessage request, Tim } var body = string.Empty; + var bodyBytes = Array.Empty(); if (request.Content is not null) { foreach (var header in request.Content.Headers) @@ -97,12 +102,15 @@ public async Task SendAsync(HttpRequestMessage request, Tim headers[header.Key] = string.Join(",", header.Value); } - body = await request.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + // Read the raw bytes so binary bodies can be asserted verbatim; keep a UTF-8 decode for the + // string-body assertions (equivalent to ReadAsStringAsync for text content). + bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + body = System.Text.Encoding.UTF8.GetString(bodyBytes); } lock (_lock) { - Received.Add(new RecordedRequest(request.Method.Method, request.RequestUri?.ToString() ?? string.Empty, body, headers)); + Received.Add(new RecordedRequest(request.Method.Method, request.RequestUri?.ToString() ?? string.Empty, body, bodyBytes, headers)); Timeouts.Add(timeout.TotalSeconds); _inFlight++; MaxObservedConcurrency = Math.Max(MaxObservedConcurrency, _inFlight); diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index 88e34f8..3e399f6 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.Threading.Tasks; using System.Text.Json.Nodes; using Apify.Client.Options; @@ -207,4 +208,81 @@ public async Task UpdateLimitsPutsToMeLimits() Assert.Contains("/users/me/limits", request.Uri, StringComparison.Ordinal); Assert.Equal(100, JsonNode.Parse(request.Body)!["maxMonthlyUsageUsd"]!.GetValue()); } + + [Fact] + public async Task DatasetListItemsJoinsMultiValueParamsAsCsv() + { + // fields/omit/unwind are list parameters: each is joined with a comma (URL-encoded as %2C). + var transport = new MockTransport().QueueResponse(200, "[]"); + await Client(transport).Dataset("ds1").ListItemsAsync(new DatasetListItemsOptions + { + Fields = new[] { "name", "url" }, + Omit = new[] { "secret" }, + Unwind = new[] { "results" }, + }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("fields=name%2Curl", uri, StringComparison.Ordinal); + Assert.Contains("omit=secret", uri, StringComparison.Ordinal); + Assert.Contains("unwind=results", uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RunListJoinsStatusAsCsv() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + await Client(transport).Runs().ListAsync(null, new RunListOptions { Status = new[] { "SUCCEEDED", "RUNNING" } }); + + Assert.Contains("status=SUCCEEDED%2CRUNNING", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task ListRequestsJoinsFilterAsCsv() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + await Client(transport).RequestQueue("q1").ListRequestsAsync(new ListRequestsOptions + { + Filter = new[] { ListRequestsOptions.FilterLocked, ListRequestsOptions.FilterPending }, + }); + + Assert.Contains("filter=locked%2Cpending", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() + { + // Under a culture that uses a comma decimal separator, the double must still be sent with a '.'. + var original = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + try + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + await Client(transport).Actor("act").StartAsync(null, new ActorStartOptions { MaxTotalChargeUsd = 12.5 }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("maxTotalChargeUsd=12.5", uri, StringComparison.Ordinal); + Assert.DoesNotContain("12%2C5", uri, StringComparison.Ordinal); // would appear if the comma culture leaked in + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Fact] + public async Task SetRecordSendsRawBytesWithVerbatimContentType() + { + // A binary write must send the bytes verbatim (incl. 0xFF) as ByteArrayContent and set the + // content type exactly as given, without appending "; charset=...". + var bytes = new byte[] { 0x00, 0xFF, 0x10, 0x7F }; + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).KeyValueStore("s1").SetRecordAsync("OUTPUT", bytes, "application/octet-stream"); + + var request = transport.LastRequest; + Assert.Equal("PUT", request.Method); + Assert.Contains("/key-value-stores/s1/records/OUTPUT", request.Uri, StringComparison.Ordinal); + Assert.Equal("application/octet-stream", request.Header("Content-Type")); + Assert.DoesNotContain("charset", request.Header("Content-Type"), StringComparison.OrdinalIgnoreCase); + Assert.Equal(bytes, request.BodyBytes); + } } From c115c6c76c68efb1021870b1bc1f674782e886bb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 00:21:54 +0000 Subject: [PATCH 4/5] chore: dev-loop compliance with updated requirements - Bump ApiSpecVersion + CHANGELOG to v2-2026-07-02T131926Z (current latest info.version; matches the Java/PHP/Rust/Go siblings). - Publish via NuGet API key (NUGET_API_KEY repo secret) instead of Trusted Publishing OIDC: per the updated requirement, Trusted Publisher is used only when officially supported, and NuGet.org Trusted Publishing is not yet GA. Adds a fail-fast guard when the secret is missing; drops id-token permission. --- .github/workflows/dotnet-publish.yml | 28 ++++++++++++++------------ CHANGELOG.md | 4 ++-- src/Apify.Client/ApifyClientVersion.cs | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml index 6b9732c..1378eea 100644 --- a/.github/workflows/dotnet-publish.yml +++ b/.github/workflows/dotnet-publish.yml @@ -5,10 +5,10 @@ name: Publish .NET client # of truth in src/Apify.Client/Apify.Client.csproj (). This workflow packs the library, # pushes it to NuGet.org, tags the release, and creates the GitHub release. # -# Publishing uses NuGet.org Trusted Publishing (OIDC): the NuGet/login action exchanges a short-lived -# GitHub OIDC token for a temporary NuGet API key just before the push, so no long-lived API key is -# stored in the repo. The only repository secret needed is NUGET_USER (the nuget.org account/profile -# name that owns the trusted-publishing policy). +# Publishing uses a NuGet API key read from the NUGET_API_KEY repository secret. NuGet.org Trusted +# Publishing (OIDC) is not used because, per the official docs, it is still being rolled out gradually +# and is not generally available, so it is not an officially supported publishing mechanism we can rely +# on. If/when Trusted Publishing reaches GA, switch to the NuGet/login OIDC flow and drop this secret. on: workflow_dispatch: inputs: @@ -24,7 +24,6 @@ concurrency: permissions: contents: write # create the tagged GitHub release - id-token: write # NuGet Trusted Publishing exchanges this OIDC token for a temporary API key jobs: publish: @@ -84,19 +83,22 @@ jobs: - name: Pack run: dotnet pack src/Apify.Client/Apify.Client.csproj --configuration Release --no-build --output ./artifacts - # Exchange the GitHub OIDC token for a short-lived NuGet API key (Trusted Publishing). Runs - # immediately before the push because the temporary key is valid for only ~1 hour. - - name: NuGet login (Trusted Publishing OIDC) + # Fail fast if the publish secret is missing so a run cannot silently pass without publishing. + - name: Require NUGET_API_KEY secret if: ${{ github.event.inputs.dry_run != 'true' }} - id: nuget_login - uses: NuGet/login@v1 - with: - user: ${{ secrets.NUGET_USER }} + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + run: | + if [ -z "${NUGET_API_KEY}" ]; then + echo "::error::NUGET_API_KEY secret is empty or missing; cannot publish to NuGet.org." + exit 1 + fi - name: Push to NuGet if: ${{ github.event.inputs.dry_run != 'true' }} env: - NUGET_API_KEY: ${{ steps.nuget_login.outputs.NUGET_API_KEY }} + # The NuGet.org API key is stored as a repository secret. + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} run: | dotnet nuget push "./artifacts/*.nupkg" \ --api-key "${NUGET_API_KEY}" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3b4da..3fd608f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.1.0 -- Initial .NET client for the Apify API (spec `v2-2026-07-01T115402Z`). +- Initial .NET client for the Apify API (spec `v2-2026-07-02T131926Z`). - Resource clients for Actors, Actor versions and environment variables, builds, runs, datasets, key-value stores, request queues, tasks, schedules, webhooks, webhook dispatches, the Apify Store, users, and logs. @@ -42,4 +42,4 @@ HMAC-SHA256 storage URL signing. - Public `ApifyClientVersion.ClientVersion` and `ApifyClientVersion.ApiSpecVersion` constants. - Integration test suite, documentation with runnable examples, and CI workflows for integration - tests and publishing (NuGet.org Trusted Publishing via OIDC). + tests and publishing (manual NuGet.org publish using an API key from a repository secret). diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index d636ae8..7d5e43f 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -20,5 +20,5 @@ public static class ApifyClientVersion /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-07-01T115402Z"; + public const string ApiSpecVersion = "v2-2026-07-02T131926Z"; } From d84efd7a7011675ed730adf3f0dc844a18015324 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 06:29:02 +0000 Subject: [PATCH 5/5] test/docs: add run update/delete + public user tests, expand docs - tests: add ActorRunIntegrationTests.UpdateAndDeleteRun (start, wait terminal, UpdateAsync statusMessage, DeleteAsync) and UserIntegrationTests.GetPublicUserById (Me() then User(id).GetAsync(), asserting on spec-guaranteed Username since UserPublicInfo has no id) - docs: add docs/models.md data-model property reference; add option-field descriptions and field tables (DownloadItemsFormat, DatasetDownloadOptions, ActorBuildOptions, Metamorph/RunResurrect/RunCharge/ValidateInput/LastRun options, ApifyApiException); document BatchDeleteRequestsAsync + WithClientKey; normalize optional-parameter notation; add method return types - changelog: mention docs/models.md Spec v2-2026-07-02T131926Z (unchanged). No public-interface changes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_017sJGjCxFP3bpbeF4LsVqK3 --- CHANGELOG.md | 5 +- docs/README.md | 18 +- docs/actors.md | 34 ++- docs/builds.md | 10 + docs/examples.md | 1 - docs/misc.md | 38 ++- docs/models.md | 273 ++++++++++++++++++ docs/runs.md | 24 +- docs/schedules.md | 6 +- docs/storages.md | 132 ++++++--- docs/tasks.md | 15 +- docs/webhooks.md | 5 +- .../Integration/ActorRunIntegrationTests.cs | 19 ++ .../Integration/UserIntegrationTests.cs | 20 ++ 14 files changed, 536 insertions(+), 64 deletions(-) create mode 100644 docs/models.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fd608f..4d3f24b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,5 +41,6 @@ automatic retries with exponential backoff and jitter, growing per-attempt timeouts, and HMAC-SHA256 storage URL signing. - Public `ApifyClientVersion.ClientVersion` and `ApifyClientVersion.ApiSpecVersion` constants. -- Integration test suite, documentation with runnable examples, and CI workflows for integration - tests and publishing (manual NuGet.org publish using an API key from a repository secret). +- Integration test suite, documentation with runnable examples, a data-model property reference + (`docs/models.md`), and CI workflows for integration tests and publishing (manual NuGet.org publish + using an API key from a repository secret). diff --git a/docs/README.md b/docs/README.md index 466ea01..30f6e4b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ All API calls are asynchronous and return `Task`/`Task`; every method accepts - [Schedules](schedules.md) - [Webhooks](webhooks.md) — webhooks and dispatches. - [Misc](misc.md) — the Apify Store, users, logs. +- [Data models](models.md) — property reference for every returned model (`Actor`, `ActorRun`, `Build`, `PaginationList`, …). - [Examples](examples.md) — runnable end-to-end examples. ## Requirements @@ -84,7 +85,7 @@ using Apify.Client; using Apify.Client.Resources; var client = new ApifyClient("my-api-token"); -BuildClient build = await client.Actor("apify/hello-world").DefaultBuildAsync(); +BuildClient defaultBuild = await client.Actor("apify/hello-world").DefaultBuildAsync(); RunClient lastRun = client.Actor("apify/hello-world").LastRun(); ``` @@ -161,6 +162,21 @@ catch (ApifyApiException e) } ``` +`ApifyApiException` members: + +| Member | Type | Description | +|---|---|---| +| `StatusCode` | `int` | HTTP status code of the error response. | +| `Type` | `string?` | Machine-readable API error type (e.g. `record-not-found`). | +| `ApiMessage` | `string` | Raw error message from the API, without the status/type prefix. | +| `Attempt` | `int` | 1-based number of the API-call attempt that produced the error. | +| `HttpMethod` | `string` | HTTP method of the failed call (e.g. `GET`, `POST`). | +| `Path` | `string` | API endpoint path (URL excluding origin). | +| `ErrorData` | `JsonObject?` | Additional structured error data from the API, if any. | + +`ApifyTransportException` is thrown instead when the request never reaches the API (network failure, +timeout, or DNS error) after all retries are exhausted. + ## Versioning - `Apify.Client.ApifyClientVersion.ClientVersion` — the semantic version of this library. diff --git a/docs/actors.md b/docs/actors.md index 25a4e1b..83e7a2b 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -6,11 +6,21 @@ Access the Actor collection with `client.Actors()` and a specific Actor with `cl ## Collection — `client.Actors()` - `ListAsync(ActorListOptions? options = null)` — list the account's Actors (one page). Returns - `PaginationList`. Options: `Offset`, `Limit`, `Desc`, `My`, `SortBy`. + `PaginationList`. - `IterateAsync(ActorListOptions? options = null)` → `IAsyncEnumerable` — lazily iterate every Actor across pages, fetching each page on demand. - `CreateAsync(object actor)` — create an Actor from any JSON-serializable definition. Returns `Actor`. +`ActorListOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `Offset` | `int?` | Number of Actors to skip from the start. | +| `Limit` | `int?` | Maximum number of Actors to return in the page. | +| `Desc` | `bool?` | Sort newest-first when `true`. | +| `My` | `bool?` | Return only Actors owned by the current account when `true`. | +| `SortBy` | `string?` | Field to sort by (e.g. `createdAt`, `lastRunStartedAt`). | + ```csharp using System; using Apify.Client; @@ -41,8 +51,20 @@ foreach (var actor in page.Items) - `Version(string versionNumber)` / `Versions()` — Actor versions. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. -`ActorStartOptions` fields: `Build`, `MemoryMbytes`, `TimeoutSecs`, `WaitForFinish`, `MaxItems`, -`MaxTotalChargeUsd`, `ContentType`, `RestartOnError`, `ForcePermissionLevel`, `Webhooks`. +`ActorStartOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `Build` | `string?` | Tag or number of the Actor build to run (e.g. `latest`). | +| `MemoryMbytes` | `int?` | Memory limit for the run, in megabytes. | +| `TimeoutSecs` | `int?` | Hard run timeout in seconds (`0` means no limit). | +| `WaitForFinish` | `int?` | Seconds the *start* request itself blocks server-side waiting for the run (max 60). | +| `MaxItems` | `int?` | Maximum number of dataset items the (pay-per-result) run may produce. | +| `MaxTotalChargeUsd` | `double?` | Maximum total USD the run is allowed to charge. | +| `ContentType` | `string?` | Content type of the `input` body (defaults to `application/json`). | +| `RestartOnError` | `bool?` | Automatically restart the run's container if it exits with an error. | +| `ForcePermissionLevel` | `string?` | Override the Actor's permission level (`LIMITED_PERMISSIONS`/`FULL_PERMISSIONS`). | +| `Webhooks` | `object?` | Ad-hoc webhooks (any JSON-serializable list) to attach to this run. | ```csharp using System; @@ -57,6 +79,12 @@ var run = await client.Actor("apify/hello-world").CallAsync( Console.WriteLine(run.Status); ``` +`ValidateInputOptions` fields: `Build` (`string?`, the Actor build whose input schema to validate +against) and `ContentType` (`string?`, content type of the input; defaults to `application/json`). + +`LastRunOptions` fields: `Status` (`string?`, only consider the last run with this status, e.g. +`SUCCEEDED`) and `Origin` (`string?`, only consider the last run started from this origin, e.g. `API`). + ## Versions and environment variables ```csharp diff --git a/docs/builds.md b/docs/builds.md index f2857c3..8c06330 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -19,6 +19,16 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil - `GetOpenApiDefinitionAsync()` → `JsonObject?`. - `Log()` → `LogClient`. +Builds are created with `client.Actor(id).BuildAsync(string versionNumber, ActorBuildOptions? options = null)`. +`ActorBuildOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `BetaPackages` | `bool?` | Build with beta versions of the Apify SDK/packages. | +| `Tag` | `string?` | Build tag to apply to the resulting image (e.g. `latest`). | +| `UseCache` | `bool?` | Reuse cached Docker layers to speed up the build. | +| `WaitForFinish` | `int?` | Seconds the build request blocks server-side waiting for completion (max 60). | + ```csharp using System; using Apify.Client; diff --git a/docs/examples.md b/docs/examples.md index 23c7a52..56f24b1 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -11,7 +11,6 @@ compile error). `ImplicitUsings` is disabled in this repository, so even `System ```csharp using System; -using System.IO; using System.Text; using Apify.Client; using Apify.Client.Models; diff --git a/docs/misc.md b/docs/misc.md index 5e3b140..890a071 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -7,12 +7,24 @@ Browse public Actors in the [Apify Store](https://apify.com/store). -- `ListAsync(StoreListOptions?)` → `PaginationList` (one page). -- `IterateAsync(StoreListOptions?)` → `IAsyncEnumerable` (lazy, all pages; - `Limit` is the page size). - -`StoreListOptions`: `Offset`, `Limit`, `Search`, `SortBy`, `Category`, `Username`, `PricingModel`, -`IncludeUnrunnableActors`, `AllowsAgenticUsers`, `ResponseFormat`. +- `ListAsync(StoreListOptions? options = null)` → `PaginationList` (one page). +- `IterateAsync(StoreListOptions? options = null)` → `IAsyncEnumerable` (lazy, all + pages; `Limit` is the page size). + +`StoreListOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `Offset` | `int?` | Number of Actors to skip from the start. | +| `Limit` | `int?` | Maximum number of Actors to return in the page. | +| `Search` | `string?` | Full-text search string to filter Actors by. | +| `SortBy` | `string?` | Field to sort by (e.g. `popularity`, `newest`). | +| `Category` | `string?` | Restrict results to a Store category. | +| `Username` | `string?` | Restrict results to a given owner's Actors. | +| `PricingModel` | `string?` | Filter by pricing model (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, …). | +| `IncludeUnrunnableActors` | `bool?` | Include Actors that cannot currently be run. | +| `AllowsAgenticUsers` | `bool?` | Only Actors that permit agentic (automated) users. | +| `ResponseFormat` | `string?` | Requested response format. | ```csharp using System; @@ -29,9 +41,10 @@ await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Se ## Users — `client.Me()` / `client.User(id)` - `GetAsync()` → `User?`. For `Me()` the raw payload includes private account details - (`ToJsonObject()`). -- `MonthlyUsageAsync(string? date = null)` → `JsonObject` (only for `Me()`). -- `LimitsAsync()` / `UpdateLimitsAsync(object newLimits)` (only for `Me()`). + (`ToJsonObject()`); for `User(id)` it returns the public profile. +- `MonthlyUsageAsync(string? date = null)` → `JsonObject` (only for `Me()`; `date` is `YYYY-MM-DD`, and + `null` reports the current month). +- `LimitsAsync()` → `JsonObject` / `UpdateLimitsAsync(object newLimits)` (only for `Me()`). ```csharp using System; @@ -45,10 +58,11 @@ var usage = await client.Me().MonthlyUsageAsync(); ## Logs — `client.Log(buildOrRunId)` -- `GetAsync(LogOptions?)` → `string?` (buffered). -- `StreamAsync(LogOptions?)` → `Stream` (live). Also `client.Run(id).GetStreamedLogAsync()`. +- `GetAsync(LogOptions? options = null)` → `string?` (buffered). +- `StreamAsync(LogOptions? options = null)` → `Stream` (live). Also `client.Run(id).GetStreamedLogAsync()`. -`LogOptions`: `Raw`, `Download`. +`LogOptions` fields: `Raw` (`bool?`, return the unprocessed log rather than the parsed form) and +`Download` (`bool?`, request a download `Content-Disposition`). ```csharp using System; diff --git a/docs/models.md b/docs/models.md new file mode 100644 index 0000000..6e3baf6 --- /dev/null +++ b/docs/models.md @@ -0,0 +1,273 @@ +# Data models + +The resource clients return strongly-typed models from the `Apify.Client.Models` namespace. Most +models are thin, read-only wrappers over the API's JSON response: every documented field is exposed as +a property, and the underlying JSON is always available via `ToJsonObject()` (declared on the shared +`ApifyResource` base class) for fields not surfaced as first-class properties. + +```csharp +using System; +using Apify.Client; + +var client = new ApifyClient("my-api-token"); +var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + +Console.WriteLine(run.Status); // typed property +Console.WriteLine(run.ToJsonObject()["id"]); // raw JSON escape hatch +``` + +Reference-typed properties are nullable (`string?`, `bool?`, …) because the API omits fields that do +not apply to a given resource; treat `null` as "not present". + +## `ApifyResource` (base class) + +Base class for every JSON-backed model below. + +| Member | Type | Description | +|---|---|---| +| `ToJsonObject()` | `JsonObject` | The raw underlying JSON object, for reading fields not exposed as typed properties. | +| `Get(string key)` | `JsonNode?` | The raw JSON value for a single key, or `null` if absent. | + +## `Actor` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The Actor's unique ID. | +| `UserId` | `string?` | ID of the user who owns the Actor. | +| `Name` | `string?` | The Actor's technical name. | +| `Username` | `string?` | Username of the Actor's owner. | +| `Title` | `string?` | Human-readable title. | +| `Description` | `string?` | Free-text description. | +| `IsPublic` | `bool?` | Whether the Actor is published publicly in the Apify Store. | +| `CreatedAt` | `string?` | ISO 8601 creation timestamp. | +| `ModifiedAt` | `string?` | ISO 8601 last-modification timestamp. | + +## `ActorRun` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The run's unique ID. | +| `ActId` | `string?` | ID of the Actor that was run. | +| `ActorTaskId` | `string?` | ID of the task the run originated from, if any. | +| `UserId` | `string?` | ID of the user who started the run. | +| `Status` | `string?` | Run status (e.g. `RUNNING`, `SUCCEEDED`, `FAILED`, `ABORTED`). | +| `StatusMessage` | `string?` | Human-readable status message. | +| `StartedAt` | `string?` | ISO 8601 start timestamp. | +| `FinishedAt` | `string?` | ISO 8601 finish timestamp (`null` while running). | +| `BuildId` | `string?` | ID of the Actor build used for the run. | +| `DefaultDatasetId` | `string?` | ID of the run's default dataset. | +| `DefaultKeyValueStoreId` | `string?` | ID of the run's default key-value store. | +| `DefaultRequestQueueId` | `string?` | ID of the run's default request queue. | +| `ContainerUrl` | `string?` | URL of the run's container (for live access while running). | +| `IsTerminal` | `bool` | `true` if `Status` is a terminal state (succeeded/failed/aborted/timed-out). | + +## `Build` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The build's unique ID. | +| `ActId` | `string?` | ID of the Actor that was built. | +| `Status` | `string?` | Build status (e.g. `RUNNING`, `SUCCEEDED`, `FAILED`). | +| `StartedAt` | `string?` | ISO 8601 start timestamp. | +| `FinishedAt` | `string?` | ISO 8601 finish timestamp (`null` while building). | +| `BuildNumber` | `string?` | The semantic build number. | +| `IsTerminal` | `bool` | `true` if `Status` is a terminal state. | + +## `ActorVersion` + +| Property | Type | Description | +|---|---|---| +| `VersionNumber` | `string?` | The version's number (e.g. `0.1`). | +| `SourceType` | `string?` | How the source is provided (e.g. `SOURCE_FILES`, `GIT_REPO`, `TARBALL`, `GITHUB_GIST`). | + +## `ActorEnvVar` + +A read/write model (used both as request input and response). Fields set to `null` are omitted from +the request JSON. + +| Member | Type | Description | +|---|---|---| +| `ActorEnvVar(string? name = null, string? value = null, bool? isSecret = null)` | constructor | Build an environment variable to create/update. | +| `Name` | `string?` | The variable name. | +| `Value` | `string?` | The variable value. | +| `IsSecret` | `bool?` | Whether the value is stored encrypted and hidden. | + +## `ActorStoreListItem` + +An entry returned when browsing the Apify Store. + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The Actor's unique ID. | +| `Name` | `string?` | The Actor's technical name. | +| `Username` | `string?` | Username of the Actor's owner. | +| `Title` | `string?` | Human-readable title. | + +## `ActorTask` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The task's unique ID. | +| `ActId` | `string?` | ID of the Actor the task runs. | +| `UserId` | `string?` | ID of the user who owns the task. | +| `Name` | `string?` | The task's technical name. | +| `Title` | `string?` | Human-readable title. | +| `CreatedAt` | `string?` | ISO 8601 creation timestamp. | +| `ModifiedAt` | `string?` | ISO 8601 last-modification timestamp. | + +## `Dataset` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The dataset's unique ID. | +| `Name` | `string?` | The dataset's name (`null` for unnamed datasets). | +| `UserId` | `string?` | ID of the owning user. | +| `CreatedAt` | `string?` | ISO 8601 creation timestamp. | +| `ModifiedAt` | `string?` | ISO 8601 last-modification timestamp. | +| `ItemCount` | `long?` | Number of items stored in the dataset. | + +## `KeyValueStore` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The store's unique ID. | +| `Name` | `string?` | The store's name (`null` for unnamed stores). | +| `UserId` | `string?` | ID of the owning user. | +| `CreatedAt` | `string?` | ISO 8601 creation timestamp. | +| `ModifiedAt` | `string?` | ISO 8601 last-modification timestamp. | + +## `KeyValueStoreRecord` + +The value of a single key-value store record. `Value` holds the raw bytes so binary records (images, +XLSX exports, …) are returned intact. + +| Property | Type | Description | +|---|---|---| +| `Key` | `string` | The record's key. | +| `Value` | `byte[]` | The raw record bytes. | +| `ContentType` | `string?` | The record's MIME type (e.g. `application/json`). | + +## `KeyValueStoreKey` + +| Property | Type | Description | +|---|---|---| +| `Key` | `string?` | The record key. | +| `Size` | `long?` | Size of the record's value in bytes. | + +## `KeyValueStoreKeysPage` + +One page of key listings (returned by `ListKeysAsync`). + +| Property | Type | Description | +|---|---|---| +| `Items` | `IReadOnlyList` | The keys in this page. | +| `Limit` | `long` | The page-size limit that was applied. | +| `IsTruncated` | `bool` | `true` if more keys exist beyond this page. | +| `ExclusiveStartKey` | `string?` | The exclusive start key this page began after. | +| `NextExclusiveStartKey` | `string?` | Start key to pass to fetch the next page. | + +## `RequestQueue` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The queue's unique ID. | +| `Name` | `string?` | The queue's name (`null` for unnamed queues). | +| `UserId` | `string?` | ID of the owning user. | +| `CreatedAt` | `string?` | ISO 8601 creation timestamp. | +| `ModifiedAt` | `string?` | ISO 8601 last-modification timestamp. | +| `TotalRequestCount` | `long?` | Total number of requests ever added to the queue. | + +## `RequestQueueRequest` + +A read/write model (request input and response). Fields set to `null` are omitted from request JSON. + +| Member | Type | Description | +|---|---|---| +| `RequestQueueRequest(string? url = null, string? uniqueKey = null)` | constructor | Build a request to add to a queue. | +| `Id` | `string?` | The request's unique ID (assigned by the queue). | +| `Url` | `string?` | The request URL. | +| `UniqueKey` | `string?` | The key used to deduplicate the request within the queue. | +| `Method` | `string?` | HTTP method (defaults to `GET` on the server). | +| `UserData` | `JsonNode?` | Arbitrary user-defined JSON payload attached to the request. | + +## `RequestQueueHead` + +The head (front) of a request queue. + +| Property | Type | Description | +|---|---|---| +| `Items` | `IReadOnlyList` | The requests at the head of the queue. | +| `Limit` | `long` | The page-size limit that was applied. | +| `HadMultipleClients` | `bool` | `true` if more than one client has accessed the queue (concurrency hint). | + +## `RequestQueueOperationInfo` + +The result of adding/updating a single request. + +| Property | Type | Description | +|---|---|---| +| `RequestId` | `string?` | ID of the affected request. | +| `UniqueKey` | `string?` | The request's unique key. | +| `WasAlreadyPresent` | `bool?` | `true` if a request with the same unique key already existed. | +| `WasAlreadyHandled` | `bool?` | `true` if that existing request was already marked handled. | + +## `BatchAddResult` + +The aggregate result of a batch add-requests operation. + +| Property | Type | Description | +|---|---|---| +| `ProcessedRequests` | `IReadOnlyList` | Requests the API accepted. | +| `UnprocessedRequests` | `IReadOnlyList` | Requests that could not be processed (after retries). | + +## `Schedule` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The schedule's unique ID. | +| `UserId` | `string?` | ID of the owning user. | +| `Name` | `string?` | The schedule's name. | +| `CronExpression` | `string?` | The cron expression controlling when it fires. | +| `IsEnabled` | `bool?` | Whether the schedule is currently enabled. | + +## `Webhook` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The webhook's unique ID. | +| `UserId` | `string?` | ID of the owning user. | +| `RequestUrl` | `string?` | URL the webhook posts to when triggered. | +| `EventTypes` | `IReadOnlyList?` | The event types that trigger the webhook. | + +## `WebhookDispatch` + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The dispatch's unique ID. | +| `WebhookId` | `string?` | ID of the webhook that was dispatched. | + +## `User` + +Returned by `client.Me().GetAsync()` (private account details) and `client.User(id).GetAsync()` +(public profile). Only the always-present fields are typed; read the rest via `ToJsonObject()`. + +| Property | Type | Description | +|---|---|---| +| `Id` | `string?` | The user's unique ID. | +| `Username` | `string?` | The user's username. | + +## `PaginationList` + +One page of a paginated listing, returned by every `ListAsync` method. + +| Property | Type | Description | +|---|---|---| +| `Items` | `IReadOnlyList` | The items on this page. | +| `Count` | `long` | Number of items on this page (equals `Items.Count`). | +| `Total` | `long` | Total number of items across all pages. | +| `Offset` | `long` | The offset this page started at. | +| `Limit` | `long` | The page-size limit that was applied. | +| `Desc` | `bool` | Whether the listing is in descending order. | + +To iterate every item across pages without managing offsets yourself, use the matching +`IterateAsync` method (an `IAsyncEnumerable`) instead — see the resource-specific docs. diff --git a/docs/runs.md b/docs/runs.md index d088ea0..deea171 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -8,7 +8,11 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' - `ListAsync(ListOptions? options = null, RunListOptions? filter = null)` → `PaginationList`. - `IterateAsync(ListOptions? options = null, RunListOptions? filter = null)` → `IAsyncEnumerable` (lazy, all pages). - `RunListOptions`: `Status` (list), `StartedAfter`, `StartedBefore`. + +`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination). `RunListOptions` fields: +`Status` (`IReadOnlyList?`, filter by one or more run statuses such as `SUCCEEDED`/`RUNNING`), +`StartedAfter` and `StartedBefore` (ISO 8601 bounds, honoured only by the Actor- and task-scoped run +collections). ## Single run — `client.Run(runId)` @@ -27,6 +31,24 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' log to `toLog` one complete message at a time. Call `Start()` to begin and `StopAsync()` (or dispose) to end. `fromStart: false` skips messages older than the helper's creation. +### Option and charge types + +`MetamorphOptions` fields: `Build` (`string?`, pin the target Actor's build) and `ContentType` +(`string?`, content type of the metamorph `input` body; defaults to `application/json`). + +`RunResurrectOptions` overrides run settings when resurrecting a finished run: `Build` (`string?`), +`MemoryMbytes` (`int?`), `TimeoutSecs` (`int?`), `MaxItems` (`int?`), `MaxTotalChargeUsd` (`double?`), +and `RestartOnError` (`bool?`). See [Actors](actors.md) for each field's meaning. + +`RunChargeOptions` describes a pay-per-event charge and is built via its constructor +`RunChargeOptions(string eventName, int? count = null, string? idempotencyKey = null)`: + +| Property | Type | Description | +|---|---|---| +| `EventName` | `string` | Name of the pay-per-event event to charge for (required, non-empty). | +| `Count` | `int?` | Number of event occurrences to charge (defaults to 1 server-side). | +| `IdempotencyKey` | `string?` | Key that deduplicates retried charges; auto-generated when omitted. | + ```csharp using Apify.Client; using Apify.Client.Options; diff --git a/docs/schedules.md b/docs/schedules.md index ec620a7..4812e70 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -5,13 +5,13 @@ Schedules automatically start Actor or task runs at specified times. Access the ## Collection -- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → - `IAsyncEnumerable` (lazy, all pages). +- `ListAsync(ListOptions? options = null)` → `PaginationList`; + `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). - `CreateAsync(object schedule)` → `Schedule`. ## Single schedule — `client.Schedule(id)` -- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `GetAsync()` → `Schedule?`; `UpdateAsync(object newFields)` → `Schedule`; `DeleteAsync()`. - `GetLogAsync()` → `string?` (invocation log; `null` if none yet). ```csharp diff --git a/docs/storages.md b/docs/storages.md index 01e7d04..4a8c56d 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -1,29 +1,71 @@ # Storages The three storage types — datasets, key-value stores and request queues — share the same collection -shape: `ListAsync(StorageListOptions?)` (one page), `IterateAsync(StorageListOptions?)` → -`IAsyncEnumerable` (lazy, all pages), and `GetOrCreateAsync(name?)`. Storages can also be reached -from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). +shape: `ListAsync(StorageListOptions? options = null)` (one page), +`IterateAsync(StorageListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages), and +`GetOrCreateAsync(string? name = null)` (dataset and key-value store collections additionally accept an +optional `JsonNode? schema = null` to register a storage schema on creation). Storages can also be +reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQueue()`). > Snippets below run inside an `async` context. `ImplicitUsings` is disabled in this repository, so all > `using` directives (including `System`) are shown explicitly and must precede any statements. -`StorageListOptions`: `Offset`, `Limit`, `Desc`, `Unnamed`, `Ownership`. +`StorageListOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `Offset` | `int?` | Number of storages to skip from the start. | +| `Limit` | `int?` | Maximum number of storages to return in the page. | +| `Desc` | `bool?` | Sort newest-first when `true`. | +| `Unnamed` | `bool?` | Include unnamed storages when `true` (they are excluded by default). | +| `Ownership` | `string?` | Filter by ownership (e.g. only storages owned by the current account). | ## Datasets `client.Datasets()` / `client.Dataset(id)`. -- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. -- `ListItemsAsync(DatasetListItemsOptions? = null)` → `PaginationList` (one page; pagination via +- `GetAsync()` → `Dataset?`; `UpdateAsync(object newFields)` → `Dataset`; `DeleteAsync()`. +- `ListItemsAsync(DatasetListItemsOptions? options = null)` → `PaginationList` (one page; pagination via response headers). -- `IterateItemsAsync(DatasetListItemsOptions? = null)` → `IAsyncEnumerable` — lazily iterate every +- `IterateItemsAsync(DatasetListItemsOptions? options = null)` → `IAsyncEnumerable` — lazily iterate every item across pages, fetching each page on demand. -- `DownloadItemsAsync(DownloadItemsFormat, DatasetDownloadOptions? = null)` → serialized items as `byte[]` - (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). +- `DownloadItemsAsync(DownloadItemsFormat format, DatasetDownloadOptions? options = null)` → serialized items as + `byte[]` (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). - `PushItemsAsync(object items)` — push one object or an array of objects. - `GetStatisticsAsync()` → `JsonObject?`. -- `CreateItemsPublicUrlAsync(DatasetListItemsOptions?, int? expiresInSecs = null)` → signed public URL. +- `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null)` → signed public URL. + +`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `Desc` (reverse +order), `Fields`/`OutputFields`/`Omit` (choose columns), `Unwind`/`Flatten` (restructure nested +fields), `Clean`/`SkipEmpty`/`SkipHidden`/`SkipFailedPages` (drop unwanted rows), `Simplified`, +`View`, and `Signature` (for signed public URLs). + +`DownloadItemsAsync`'s `format` argument is the `DownloadItemsFormat` enum, whose values map to the +API's export formats: + +| Value | Format | +|---|---| +| `Json` | JSON array. | +| `Jsonl` | Newline-delimited JSON. | +| `Csv` | Comma-separated values. | +| `Xlsx` | Microsoft Excel (XLSX) workbook (binary). | +| `Xml` | XML. | +| `Rss` | RSS feed. | +| `Html` | HTML table. | + +`DatasetDownloadOptions` adds format-specific export options on top of the item filtering/projection: + +| Field | Type | Description | +|---|---|---| +| `Items` | `DatasetListItemsOptions?` | The shared item filtering/projection options to apply before export. | +| `Attachment` | `bool?` | Set `Content-Disposition: attachment` on the response. | +| `Bom` | `bool?` | Prepend a UTF-8 BOM (useful for Excel-compatible CSV). | +| `Delimiter` | `string?` | The CSV field delimiter (default `,`). | +| `SkipHeaderRow` | `bool?` | Omit the CSV header row. | +| `XmlRoot` | `string?` | Name of the root XML element (default `items`). | +| `XmlRow` | `string?` | Name of the per-item XML element (default `item`). | +| `FeedTitle` | `string?` | Title used for RSS/Atom feed exports. | +| `FeedDescription` | `string?` | Description used for RSS/Atom feed exports. | ```csharp using System; @@ -44,17 +86,23 @@ Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the `client.KeyValueStores()` / `client.KeyValueStore(id)`. -- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. -- `ListKeysAsync(ListKeysOptions?)` → `KeyValueStoreKeysPage`. -- `RecordExistsAsync(key)` → `bool`. -- `GetRecordAsync(key, GetRecordOptions? = null)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a - `byte[]` of the record's raw bytes (so binary records survive intact); decode it according to - `KeyValueStoreRecord.ContentType` — e.g. `Encoding.UTF8.GetString(record.Value)` for text, or - `JsonSerializer.Deserialize(record.Value)` for JSON. -- `SetRecordAsync(key, byte[] value, contentType, SetRecordOptions?)` and `SetRecordJsonAsync(key, value)` - (serializes `value` to JSON bytes). -- `DeleteRecordAsync(key)`. -- `GetRecordPublicUrlAsync(key)` and `CreateKeysPublicUrlAsync(ListKeysOptions?, int? expiresInSecs)`. +- `GetAsync()` → `KeyValueStore?`; `UpdateAsync(object newFields)` → `KeyValueStore`; `DeleteAsync()`. +- `ListKeysAsync(ListKeysOptions? options = null)` → `KeyValueStoreKeysPage`. +- `RecordExistsAsync(string key)` → `bool`. +- `GetRecordAsync(string key, GetRecordOptions? options = null)` → `KeyValueStoreRecord?`. + `KeyValueStoreRecord.Value` is a `byte[]` of the record's raw bytes (so binary records survive intact); + decode it according to `KeyValueStoreRecord.ContentType` — e.g. `Encoding.UTF8.GetString(record.Value)` + for text, or `JsonSerializer.Deserialize(record.Value)` for JSON. +- `SetRecordAsync(string key, byte[] value, string contentType, SetRecordOptions? options = null)` and + `SetRecordJsonAsync(string key, object? value)` (serializes `value` to JSON bytes). +- `DeleteRecordAsync(string key)`. +- `GetRecordPublicUrlAsync(string key)` and + `CreateKeysPublicUrlAsync(ListKeysOptions? options = null, int? expiresInSecs = null)`. + +`ListKeysOptions` fields: `Limit` (page size), `ExclusiveStartKey` (start after this key), +`Prefix` (only keys with this prefix), `Collection` (a named record collection), and `Signature` +(for signed public URLs). `GetRecordOptions` fields: `Attachment` (request a download disposition) +and `Signature`. `SetRecordOptions` fields: `TimeoutSecs` and `DoNotRetryTimeouts`. ```csharp using System; @@ -82,19 +130,35 @@ await client.KeyValueStore(store.Id!).SetRecordAsync("blob", new byte[] { 0x00, ## Request queues -`client.RequestQueues()` / `client.RequestQueue(id, RequestQueueClientOptions?)`. The options set a -stable `ClientKey` (required to manage locks the client created) and a per-queue `TimeoutSecs`. - -- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. -- `AddRequestAsync(RequestQueueRequest, bool forefront = false)` → `RequestQueueOperationInfo`. -- `GetRequestAsync(id)`, `UpdateRequestAsync(request, forefront)`, `DeleteRequestAsync(id)`. -- `ListHeadAsync(int? limit)` → `RequestQueueHead`; `ListAndLockHeadAsync(lockSecs, limit?)`. -- `BatchAddRequestsAsync(IReadOnlyList, forefront, BatchAddRequestsOptions?)` — - auto-chunks by count (25) and payload size (~9 MiB) and retries unprocessed requests. Every request - needs a non-empty `UniqueKey`. -- `ListRequestsAsync(ListRequestsOptions?)` and `PaginateRequestsAsync(PaginateRequestsOptions?)` - (`IAsyncEnumerable`). -- Lock management: `ProlongRequestLockAsync`, `DeleteRequestLockAsync`, `UnlockRequestsAsync`. +`client.RequestQueues()` / `client.RequestQueue(id, RequestQueueClientOptions? options = null)`. The +options set a stable `ClientKey` (required to manage locks the client created) and a per-queue +`TimeoutSecs`. + +- `GetAsync()` → `RequestQueue?`; `UpdateAsync(object newFields)` → `RequestQueue`; `DeleteAsync()`. +- `AddRequestAsync(RequestQueueRequest request, bool forefront = false)` → `RequestQueueOperationInfo`. +- `GetRequestAsync(string id)`, `UpdateRequestAsync(RequestQueueRequest request, bool forefront = false)`, + `DeleteRequestAsync(string id)`. +- `ListHeadAsync(int? limit = null)` → `RequestQueueHead`; + `ListAndLockHeadAsync(int lockSecs, int? limit = null)`. +- `BatchAddRequestsAsync(IReadOnlyList requests, bool forefront = false, BatchAddRequestsOptions? options = null)` + → `BatchAddResult` — auto-chunks by count (25) and payload size (~9 MiB) and retries unprocessed + requests. Every request needs a non-empty `UniqueKey`. +- `BatchDeleteRequestsAsync(object requests)` → `JsonObject` — delete a batch of requests in one call + (`requests` is any JSON-serializable list of requests/keys to remove). +- `ListRequestsAsync(ListRequestsOptions? options = null)` → `JsonObject` and + `PaginateRequestsAsync(PaginateRequestsOptions? options = null)` → `IAsyncEnumerable`. +- Lock management: `ProlongRequestLockAsync(string id, int lockSecs, bool forefront = false)` → `JsonObject`, + `DeleteRequestLockAsync(string id, bool forefront = false)`, `UnlockRequestsAsync()` → `JsonObject`. +- `WithClientKey(string clientKey)` → `RequestQueueClient` — returns a copy of this client bound to the + given client key (a fluent alternative to passing `RequestQueueClientOptions.ClientKey` on + `client.RequestQueue(id, options)`); the client key ties lock ownership to this client. + +`BatchAddRequestsOptions` fields: `MaxUnprocessedRequestsRetries` (retry attempts for requests the API +leaves unprocessed), `MaxParallel` (how many chunks are sent concurrently), and +`MinDelayBetweenUnprocessedRequestsRetriesMillis` (backoff before retrying unprocessed requests). +`ListRequestsOptions`/`PaginateRequestsOptions` fields: `Limit`, `ExclusiveStartId`, `Cursor`, and +`Filter` (an `IReadOnlyList?` — one or more of `"pending"`/`"locked"`, so several states can be +requested at once); `PaginateRequestsOptions` also has `MaxPageLimit`. ```csharp using System; diff --git a/docs/tasks.md b/docs/tasks.md index e9ee3c4..e67147d 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -5,20 +5,25 @@ a specific task with `client.Task(id)`. ## Collection -- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → - `IAsyncEnumerable` (lazy, all pages). +- `ListAsync(ListOptions? options = null)` → `PaginationList`; + `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). - `CreateAsync(object task)` → `ActorTask`. ## Single task — `client.Task(id)` -- `GetAsync()`, `UpdateAsync(newFields)`, `DeleteAsync()`. +- `GetAsync()` → `ActorTask?`; `UpdateAsync(object newFields)` → `ActorTask`; `DeleteAsync()`. - `StartAsync(object? input = null, TaskStartOptions? options = null)` → `ActorRun`. - `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null, Action? log = null)` → `ActorRun` (`log`, if set, redirects the run's live log to that sink for the duration of the wait). -- `GetInputAsync()` / `UpdateInputAsync(object input)`. -- `LastRun(LastRunOptions?)` → `RunClient`; `Runs()` → `RunCollectionClient`. +- `GetInputAsync()` → `JsonNode?` / `UpdateInputAsync(object input)` → `JsonNode?`. +- `LastRun(LastRunOptions? options = null)` → `RunClient`; `Runs()` → `RunCollectionClient`. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. +`TaskStartOptions` overrides the task's stored run settings for a single start: `Build`, +`MemoryMbytes`, `TimeoutSecs`, `WaitForFinish` (server-side wait on the start call, max 60), +`MaxItems`, `MaxTotalChargeUsd`, `RestartOnError`, and `Webhooks` (ad-hoc webhooks for this run). See +[Actors](actors.md) for the meaning of each field. + The model is named `ActorTask` (not `Task`) to avoid colliding with `System.Threading.Tasks.Task`. ```csharp diff --git a/docs/webhooks.md b/docs/webhooks.md index 70f7b56..11a2aa6 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -6,8 +6,9 @@ Webhooks notify an external service when specific events occur. Access the accou ## Webhook collection — `client.Webhooks()` -- `ListAsync(ListOptions?)` → `PaginationList`; `IterateAsync(ListOptions?)` → - `IAsyncEnumerable` (lazy, all pages). Webhook dispatches expose the same pair. +- `ListAsync(ListOptions? options = null)` → `PaginationList`; + `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). Webhook + dispatches expose the same pair. - `CreateAsync(object webhook)` → `Webhook`. Webhooks nested under an Actor or task (`client.Actor(id).Webhooks()`, `client.Task(id).Webhooks()`) diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs index ceb84c2..8a04fa2 100644 --- a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -34,6 +34,25 @@ public async Task RunActorAndReadOutputs() await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); } + [SkippableFact] + public async Task UpdateAndDeleteRun() + { + var client = RequireClient(); + + // Start a run and wait for it to reach a terminal state before mutating it. + var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + Assert.Equal("SUCCEEDED", run.Status); + + var statusMessage = "updated by dotnet client integration test"; + var updated = await client.Run(run.Id!).UpdateAsync(new { statusMessage }); + Assert.Equal(run.Id, updated.Id); + Assert.Equal(statusMessage, updated.StatusMessage); + + // Delete the run; DeleteAsync throws on a non-success status, so a clean return is the check. + // (No read-after-delete assertion — that would rely on strong replica consistency and could flake.) + await client.Run(run.Id!).DeleteAsync(); + } + [SkippableFact] public async Task LastRunAccess() { diff --git a/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs index df32cf6..09182c1 100644 --- a/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/UserIntegrationTests.cs @@ -15,6 +15,26 @@ public async Task GetOwnAccount() Assert.False(string.IsNullOrEmpty(user!.Id)); } + [SkippableFact] + public async Task GetPublicUserById() + { + var client = RequireClient(); + + // Resolve our own id and username via the private `me` endpoint, then fetch the same user + // through the public `/users/{userId}` endpoint to exercise the non-`me` code path. + var me = await client.Me().GetAsync(); + Assert.NotNull(me); + Assert.False(string.IsNullOrEmpty(me!.Id)); + Assert.False(string.IsNullOrEmpty(me.Username)); + + var publicUser = await client.User(me.Id!).GetAsync(); + Assert.NotNull(publicUser); + + // The public endpoint returns UserPublicInfo, whose schema exposes `username` (not `id`), so + // verify we resolved the right user via the username rather than an unspecified `id` field. + Assert.Equal(me.Username, publicUser!.Username); + } + [SkippableFact] public async Task GetMonthlyUsage() {