Add cursor-based pagination and safer data set piece detection (#914)

* feat: add cursor-based pagination for data set and piece queries

* docs: Document active piece field migration

* fix: derive hasActivePieces from leaf count

Use getDataSetLeafCount as an O(1) presence proxy instead of a one-item
cursor scan, which still walks to nextPieceId on drained data sets.

* fix: replace SDK activePieceCount with hasActivePieces

Listing paths now report piece presence from leaf count, and getActivePieceCount paginates the cursor API instead of the linear-scan getter.

Author: hugomrdias

docs/src/content/docs/developer-guides/migration-guide.md

@@ -104,6 +104,44 @@ for await (const piece of paginate(({ cursor }) =>
 
 Raw `*Call` helpers in `@filoz/synapse-core` remain ABI-oriented: provide their required contract-facing `offset` or `startPieceId` and `limit` fields explicitly when constructing multicalls.
 
+### Action: Replace core `activePieceCount` with `hasActivePieces`
+
+The enriched `PdpDataSet` values returned by `getPdpDataSet()` and `getPdpDataSets()` no longer include an exact `activePieceCount`. They now expose `hasActivePieces`, derived from a non-zero `getDataSetLeafCount` read. Leaf count is an O(1) storage lookup, unlike `getActivePiecesByCursor` and `getActivePieceCount`, which scan piece IDs and can run out of gas on large or fully drained data sets:
+
+```ts
+// before
+const dataSet = await getPdpDataSet(client, { dataSetId })
+if (dataSet && dataSet.activePieceCount > 0n) {
+  // the data set has pieces
+}
+
+// after
+const dataSet = await getPdpDataSet(client, { dataSetId })
+if (dataSet?.hasActivePieces) {
+  // the data set has pieces
+}
+```
+
+`WarmStorageService.hasActivePieces()` keeps the same public API, but now uses the same leaf-count proxy instead of calculating an exact count. `EnhancedDataSetInfo` from `getClientDataSetsWithDetails()` / `findDataSets()` also exposes `hasActivePieces` instead of `activePieceCount`.
+
+The core `getActivePieceCount()` action remains available, but the underlying contract getter scans the data set's piece-ID range and can fail for large data sets. `WarmStorageService.getActivePieceCount()` now paginates `getActivePiecesByCursor` to derive an exact count:
+
+```ts
+const activePieceCount = await warmStorageService.getActivePieceCount({ dataSetId })
+```
+
+To paginate explicitly in core:
+
+```ts
+let activePieceCount = 0n
+
+for await (const _piece of paginate(({ cursor }) =>
+  getActivePiecesByCursor(client, { dataSetId, cursor })
+)) {
+  activePieceCount++
+}
+```
+
 ---
 
 ## 1.0.0

docs/src/content/docs/developer-guides/storage/storage-operations.mdx

@@ -196,7 +196,7 @@ These APIs are useful when you want to inspect existing data sets, query stored
 
 ### Getting All Data Sets
 
-Retrieve all data sets owned by your account to inspect piece counts, CDN status, and metadata:
+Retrieve all data sets owned by your account to inspect piece presence, CDN status, and metadata:
 
 ```ts twoslash
 // @lib: esnext,dom
@@ -210,7 +210,7 @@ for (const ds of dataSets) {
   console.log(`Dataset ${ds.pdpVerifierDataSetId}:`, {
     live: ds.isLive,
     cdn: ds.withCDN,
-    pieces: ds.activePieceCount,
+    hasActivePieces: ds.hasActivePieces,
     metadata: ds.metadata
   });
 }

packages/synapse-core/src/mocks/jsonrpc/index.ts

@@ -570,7 +570,7 @@ export const presets = {
         false,
       ],
       getDataSetStorageProvider: () => [ADDRESSES.serviceProvider1, ADDRESSES.zero],
-      getDataSetLeafCount: () => [0n],
+      getDataSetLeafCount: () => [2n],
       getScheduledRemovals: () => [[]],
       getNextChallengeEpoch: () => [5000n],
       findPieceIdsByCid: () => [[0n]],

packages/synapse-core/src/pdp-verifier/get-active-piece-count.ts

@@ -6,6 +6,7 @@ import { asChain } from '../chains.ts'
 import type { ActionCallChain } from '../types.ts'
 import { STRING_ERRORS, stringErrorEquals } from '../utils/contract-errors.ts'
 import { toReadClient } from '../utils/read-client.ts'
+import type { getActivePiecesByCursor } from './get-active-pieces-by-cursor.ts'
 
 export namespace getActivePieceCount {
   export type OptionsType = {
@@ -23,6 +24,10 @@ export namespace getActivePieceCount {
 /**
  * Get the active piece count for a data set (non-zero leaf count)
  *
+ * This contract method scans the complete historical piece-ID range and can
+ * run out of gas for large data sets. Prefer explicitly traversing
+ * {@link getActivePiecesByCursor} when reliability for large sets is required.
+ *
  * @example
  * ```ts
  * import { getActivePieceCount } from '@filoz/synapse-core/pdp-verifier'

packages/synapse-core/src/warm-storage/find-matching-data-sets.ts

@@ -41,7 +41,7 @@ export function metadataMatches(dataSetMetadata: MetadataObject, requestedMetada
  * Only active datasets are considered (live, managed, pdpEndEpoch === 0n).
  *
  * Sort order:
- *   1. Datasets with pieces (activePieceCount > 0) before empty datasets
+ *   1. Datasets with pieces before empty datasets
  *   2. Within each group, older datasets (lower ID) first
  *
  * @param dataSets - Datasets to search (typically filtered to a single provider)
@@ -54,10 +54,8 @@ export function findMatchingDataSets(dataSets: SelectionDataSet[], metadata: Met
   )
 
   return matching.sort((a, b) => {
-    // Datasets with pieces sort before empty ones
-    if (a.activePieceCount > 0n && b.activePieceCount === 0n) return -1
-    if (b.activePieceCount > 0n && a.activePieceCount === 0n) return 1
-    // Within same group, oldest dataset first (lower ID)
+    if (a.hasActivePieces && !b.hasActivePieces) return -1
+    if (b.hasActivePieces && !a.hasActivePieces) return 1
     return Number(a.dataSetId - b.dataSetId)
   })
 }

packages/synapse-core/src/warm-storage/get-pdp-data-set.ts

@@ -2,7 +2,8 @@ import { type Address, type Chain, type Client, isAddressEqual, type ReadContrac
 import { multicall } from 'viem/actions'
 import { asChain } from '../chains.ts'
 import { dataSetLiveCall } from '../pdp-verifier/data-set-live.ts'
-import { getActivePieceCountCall } from '../pdp-verifier/get-active-piece-count.ts'
+import type { getActivePiecesByCursor } from '../pdp-verifier/get-active-pieces-by-cursor.ts'
+import { type getDataSetLeafCount, getDataSetLeafCountCall } from '../pdp-verifier/get-data-set-leaf-count.ts'
 import { getDataSetListenerCall } from '../pdp-verifier/get-data-set-listener.ts'
 import { getPDPProviderCall, parsePDPProvider } from '../sp-registry/get-pdp-provider.ts'
 import { toReadClient } from '../utils/read-client.ts'
@@ -25,7 +26,14 @@ export namespace getPdpDataSet {
 }
 
 /**
- * Get a PDP data set by ID
+ * Get a PDP data set by ID.
+ *
+ * The result reports piece presence from a non-zero {@link getDataSetLeafCount}
+ * read, which is an O(1) storage lookup, rather than scanning piece IDs. Exact
+ * active-piece counts are omitted because the contract's count getter performs
+ * a linear scan and can fail for large data sets. To derive an exact count
+ * explicitly, traverse {@link getActivePiecesByCursor} with `paginate` and
+ * count the yielded pieces.
  *
  * @param client - The client to use to get the PDP data set.
  * @param options - {@link getPdpDataSet.OptionsType}
@@ -75,7 +83,7 @@ export async function getPdpDataSet(
 }
 
 /**
- * Read the PDP data set info.
+ * Read PDP data set information.
  *
  * @param client - The client to use to read the PDP data set info.
  * @param options
@@ -89,7 +97,7 @@ export async function readPdpDataSetInfo(
   }
 ): Promise<PdpDataSetInfo> {
   const chain = asChain(client.chain)
-  const [live, listener, _metadata, _pdpProvider, activePieceCount] = await multicall(toReadClient(client), {
+  const [live, listener, _metadata, _pdpProvider, leafCount] = await multicall(toReadClient(client), {
     allowFailure: false,
     contracts: [
       dataSetLiveCall({
@@ -108,7 +116,7 @@ export async function readPdpDataSetInfo(
         chain: client.chain,
         providerId: options.providerId,
       }),
-      getActivePieceCountCall({
+      getDataSetLeafCountCall({
         chain: client.chain,
         dataSetId: options.dataSetInfo.dataSetId,
       }),
@@ -124,6 +132,6 @@ export async function readPdpDataSetInfo(
     cdn: options.dataSetInfo.cdnRailId > 0n && 'withCDN' in metadata,
     metadata,
     provider: pdpProvider,
-    activePieceCount,
+    hasActivePieces: leafCount > 0n,
   }
 }

packages/synapse-core/src/warm-storage/get-pdp-data-sets.ts

@@ -3,7 +3,8 @@ import { multicall } from 'viem/actions'
 import { asChain } from '../chains.ts'
 import type { Page, paginate } from '../pagination.ts'
 import { type dataSetLive, dataSetLiveCall } from '../pdp-verifier/data-set-live.ts'
-import { type getActivePieceCount, getActivePieceCountCall } from '../pdp-verifier/get-active-piece-count.ts'
+import type { getActivePiecesByCursor } from '../pdp-verifier/get-active-pieces-by-cursor.ts'
+import { type getDataSetLeafCount, getDataSetLeafCountCall } from '../pdp-verifier/get-data-set-leaf-count.ts'
 import { type getDataSetListener, getDataSetListenerCall } from '../pdp-verifier/get-data-set-listener.ts'
 import { type getPDPProvider, getPDPProviderCall, parsePDPProvider } from '../sp-registry/get-pdp-provider.ts'
 import type { PDPProvider } from '../sp-registry/types.ts'
@@ -24,7 +25,7 @@ type DataSetEnrichmentResults = [
   dataSetLive.OutputType,
   getDataSetListener.ContractOutputType,
   getAllDataSetMetadata.ContractOutputType,
-  getActivePieceCount.OutputType,
+  getDataSetLeafCount.OutputType,
 ]
 
 export namespace getPdpDataSets {
@@ -41,7 +42,12 @@ export namespace getPdpDataSets {
  *
  * Only the current source page is enriched, in bounded batches with source
  * order preserved. Pass `nextCursor` back as `cursor`; treat it as
- * opaque. Use {@link paginate} to traverse every page.
+ * opaque. Use {@link paginate} to traverse every page. Results report piece
+ * presence from a non-zero {@link getDataSetLeafCount} read, which is an O(1)
+ * storage lookup, rather than scanning piece IDs. Exact active-piece counts are
+ * omitted because the contract's count getter performs a linear scan and can
+ * fail for large data sets. To derive a count explicitly, traverse
+ * {@link getActivePiecesByCursor} and count the yielded pieces.
  *
  * @param client - The client to use to get data sets for a client address.
  * @param options - {@link getPdpDataSets.OptionsType}
@@ -97,7 +103,7 @@ export async function getPdpDataSets(
 
 /**
  * Enrich one bounded batch of source data sets with their PDP state, metadata,
- * provider details, and active-piece counts.
+ * provider details, and active-piece presence.
  *
  * The four data-set-specific reads are flattened into one Viem multicall. PDP
  * provider reads are deduplicated by provider ID and cached in `providers`, so
@@ -128,7 +134,7 @@ async function enrichDataSetBatch(
     dataSetLiveCall({ chain: client.chain, dataSetId }),
     getDataSetListenerCall({ chain: client.chain, dataSetId }),
     getAllDataSetMetadataCall({ chain: client.chain, dataSetId }),
-    getActivePieceCountCall({ chain: client.chain, dataSetId }),
+    getDataSetLeafCountCall({ chain: client.chain, dataSetId }),
   ])
   const providerCalls = missingProviderIds.map((providerId) => getPDPProviderCall({ chain: client.chain, providerId }))
   const results = await multicall(toReadClient(client), {
@@ -149,7 +155,7 @@ async function enrichDataSetBatch(
 
   return dataSets.map((dataSet, index) => {
     const resultOffset = index * DATA_SET_CALL_COUNT
-    const [live, listener, rawMetadata, activePieceCount] = results.slice(
+    const [live, listener, rawMetadata, leafCount] = results.slice(
       resultOffset,
       resultOffset + DATA_SET_CALL_COUNT
     ) as DataSetEnrichmentResults
@@ -166,7 +172,7 @@ async function enrichDataSetBatch(
       cdn: dataSet.cdnRailId > 0n && 'withCDN' in metadata,
       metadata,
       provider,
-      activePieceCount,
+      hasActivePieces: leafCount > 0n,
     }
   })
 }

packages/synapse-core/src/warm-storage/location-types.ts

@@ -2,12 +2,12 @@ import type { PDPProvider } from '../sp-registry/types.ts'
 import type { MetadataObject } from '../utils/metadata.ts'
 
 /**
- * Dataset with piece count, for provider selection.
+ * Dataset with piece-presence information used for provider selection.
  *
  * Picks the fields that selectProviders() and findMatchingDataSets()
- * need, plus activePieceCount which is fetched separately via multicall.
+ * need, plus whether the data set contains at least one active piece.
  *
- * Core callers can spread a PdpDataSet directly: `{ ...ds, activePieceCount }`.
+ * Core callers can spread a PdpDataSet directly.
  * SDK callers map from EnhancedDataSetInfo (different field names).
  */
 export interface SelectionDataSet {
@@ -17,8 +17,8 @@ export interface SelectionDataSet {
   providerId: bigint
   /** Data set metadata (key-value pairs) */
   metadata: MetadataObject
-  /** Number of active pieces in the dataset (0 = empty) */
-  activePieceCount: bigint
+  /** Whether the data set contains at least one active piece */
+  hasActivePieces: boolean
   /** End epoch for PDP service (0 = active) */
   pdpEndEpoch: bigint
   /** Whether the data set is live in the PDP Verifier */
@@ -44,7 +44,7 @@ export interface ProviderSelectionInput {
   /** Array of endorsed provider IDs (from endorsements.getProviderIds).
    *  Non-empty = restrict to endorsed only. Empty = use all providers. */
   endorsedIds: bigint[]
-  /** Client's existing datasets with metadata and piece counts */
+  /** Client's existing datasets with metadata and piece-presence information */
   clientDataSets: SelectionDataSet[]
 }
 

packages/synapse-core/src/warm-storage/types.ts

@@ -44,8 +44,8 @@ export type PdpDataSetInfo = {
   metadata: MetadataObject
   /** PDP provider associated with the data set. */
   provider: PDPProvider
-  /** Number of active (non-zero) pieces in the data set. */
-  activePieceCount: bigint
+  /** Whether the data set contains at least one active piece (non-zero leaf count). */
+  hasActivePieces: boolean
 }
 
 export interface PdpDataSet extends DataSetInfo, PdpDataSetInfo {}

packages/synapse-core/test/find-matching-data-sets.test.ts

@@ -8,7 +8,7 @@ function makeDataSet(
 ): SelectionDataSet {
   return {
     metadata: {},
-    activePieceCount: 0n,
+    hasActivePieces: false,
     pdpEndEpoch: 0n,
     live: true,
     managed: true,
@@ -71,18 +71,18 @@ describe('findMatchingDataSets', () => {
 
   it('sorts datasets with pieces before empty ones', () => {
     const dataSets = [
-      makeDataSet({ dataSetId: 1n, providerId: 1n, metadata: { source: 'app' }, activePieceCount: 0n }),
-      makeDataSet({ dataSetId: 2n, providerId: 2n, metadata: { source: 'app' }, activePieceCount: 5n }),
+      makeDataSet({ dataSetId: 1n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: false }),
+      makeDataSet({ dataSetId: 2n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }),
     ]
     const result = findMatchingDataSets(dataSets, { source: 'app' })
     assert.equal(result[0].dataSetId, 2n)
     assert.equal(result[1].dataSetId, 1n)
   })
 
-  it('sorts by ID ascending within same piece group', () => {
+  it('sorts by ID ascending', () => {
     const dataSets = [
-      makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, activePieceCount: 3n }),
-      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, activePieceCount: 3n }),
+      makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: true }),
+      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }),
     ]
     const result = findMatchingDataSets(dataSets, { source: 'app' })
     assert.equal(result[0].dataSetId, 5n)
@@ -134,15 +134,14 @@ describe('findMatchingDataSets', () => {
     assert.equal(result.length, 0)
   })
 
-  it('full sorting: pieces first, then by ID within groups', () => {
+  it('sorts by piece presence and then by ID', () => {
     const dataSets = [
-      makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, activePieceCount: 0n }),
-      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, activePieceCount: 3n }),
-      makeDataSet({ dataSetId: 3n, providerId: 3n, metadata: { source: 'app' }, activePieceCount: 0n }),
-      makeDataSet({ dataSetId: 8n, providerId: 4n, metadata: { source: 'app' }, activePieceCount: 7n }),
+      makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: false }),
+      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }),
+      makeDataSet({ dataSetId: 3n, providerId: 3n, metadata: { source: 'app' }, hasActivePieces: false }),
+      makeDataSet({ dataSetId: 8n, providerId: 4n, metadata: { source: 'app' }, hasActivePieces: true }),
     ]
     const result = findMatchingDataSets(dataSets, { source: 'app' })
-    // Pieces first (5n, 8n by ID ascending), then empty (3n, 10n by ID ascending)
     assert.deepEqual(
       result.map((ds) => ds.dataSetId),
       [5n, 8n, 3n, 10n]