docs: clarify documentation and comments

Author: silent-cipher

apps/backend/.env.example

@@ -27,17 +27,8 @@
 # (network-dependent default: off on mainnet, so a shared =true must not enable
 # the canary there).
 #
-# SPs get full-rate testing (DEALS_PER_SP_PER_HOUR, DATASET_CREATIONS_PER_SP_PER_HOUR,
-# MIN_NUM_DATASETS_FOR_CHECKS, and the DATASET_LIFECYCLE_CHECK_ENABLED canary) only
-# if they're on-chain approved OR listed in
-# EXPECTED_APPROVED_SP_IDS/EXPECTED_APPROVED_SP_ADDRESSES (either one is enough —
-# they're checked independently, same as BLOCKED_SP_IDS/BLOCKED_SP_ADDRESSES).
-# Every other active, unblocked SP (new/unknown/not-yet-vetted) is throttled to a
-# fixed trickle tier (1 deal + 1 data-set-creation attempt every 4 hours, 1
-# data-set target; see `trickleTierRates` in src/config/constants.ts) to bound
-# wallet spend on SPs dealbot hasn't vetted yet. The lifecycle-check canary is
-# never scheduled at all for trickle-tier SPs (it creates+terminates a real
-# throwaway dataset every run). See #681.
+# Full-rate/trickle tiering is applied after provider eligibility. See
+# docs/environment-variables.md#full-rate-vs-trickle-tier.
 #
 # Process globals (database, HTTP ports, ClickHouse batching, pg-boss) have no
 # per-network form. Variables for INACTIVE networks are ignored; only active
@@ -153,8 +144,8 @@ CALIBRATION_PULL_PIECE_CLEANUP_INTERVAL_SECONDS=604800
 # CALIBRATION_BLOCKED_SP_IDS=1234,5678
 # CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
 
-# SPs eligible for full-rate testing beyond those already isApproved on-chain
-# (see #681) — candidates being considered for approval.
+# Optional full-rate overrides for unapproved providers. Effective only when
+# CALIBRATION_USE_ONLY_APPROVED_PROVIDERS=false.
 # CALIBRATION_EXPECTED_APPROVED_SP_IDS=1234,5678
 # CALIBRATION_EXPECTED_APPROVED_SP_ADDRESSES=0xAbCd...,0x1234...
 
@@ -193,8 +184,8 @@ MAINNET_MAINTENANCE_WINDOW_MINUTES=20
 # MAINNET_BLOCKED_SP_IDS=1234,5678
 # MAINNET_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
 
-# SPs eligible for full-rate testing beyond those already isApproved on-chain
-# (see #681) — candidates being considered for approval.
+# Optional full-rate overrides for unapproved providers. Effective only when
+# MAINNET_USE_ONLY_APPROVED_PROVIDERS=false.
 # MAINNET_EXPECTED_APPROVED_SP_IDS=1234,5678
 # MAINNET_EXPECTED_APPROVED_SP_ADDRESSES=0xAbCd...,0x1234...
 

apps/backend/src/common/sp-tier.ts

@@ -1,12 +1,6 @@
-import { INetworkConfig } from "src/config/types.js";
+import type { INetworkConfig } from "src/config/types.js";
 
-/**
- * Returns true if the provider qualifies for the full-rate testing tier:
- * already `isApproved` on-chain (approved SPs must stay fully monitored
- * regardless of manual-list staleness), or manually curated as an
- * expected-approval candidate. Every other SP (new, unknown, or not yet
- * vetted) defaults to the trickle tier — see #681.
- */
+/** Returns whether a provider qualifies for full-rate testing. */
 export function isFullRateTier(
   cfg: Pick<INetworkConfig, "expectedApprovedSpAddresses" | "expectedApprovedSpIds">,
   address: string,

apps/backend/src/config/constants.ts

@@ -37,12 +37,8 @@ export const networkDefaults = {
 } satisfies NetworkDefaults;
 
 /**
- * Rate/target ceiling applied to SPs that are not on the full-rate tier (see
- * `isFullRateTier` in `common/sp-tier.ts`) — i.e. not `isApproved` on-chain
- * and not in `EXPECTED_APPROVED_SP_IDS`/`EXPECTED_APPROVED_SP_ADDRESSES`.
- *
- * Deliberately NOT env-configurable per network: this tier exists to bound
- * worst-case wallet spend on new/unknown SPs (#681).
+ * Fixed rates and dataset target for eligible providers outside the full-rate
+ * tier. Kept non-configurable to limit wallet-spend exposure.
  */
 export const trickleTierRates = {
   dealsPerSpPerHour: 1 / 4, // 1 data-storage check every 4 hours

apps/backend/src/config/types.ts

@@ -141,14 +141,7 @@ export type BaseNetworkConfig = {
   blockedSpIds: Set<string>;
   blockedSpAddresses: Set<string>;
 
-  /**
-   * Manually curated SPs that should receive full-rate testing even though
-   * they aren't (yet) `isApproved` on-chain — e.g. candidates being
-   * considered for approval. Combined with `isApproved` to gate the
-   * full-rate tier (see `isFullRateTier` in `common/sp-tier.ts`); every
-   * other active, unblocked SP defaults to the trickle tier
-   * (`trickleTierRates` in `config/constants.ts`). See #681.
-   */
+  /** Provider IDs and addresses granted full-rate testing before on-chain approval. */
   expectedApprovedSpIds: Set<string>;
   expectedApprovedSpAddresses: Set<string>;
 

apps/backend/src/deal/deal.service.ts

@@ -124,8 +124,7 @@ export class DealService {
    * Pick which data-set slot this deal will target.
    *
    * Policy:
-   *   - If `minDataSets > 1` (the caller's per-SP tier target — see
-   *     `isFullRateTier`) and a random index > 0 is selected, probe that
+   *   - If `minDataSets > 1` and a random index > 0 is selected, probe that
    *     slot first. If live, use it. If missing or terminated, fall through
    *     to baseline (data_set_creation owns repair/provisioning).
    *   - Probe baseline. If terminated, throw `DealJobTerminatedDataSetError`

apps/backend/src/jobs/jobs.service.spec.ts

@@ -1566,7 +1566,6 @@ describe("JobsService schedule rows", () => {
     } as unknown as JobsServiceDeps[0];
     service = buildService({ configService });
 
-    // Neither approved nor on the expected-approved list -> trickle tier.
     providerRegistryRepositoryMock.findActiveAddresses.mockResolvedValueOnce([{ address: "0xaaa", isApproved: false }]);
 
     await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK);

apps/backend/src/jobs/jobs.service.ts

@@ -1078,9 +1078,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
       return;
     }
 
-    // Defensive gate: schedules are only created for full-rate-tier providers, but a
-    // stale enqueued job (e.g. after the provider dropped out of the full-rate tier)
-    // must still no-op safely rather than spend on a throwaway data set.
+    // Re-check eligibility because queued jobs can outlive their schedule.
     const provider = await this.storageProviderRepository.findByAddress(spAddress, network);
     const isFullTier = isFullRateTier(networkCfg, spAddress, provider?.isApproved ?? false, provider?.id);
     if (!isFullTier) {
@@ -1254,17 +1252,6 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
     }
   }
 
-  /**
-   * Computes job-cadence intervals for a network.
-   *
-   * `deal`/`data_set_creation` (and the `minDataSets` target) are tiered:
-   * a `Full` variant driven by the network's configured rate, and a fixed,
-   * conservative `Trickle` variant (`trickleTierRates`) applied to SPs that
-   * aren't on the full-rate tier — see `isFullRateTier`. Every other job
-   * type stays a single network-wide cadence for now; only deals and
-   * data-set creation drive the wallet-spend the trickle tier exists to
-   * bound (#681).
-   */
   private getIntervalSecondsForRates(network: Network): {
     dealIntervalSecondsFull: number;
     dealIntervalSecondsTrickle: number;
@@ -1394,9 +1381,6 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
     const trickleTierAddresses: string[] = [];
 
     for (const { address, providerId, isApproved } of unblockedProviders) {
-      // Full-rate tier: on-chain approved, or manually curated as an
-      // expected-approval candidate. Everyone else gets the trickle tier,
-      // which bounds worst-case spend on new/unknown SPs — see #681.
       let dealIntervalSeconds = dealIntervalSecondsFull;
       let dataSetCreationIntervalSeconds = dataSetCreationIntervalSecondsFull;
       let minDataSets = minDataSetsFull;
@@ -1434,7 +1418,6 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
           dataSetCreationStartAt,
         );
       }
-      // The trickle-tier SPs never get the lifecycle canary scheduled at all.
       if (lifecycleCheckScheduleEnabled && isFullTier) {
         await this.jobScheduleRepository.upsertSchedule(
           "data_set_lifecycle_check",
@@ -1492,9 +1475,6 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
         });
       }
     } else if (trickleTierAddresses.length > 0) {
-      // A provider that dropped out of the full-rate tier (e.g. lost
-      // isApproved, or was removed from EXPECTED_APPROVED) isn't eligible
-      // for dataSet lifecycle canary check.
       const removed = await this.jobScheduleRepository.deleteSchedulesForAddresses(
         "data_set_lifecycle_check",
         trickleTierAddresses,

apps/backend/src/jobs/repositories/job-schedule.repository.ts

@@ -144,18 +144,6 @@ export class JobScheduleRepository {
     return typeof rowCount === "number" ? rowCount : 0;
   }
 
-  /**
-   * Deletes schedule rows for a specific job type and set of addresses.
-   *
-   * Used to stop a per-SP job (e.g. `data_set_lifecycle_check`) for
-   * addresses that dropped out of eligibility (e.g. left the full-rate
-   * tier — see #681) without touching their other job-type schedules.
-   *
-   * @param jobType - The job type to remove schedules for.
-   * @param addresses - Provider addresses whose schedule for this job type should be deleted.
-   * @param network - Only remove schedules belonging to this network.
-   * @returns Array of storage provider addresses whose schedules were deleted.
-   */
   async deleteSchedulesForAddresses(jobType: JobType, addresses: string[], network: Network): Promise<string[]> {
     if (addresses.length === 0) return [];
     const [rows] = (await this.dataSource.query(

docs/environment-variables.md

@@ -518,7 +518,7 @@ CALIBRATION_SUBGRAPH_ENDPOINT=https://api.goldsky.com/api/public/<project>/subgr
 
 **Role**: Minimum number of datasets provisioned per storage provider before running checks on this network. When > 1, the `data_set_creation` job is responsible for provisioning any additional datasets.
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs target a fixed 1 dataset regardless of this value. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs target one dataset regardless of this value.
 
 ---
 
@@ -546,14 +546,16 @@ Dealbot uses pg-boss for rate-based scheduling — see [Jobs (pg-boss)](#jobs-pg
 
 ### Full-rate vs. trickle tier
 
-Every active, unblocked SP is assigned to one of two testing tiers, re-evaluated on every scheduler tick:
+Provider eligibility is applied before tiering. When `<NET>_USE_ONLY_APPROVED_PROVIDERS=true`, only active, on-chain-approved providers are scheduled; expected-approved lists do not override this setting. When it is `false`, all active providers are eligible unless blocked.
+
+Eligible providers are assigned to one of two tiers on each scheduler tick:
 
 - **Full-rate tier**: the SP is already `isApproved` on-chain (FWSS), **or** its ID/address is listed in [`<NET>_EXPECTED_APPROVED_SP_IDS`](#net_expected_approved_sp_ids) / [`<NET>_EXPECTED_APPROVED_SP_ADDRESSES`](#net_expected_approved_sp_addresses). It gets the configured `<NET>_DEALS_PER_SP_PER_HOUR`, `<NET>_DATASET_CREATIONS_PER_SP_PER_HOUR`, and `<NET>_MIN_NUM_DATASETS_FOR_CHECKS` rates, and (if enabled) the `data_set_lifecycle_check` canary.
-- **Trickle tier**: every other SP — new, unknown, or not yet vetted. Deals and dataset-creation attempts are capped at a fixed rate of one every 4 hours, with a target of 1 dataset, regardless of the configured full-rate values. The `data_set_lifecycle_check` canary is never scheduled for trickle-tier SPs at all (not just at a slower rate), since it creates and terminates a real throwaway dataset on every run.
+- **Trickle tier**: every other eligible SP. Deals and dataset-creation attempts run at a fixed rate of one every 4 hours, with a target of one dataset. The `data_set_lifecycle_check` canary is not scheduled because every run creates and terminates a real throwaway dataset.
 
-The trickle tier's rate is a fixed constant (`trickleTierRates` in `src/config/constants.ts`), not env-configurable per network — it exists specifically to bound worst-case wallet spend on SPs dealbot hasn't vetted yet, so it can't be loosened by config alone. Retrieval, pull-check, and piece-cleanup cadences are not tiered; they apply uniformly regardless of tier.
+Trickle rates are fixed in `trickleTierRates` and are not configurable. Retrieval, pull-check, and piece-cleanup cadences are shared by both tiers.
 
-**Why**: before this, every active SP — including brand-new or ephemeral ones that appeared on-chain and were not yet blocklisted — received full-rate testing immediately, which could drain the dealbot wallet before anyone noticed and blocklisted the SP. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Why**: when testing includes unapproved providers, newly registered or ephemeral SPs previously received full-rate testing until manually blocklisted, increasing wallet-spend exposure. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
 
 ### `<NET>_DEALS_PER_SP_PER_HOUR`
 
@@ -566,7 +568,7 @@ The trickle tier's rate is a fixed constant (`trickleTierRates` in `src/config/c
 
 **Notes**: Fractional values are supported (e.g. `0.25` ⇒ one deal every 4 hours per SP).
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs are capped at a fixed one deal every 4 hours regardless of this value. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs use the fixed trickle rate.
 
 ---
 
@@ -644,7 +646,7 @@ The trickle tier's rate is a fixed constant (`trickleTierRates` in `src/config/c
 
 **Role**: Target dataset-creation rate per storage provider on this network.
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs are capped at a fixed one attempt every 4 hours regardless of this value. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). Trickle-tier SPs use the fixed trickle rate.
 
 ---
 
@@ -745,14 +747,12 @@ check types (data-storage, retrieval, and data-retention). Matching is case-inse
 - **Required**: No
 - **Default**: `""` (empty — no manually-listed candidates)
 
-**Role**: Manually curated list, by provider numeric ID, of SPs that should receive [full-rate tier](#full-rate-vs-trickle-tier) testing even though they aren't (yet) `isApproved` on-chain — e.g. candidates being considered for approval that need enough test data to evaluate. SPs already `isApproved` on-chain get the full-rate tier automatically and do not need to be listed here.
+**Role**: Provider IDs granted [full-rate testing](#full-rate-vs-trickle-tier) before on-chain approval. Approved providers do not need to be listed. This setting has no effect when `<NET>_USE_ONLY_APPROVED_PROVIDERS=true` because unapproved providers are excluded before tiering.
 
-**Notes**: An SP only needs to appear in **either** `<NET>_EXPECTED_APPROVED_SP_IDS` **or** `<NET>_EXPECTED_APPROVED_SP_ADDRESSES` — not both — to qualify. They're checked independently (matching either one is sufficient), the same as `<NET>_BLOCKED_SP_IDS`/`<NET>_BLOCKED_SP_ADDRESSES`. Use whichever identifier you have on hand; listing both for the same SP is harmless but redundant.
+**Notes**: Matching either the ID list or address list is sufficient.
 
 **Example**: `<NET>_EXPECTED_APPROVED_SP_IDS=1234,5678`
 
-**See also**: [issue #681](https://github.com/FilOzone/dealbot/issues/681)
-
 ---
 
 ### `<NET>_EXPECTED_APPROVED_SP_ADDRESSES`
@@ -761,14 +761,10 @@ check types (data-storage, retrieval, and data-retention). Matching is case-inse
 - **Required**: No
 - **Default**: `""` (empty — no manually-listed candidates)
 
-**Role**: Address-based counterpart to [`<NET>_EXPECTED_APPROVED_SP_IDS`](#net_expected_approved_sp_ids) — same effect, matched by provider address instead of numeric ID. Matching is case-insensitive.
-
-**Notes**: Either this or `<NET>_EXPECTED_APPROVED_SP_IDS` is sufficient for a given SP; you don't need to set both.
+**Role**: Address-based counterpart to [`<NET>_EXPECTED_APPROVED_SP_IDS`](#net_expected_approved_sp_ids). Matching is case-insensitive.
 
 **Example**: `<NET>_EXPECTED_APPROVED_SP_ADDRESSES=0xAbCd...,0x1234...`
 
-**See also**: [issue #681](https://github.com/FilOzone/dealbot/issues/681)
-
 ---
 
 ### `<NET>_MAX_DATASET_STORAGE_SIZE_BYTES`
@@ -947,7 +943,7 @@ These variables are **global** (not per-network) and control the shared pg-boss
 
 **Notes**: Fractional values are supported. For example, `0.25` means one deal every 4 hours per storage provider.
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier).
 
 ---
 
@@ -980,7 +976,7 @@ These variables are **global** (not per-network) and control the shared pg-boss
 - Increase if you want more datasets per provider to raise the density of data-retention proof samples.
 - Decrease to reduce on-chain footprint per provider during testing.
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier).
 
 **See also**: [`docs/data-set-creation.md`](./data-set-creation.md)
 
@@ -1011,7 +1007,7 @@ These variables are **global** (not per-network) and control the shared pg-boss
 
 **Notes**: Fractional values are supported. For example, `0.5` means one dataset creation every 2 hours per storage provider.
 
-**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier). See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only applies to SPs on the [full-rate tier](#full-rate-vs-trickle-tier).
 
 ---
 
@@ -1025,7 +1021,7 @@ These variables are **global** (not per-network) and control the shared pg-boss
 
 **Notes**: Self-contained — it does not touch the managed check data sets and does not depend on `data_set_creation`. When disabled, stale schedules are removed so they stop enqueuing no-op jobs.
 
-**Tiering**: Regardless of this setting, the canary is **never scheduled for trickle-tier SPs** — see [full-rate vs. trickle tier](#full-rate-vs-trickle-tier). It creates and terminates a real throwaway dataset every run, so trickle-tier (unvetted) SPs are fully excluded rather than just throttled. A provider that drops out of the full-rate tier has its stale schedule row removed on the next scheduler tick. See [issue #681](https://github.com/FilOzone/dealbot/issues/681).
+**Tiering**: Only [full-rate providers](#full-rate-vs-trickle-tier) receive this canary because each run creates and terminates a real throwaway dataset.
 
 **See also**: [`docs/checks/data-set-lifecycle-check.md`](./checks/data-set-lifecycle-check.md)
 
@@ -1043,7 +1039,7 @@ These variables are **global** (not per-network) and control the shared pg-boss
 
 **Notes**: Independent of `DATASET_CREATIONS_PER_SP_PER_HOUR`. Fractional values are supported.
 
-**Tiering**: This rate is irrelevant for trickle-tier SPs — the canary is never scheduled for them at all. See [`DATASET_LIFECYCLE_CHECK_ENABLED`](#dataset_lifecycle_check_enabled) and [full-rate vs. trickle tier](#full-rate-vs-trickle-tier).
+**Tiering**: Only applies to [full-rate providers](#full-rate-vs-trickle-tier).
 
 ---