Old datacap balance of Allocators weekly

prisma/migrations/20250318163009_old_datacap_balance/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "old_datacap_balance_nv22" (
+    "allocator" TEXT NOT NULL,
+    "old_dc_balance" BIGINT NOT NULL,
+
+    CONSTRAINT "old_datacap_balance_nv22_pkey" PRIMARY KEY ("allocator")
+);
+
+-- CreateTable
+CREATE TABLE "old_datacap_balance_weekly" (
+    "week" TIMESTAMP(3) NOT NULL,
+    "allocator" TEXT NOT NULL,
+    "old_dc_balance" BIGINT NOT NULL,
+
+    CONSTRAINT "old_datacap_balance_weekly_pkey" PRIMARY KEY ("week","allocator")
+);

prisma/migrations/20250318163723_old_datacap_balance_allocations/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `allocations` to the `old_datacap_balance_weekly` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "old_datacap_balance_weekly" ADD COLUMN     "allocations" BIGINT NOT NULL;

prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
+# It should be added in your version-control system (e.g., Git)
 provider = "postgresql"

prisma/schema.prisma

@@ -328,6 +328,22 @@ model ipni_publisher_advertisement {
   @@index([publisher_id])
 }
 
+model old_datacap_balance_nv22 {
+  allocator      String
+  old_dc_balance BigInt
+
+  @@id(allocator)
+}
+
+model old_datacap_balance_weekly {
+  week           DateTime
+  allocator      String
+  old_dc_balance BigInt
+  allocations    BigInt
+
+  @@id([week, allocator])
+}
+
 enum ClientReportCheck {
   STORAGE_PROVIDER_DISTRIBUTION_ALL_LOCATED_IN_THE_SAME_REGION
   STORAGE_PROVIDER_DISTRIBUTION_PROVIDERS_EXCEED_PROVIDER_DEAL

prisma/sql/getOldDatacapBalanceWeekly.sql

@@ -0,0 +1,38 @@
+with "weeks" as (select date_trunc('week', "dates") as "week"
+                 from generate_series(
+                              to_timestamp(3847920 * 30 + 1598306400) - interval '1 week', -- start week ahead of nv22
+                              current_timestamp,
+                              '1 week'::interval
+                      ) as "dates"),
+
+     "old_allocators_weekly" as (select distinct "weeks"."week",
+                                                 "old_datacap_balance_nv22"."allocator"
+                                 from "weeks",
+                                      "old_datacap_balance_nv22"),
+
+     "old_dc_balance" as (select "old_allocators_weekly"."week",
+                                 "old_allocators_weekly"."allocator",
+                                 greatest(
+                                         0,
+                                         (
+                                             "old_datacap_balance_nv22"."old_dc_balance"
+                                                 -
+                                             coalesce("allocators_weekly_acc"."total_sum_of_allocations", 0)
+                                             )
+                                 ) as "old_dc_balance"
+                          from "old_allocators_weekly"
+                                   inner join "old_datacap_balance_nv22" using ("allocator")
+                                   left join "allocators_weekly_acc" using ("week", "allocator"))
+
+select "week",
+       "allocator",
+       "old_dc_balance",
+       coalesce(
+               (
+                           lag("old_dc_balance") over (partition by "allocator" order by "week" asc)
+                       - "old_dc_balance"
+                   ),
+               0
+       ) as "allocations"
+from "old_dc_balance"
+order by "week";

prismaDmob/sql/getOldDatacapBalanceNv22.sql

@@ -0,0 +1,17 @@
+with "old_dc_allowance" as (select "addressId"      as "allocator",
+                                   sum("allowance") as "total_allowance"
+                            from "verifier_allowance"
+                            where "height" < 3847920 -- nv22 start
+                            group by "addressId"),
+
+     "old_dc_allocations" as (select "verifierAddressId" as "allocator",
+                                     sum("allowance")    as "total_allowance"
+                              from "verified_client_allowance"
+                              where "height" < 3847920 -- nv22 start
+                              group by "verifierAddressId")
+
+select "dc_in"."allocator",
+       greatest(0, "dc_in"."total_allowance" - "dc_out"."total_allowance")::bigint as "old_dc_balance"
+from "old_dc_allowance" as "dc_in"
+         inner join "old_dc_allocations" as "dc_out"
+                    using ("allocator");

src/aggregation/aggregation-table.ts

@@ -9,6 +9,8 @@ export enum AggregationTable {
   ClientProviderDistributionWeekly,
   ClientProviderDistributionWeeklyAcc,
   ClientReplicaDistribution,
+  OldDatacapBalanceNv22,
+  OldDatacapBalanceWeekly,
   ProviderFirstClient,
   ProviderRetrievabilityDaily,
   ProvidersWeekly,

src/aggregation/runners/old-datacap-balance-nv22.runner.ts

@@ -0,0 +1,55 @@
+import { getOldDatacapBalanceNv22 } from 'prismaDmob/generated/client/sql';
+import {
+  AggregationRunner,
+  AggregationRunnerRunServices,
+} from '../aggregation-runner';
+import { AggregationTable } from '../aggregation-table';
+
+export class OldDatacapBalanceNv22Runner implements AggregationRunner {
+  public async run({
+    prismaService,
+    prismaDmobService,
+    prometheusMetricService,
+  }: AggregationRunnerRunServices): Promise<void> {
+    const runnerName = this.getName();
+
+    const {
+      startGetDataTimerByRunnerNameMetric,
+      startStoreDataTimerByRunnerNameMetric,
+    } = prometheusMetricService.aggregateMetrics;
+
+    const getDataEndTimerMetric =
+      startGetDataTimerByRunnerNameMetric(runnerName);
+
+    const result = await prismaDmobService.$queryRawTyped(
+      getOldDatacapBalanceNv22(),
+    );
+
+    getDataEndTimerMetric();
+
+    const data = result.map((dmobResult) => ({
+      allocator: dmobResult.allocator,
+      old_dc_balance: dmobResult.old_dc_balance,
+    }));
+
+    const storeDataEndTimerMetric =
+      startStoreDataTimerByRunnerNameMetric(runnerName);
+
+    await prismaService.$executeRaw`delete from old_datacap_balance_nv22;`;
+    await prismaService.old_datacap_balance_nv22.createMany({ data });
+
+    storeDataEndTimerMetric();
+  }
+
+  getFilledTables(): AggregationTable[] {
+    return [AggregationTable.OldDatacapBalanceNv22];
+  }
+
+  getDependingTables(): AggregationTable[] {
+    return [];
+  }
+
+  getName(): string {
+    return 'Old Datacap Balance Nv22 Runner';
+  }
+}

src/aggregation/runners/old-datacap-balance-weekly.runner.ts

@@ -0,0 +1,59 @@
+import { getOldDatacapBalanceWeekly } from 'prisma/generated/client/sql';
+import {
+  AggregationRunner,
+  AggregationRunnerRunServices,
+} from '../aggregation-runner';
+import { AggregationTable } from '../aggregation-table';
+
+export class OldDatacapBalanceWeeklyRunner implements AggregationRunner {
+  public async run({
+    prismaService,
+    prometheusMetricService,
+  }: AggregationRunnerRunServices): Promise<void> {
+    const runnerName = this.getName();
+
+    const {
+      startGetDataTimerByRunnerNameMetric,
+      startStoreDataTimerByRunnerNameMetric,
+    } = prometheusMetricService.aggregateMetrics;
+
+    const getDataEndTimerMetric =
+      startGetDataTimerByRunnerNameMetric(runnerName);
+
+    const result = await prismaService.$queryRawTyped(
+      getOldDatacapBalanceWeekly(),
+    );
+
+    getDataEndTimerMetric();
+
+    const data = result.map((result) => ({
+      week: result.week,
+      allocator: result.allocator,
+      old_dc_balance: result.old_dc_balance,
+      allocations: result.allocations,
+    }));
+
+    const storeDataEndTimerMetric =
+      startStoreDataTimerByRunnerNameMetric(runnerName);
+
+    await prismaService.$executeRaw`delete from old_datacap_balance_weekly;`;
+    await prismaService.old_datacap_balance_weekly.createMany({ data });
+
+    storeDataEndTimerMetric();
+  }
+
+  getFilledTables(): AggregationTable[] {
+    return [AggregationTable.OldDatacapBalanceWeekly];
+  }
+
+  getDependingTables(): AggregationTable[] {
+    return [
+      AggregationTable.AllocatorsWeeklyAcc,
+      AggregationTable.OldDatacapBalanceNv22,
+    ];
+  }
+
+  getName(): string {
+    return 'Old Datacap Balance Weekly Runner';
+  }
+}

src/app.module.ts

@@ -16,6 +16,8 @@ import { ClientClaimsRunner } from './aggregation/runners/client-claims.runner';
 import { ClientProviderDistributionAccRunner } from './aggregation/runners/client-provider-distribution-acc.runner';
 import { ClientProviderDistributionWeeklyRunner } from './aggregation/runners/client-provider-distribution-weekly.runner';
 import { ClientReplicaDistributionRunner } from './aggregation/runners/client-replica-distribution.runner';
+import { OldDatacapBalanceNv22Runner } from './aggregation/runners/old-datacap-balance-nv22.runner';
+import { OldDatacapBalanceWeeklyRunner } from './aggregation/runners/old-datacap-balance-weekly.runner';
 import { ProviderFirstClientRunner } from './aggregation/runners/provider-first-client.runner';
 import { ProviderRetrievabilityBackfillRunner } from './aggregation/runners/provider-retrievability-backfill.runner';
 import { ProviderRetrievabilityRunner } from './aggregation/runners/provider-retrievability.runner';
@@ -31,6 +33,7 @@ import { AllocatorsAccController } from './controller/stats/allocators/allocator
 import { StorageProvidersAccController } from './controller/stats/storage-providers/storage-providers.controller';
 import { AllocatorsController } from './controller/stats/allocators/allocators.controller';
 import { StorageProvidersController } from './controller/stats/storage-providers/storage-providers.controller';
+import { OldDatacapController } from './controller/stats/old-datacap/old-datacap.controller';
 import { PostgresService } from './db/postgres.service';
 import { PostgresDmobService } from './db/postgresDmob.service';
 import { PrismaService } from './db/prisma.service';
@@ -43,6 +46,7 @@ import { GoogleApisService } from './service/googleapis/googleapis.service';
 import { LotusApiService } from './service/lotus-api/lotus-api.service';
 import { LocationService } from './service/location/location.service';
 import { AllocatorTechService } from './service/allocator-tech/allocator-tech.service';
+import { OldDatacapService } from './service/old-datacap/old-datacap.service';
 import { ClientReportGeneratorJobService } from './jobs/client-report-generator-job/client-report-generator-job.service';
 import { ClientProviderDistributionRunner } from './aggregation/runners/client-provider-distribution.runner';
 import { RequestLoggerMiddleware } from './middleware/request-logger.middleware';
@@ -89,6 +93,7 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
     ClientReportControllerRedirect,
     AllocatorReportController,
     AllocatorReportControllerRedirect,
+    OldDatacapController,
     AppController,
   ],
   providers: [
@@ -112,6 +117,8 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
     ClientProviderDistributionWeeklyRunner,
     ClientProviderDistributionAccRunner,
     ClientReplicaDistributionRunner,
+    OldDatacapBalanceNv22Runner,
+    OldDatacapBalanceWeeklyRunner,
     ProviderFirstClientRunner,
     ProviderRetrievabilityRunner,
     ProviderRetrievabilityBackfillRunner,
@@ -135,6 +142,7 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
     ClientReportChecksService,
     AllocatorReportService,
     StorageProviderReportService,
+    OldDatacapService,
     { provide: APP_FILTER, useClass: ErrorHandlerMiddleware },
     { provide: APP_INTERCEPTOR, useClass: CacheInterceptor },
     {
@@ -158,6 +166,8 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
         clientProviderDistributionWeeklyRunner,
         clientProviderDistributionAccRunner,
         clientReplicaDistributionRunner,
+        oldDatacapBalanceNv22Runner,
+        oldDatacapBalanceWeeklyRunner,
         providerFirstClientRunner,
         providerRetrievabilityRunner,
         providerRetrievabilityBackfillRunner,
@@ -175,6 +185,8 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
         clientProviderDistributionWeeklyRunner,
         clientProviderDistributionAccRunner,
         clientReplicaDistributionRunner,
+        oldDatacapBalanceNv22Runner,
+        oldDatacapBalanceWeeklyRunner,
         providerFirstClientRunner,
         providerRetrievabilityRunner,
         providerRetrievabilityBackfillRunner,
@@ -193,6 +205,8 @@ import { GitHubTriggersHandlerService } from './service/github-triggers-handler-
         ClientProviderDistributionWeeklyRunner,
         ClientProviderDistributionAccRunner,
         ClientReplicaDistributionRunner,
+        OldDatacapBalanceNv22Runner,
+        OldDatacapBalanceWeeklyRunner,
         ProviderFirstClientRunner,
         ProviderRetrievabilityRunner,
         ProviderRetrievabilityBackfillRunner,

src/controller/stats/old-datacap/old-datacap.controller.ts

@@ -0,0 +1,17 @@
+import { Controller, Get } from '@nestjs/common';
+import { OldDatacapService } from 'src/service/old-datacap/old-datacap.service';
+import { ApiOkResponse } from '@nestjs/swagger';
+import { OldDatacapAllocatorBalanceWeekResponse } from 'src/service/old-datacap/types.allocator';
+import { CacheTTL } from '@nestjs/cache-manager';
+
+@Controller('stats/old-datacap')
+@CacheTTL(1000 * 60 * 30) // 30 minutes
+export class OldDatacapController {
+  constructor(private readonly oldDatacapService: OldDatacapService) {}
+
+  @Get('allocator-balance')
+  @ApiOkResponse({ type: OldDatacapAllocatorBalanceWeekResponse })
+  public async getAllocatorBalance(): Promise<OldDatacapAllocatorBalanceWeekResponse> {
+    return await this.oldDatacapService.getAllocatorBalance();
+  }
+}

src/service/old-datacap/old-datacap.service.ts

@@ -0,0 +1,38 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { PrismaService } from 'src/db/prisma.service';
+import {
+  OldDatacapAllocatorBalanceWeek,
+  OldDatacapAllocatorBalanceWeekResponse,
+} from './types.allocator';
+
+@Injectable()
+export class OldDatacapService {
+  private readonly logger = new Logger(OldDatacapService.name);
+
+  constructor(private readonly prismaService: PrismaService) {}
+
+  public async getAllocatorBalance(): Promise<OldDatacapAllocatorBalanceWeekResponse> {
+    const dbResults =
+      await this.prismaService.old_datacap_balance_weekly.groupBy({
+        by: ['week'],
+        _count: {
+          allocator: true,
+        },
+        _sum: {
+          old_dc_balance: true,
+          allocations: true,
+        },
+        skip: 1, // first week is a week before nv22, we don't care about it
+        orderBy: {
+          week: 'asc',
+        },
+      });
+    const results: OldDatacapAllocatorBalanceWeek[] = dbResults.map((r) => ({
+      week: r.week,
+      allocators: r._count.allocator,
+      oldDatacap: r._sum.old_dc_balance,
+      allocations: r._sum.allocations,
+    }));
+    return { results };
+  }
+}