Add RPA metrics(in bucket) endpoint for SPs with filters and DC

lukasz-wal · lukasz-wal · commit 4491dc4e5e2b · 2026-02-17T15:18:37.000+01:00
diff --git a/prisma/sql/getUrlFinderProviderMetricWeeklyAcc.sql b/prisma/sql/getUrlFinderProviderMetricWeeklyAcc.sql
@@ -0,0 +1,50 @@
+-- @param {String} $1:metricType
+-- @param {Float} $2:bucketSize
+-- @param {DateTime} $3:startDate
+-- @param {DateTime} $4:endDate
+
+WITH "metric_data" AS (
+    select
+    	provider,
+        value,
+        DATE_TRUNC('week', "tested_at")                       AS "week"
+    FROM "storage_provider_url_finder_metric_value"
+    JOIN "storage_provider_url_finder_metric"
+      ON "storage_provider_url_finder_metric_value"."metric_id" = "storage_provider_url_finder_metric"."id"
+    WHERE "storage_provider_url_finder_metric"."metric_type"::text = $1
+      AND ($3::date IS NULL OR "tested_at" >= $3)
+      AND ($4::date IS NULL OR "tested_at" <= $4)
+      AND value IS NOT NULL
+),
+"provider_weekly" AS (select      "week"            AS "week",
+                                  "provider"        AS "provider",
+                                  "total_deal_size" AS "total_deal_size"
+                           from "providers_weekly_acc"
+                           where ($3::date is null or "week" >= $3)
+                             and ($4::date is null or "week" <= $4)),
+"bucketed" AS (
+    SELECT
+        "metric_data"."week",
+        "metric_data"."provider",
+        FLOOR("metric_data"."value" / $2)           AS "bucket_index"
+    FROM "metric_data"
+),
+"bucketed_with_deal" AS (
+    SELECT
+        "bucketed"."week",
+        "bucketed"."bucket_index",
+        "provider_weekly"."total_deal_size"
+    FROM "bucketed"
+    LEFT JOIN "provider_weekly"
+      ON "provider_weekly"."week" = "bucketed"."week"
+     AND "provider_weekly"."provider" = "bucketed"."provider"
+)
+SELECT
+    "week"                                                    AS "week",
+    "bucket_index" * $2                                       AS "valueFromExclusive",
+    ("bucket_index" + 1) * $2                                 AS "valueToInclusive",
+    COUNT(*)                                                  AS "count",
+    SUM("total_deal_size")::bigint                            AS "totalDatacap"
+FROM "bucketed_with_deal"
+GROUP BY "week", "bucket_index"
+ORDER BY "week", "bucket_index";
diff --git a/src/controller/stats/storage-providers/storage-providers-stats.controller.ts b/src/controller/stats/storage-providers/storage-providers-stats.controller.ts
@@ -1,13 +1,18 @@
 import { CacheTTL } from '@nestjs/cache-manager';
 import { Controller, Get, Query } from '@nestjs/common';
 import { ApiOkResponse, ApiOperation } from '@nestjs/swagger';
+import { groupBy } from 'lodash';
+import { StorageProviderUrlFinderMetricType } from 'prisma/generated/client';
+import { getUrlFinderProviderMetricWeeklyAcc } from 'prisma/generated/client/sql';
 import { DataType } from 'src/controller/allocators/types.allocators';
 import { FilPlusEditionControllerBase } from 'src/controller/base/filplus-edition-controller-base';
 import { FilPlusEditionRequest } from 'src/controller/base/types.filplus-edition-controller-base';
 import { StorageProviderComplianceMetricsRequest } from 'src/controller/storage-providers/types.storage-providers';
 import { PrismaService } from 'src/db/prisma.service';
 import {
+  HistogramTotalDatacap,
   HistogramWeek,
+  HistogramWeekResults,
   RetrievabilityWeek,
 } from 'src/service/histogram-helper/types.histogram-helper';
 import { IpniMisreportingCheckerService } from 'src/service/ipni-misreporting-checker/ipni-misreporting-checker.service';
@@ -22,9 +27,12 @@ import {
   StorageProviderComplianceMetrics,
   StorageProviderComplianceWeek,
 } from 'src/service/storage-provider/types.storage-provider';
-import { stringToBool, stringToDate } from 'src/utils/utils';
+import { bigIntToNumber, stringToBool, stringToDate } from 'src/utils/utils';
 import { GetRetrievabilityWeeklyRequest } from '../allocators/types.allocator-stats';
-import { UrlFinderStorageProviderMetricDataRequest } from './types.storage-providers-stats';
+import {
+  UrlFinderStorageProviderMetricBaseRequest,
+  UrlFinderStorageProviderMetricTypeRequest,
+} from './types.storage-providers-stats';
 
 @Controller('stats/acc/providers')
 @CacheTTL(1000 * 60 * 30) // 30 minutes
@@ -116,7 +124,7 @@ export class StorageProvidersAccStatsController extends FilPlusEditionController
       'Get SP Url Finder retrieval result codes metrics for storage providers',
   })
   public async getStorageProvidersUrlFinderRetrievalCodesData(
-    @Query() query: UrlFinderStorageProviderMetricDataRequest,
+    @Query() query: UrlFinderStorageProviderMetricBaseRequest,
   ): Promise<StorageProviderMetricHistogramDailyResponse> {
     const metrics =
       await this.storageProviderUrlFinderService.getUrlFinderSnapshotsForProviders(
@@ -131,4 +139,70 @@ export class StorageProvidersAccStatsController extends FilPlusEditionController
 
     return result;
   }
+
+  @Get('/rpa/metric/')
+  @ApiOperation({
+    summary: 'Get RPA metrics for storage providers',
+  })
+  public async getStorageProvidersUrlFinderMetricData(
+    @Query() query: UrlFinderStorageProviderMetricTypeRequest,
+  ): Promise<HistogramWeek> {
+    const startDate = query?.startDate
+      ? stringToDate(query?.startDate)
+      : undefined;
+
+    const endDate = query?.endDate ? stringToDate(query?.endDate) : undefined;
+    let bucketSize = 2000;
+
+    switch (query.metricType) {
+      case StorageProviderUrlFinderMetricType.RPA_RETRIEVABILITY:
+        bucketSize = 0.05;
+        break;
+      case StorageProviderUrlFinderMetricType.TTFB:
+        bucketSize = 2000;
+        break;
+      case StorageProviderUrlFinderMetricType.BANDWIDTH:
+        bucketSize = 10;
+        break;
+    }
+
+    const metricWeekData = await this.prismaService.$queryRawTyped(
+      getUrlFinderProviderMetricWeeklyAcc(
+        query.metricType,
+        bucketSize,
+        startDate,
+        endDate,
+      ),
+    );
+
+    const rowsByWeek = groupBy(metricWeekData, (r) => r.week.toISOString());
+
+    const weekResults: HistogramWeekResults[] = Object.entries(rowsByWeek).map(
+      ([weekIso, weekRows]) => {
+        const histograms = weekRows.map(
+          (r) =>
+            new HistogramTotalDatacap(
+              r.valueFromExclusive,
+              r.valueToInclusive,
+              bigIntToNumber(r.count),
+              r.totalDatacap,
+            ),
+        );
+
+        const total = weekRows.reduce(
+          (sum, r) => sum + bigIntToNumber(r.count),
+          0,
+        );
+
+        return new HistogramWeekResults(new Date(weekIso), total, histograms);
+      },
+    );
+
+    const totalAcrossAllWeeks = weekResults.reduce(
+      (sum, w) => sum + w.total,
+      0,
+    );
+
+    return new HistogramWeek(totalAcrossAllWeeks, weekResults);
+  }
 }
diff --git a/src/controller/stats/storage-providers/types.storage-providers-stats.ts b/src/controller/stats/storage-providers/types.storage-providers-stats.ts
@@ -1,6 +1,7 @@
 import { ApiPropertyOptional } from '@nestjs/swagger';
+import { StorageProviderUrlFinderMetricType } from 'prisma/generated/client';
 
-export class UrlFinderStorageProviderMetricDataRequest {
+export class UrlFinderStorageProviderMetricBaseRequest {
   @ApiPropertyOptional({
     description: 'Requested start date to fetch historical metrics',
     format: 'date-time',
@@ -15,3 +16,12 @@ export class UrlFinderStorageProviderMetricDataRequest {
   })
   endDate?: string;
 }
+
+export class UrlFinderStorageProviderMetricTypeRequest extends UrlFinderStorageProviderMetricBaseRequest {
+  @ApiPropertyOptional({
+    description: 'Compliance score to filter by',
+    enum: StorageProviderUrlFinderMetricType,
+    example: StorageProviderUrlFinderMetricType.RPA_RETRIEVABILITY,
+  })
+  metricType: StorageProviderUrlFinderMetricType;
+}
diff --git a/src/service/histogram-helper/histogram-helper.service.ts b/src/service/histogram-helper/histogram-helper.service.ts
@@ -1,9 +1,9 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { groupBy } from 'lodash';
 import {
-  Histogram,
-  HistogramWeekResults,
+  HistogramTotalDatacap,
   HistogramWeekFlat,
+  HistogramWeekResults,
 } from './types.histogram-helper';
 
 @Injectable()
@@ -21,7 +21,7 @@ export class HistogramHelperService {
 
     for (const week in histogramsByWeek) {
       const weekResults = histogramsByWeek[week].map((r) => {
-        return new Histogram(
+        return new HistogramTotalDatacap(
           r.valueFromExclusive,
           r.valueToInclusive,
           r.count,
@@ -94,7 +94,9 @@ export class HistogramHelperService {
 
       if (missingBuckets.length > 0) {
         histogramWeek.results.push(
-          ...missingBuckets.map((v) => new Histogram(v - maxMinSpan, v, 0, 0n)),
+          ...missingBuckets.map(
+            (v) => new HistogramTotalDatacap(v - maxMinSpan, v, 0, 0n),
+          ),
         );
 
         histogramWeek.results.sort(
diff --git a/src/service/histogram-helper/types.histogram-helper.ts b/src/service/histogram-helper/types.histogram-helper.ts
@@ -1,6 +1,6 @@
 import { ApiProperty } from '@nestjs/swagger';
 
-export class Histogram {
+export class HistogramBase {
   @ApiProperty({
     description: 'Bucket (valueFromExclusive, valueToInclusive> starting value',
   })
@@ -16,6 +16,18 @@ export class Histogram {
   })
   count: number;
 
+  constructor(
+    valueFromExclusive: number,
+    valueToInclusive: number,
+    count: number,
+  ) {
+    this.valueFromExclusive = valueFromExclusive;
+    this.valueToInclusive = valueToInclusive;
+    this.count = count;
+  }
+}
+
+export class HistogramTotalDatacap extends HistogramBase {
   @ApiProperty({
     description:
       'Total datacap of allocators / storage providers in the bucket',
@@ -31,9 +43,7 @@ export class Histogram {
     count: number,
     totalDatacap: bigint,
   ) {
-    this.valueFromExclusive = valueFromExclusive;
-    this.valueToInclusive = valueToInclusive;
-    this.count = count;
+    super(valueFromExclusive, valueToInclusive, count);
     this.totalDatacap = totalDatacap;
   }
 }
@@ -52,17 +62,17 @@ export class HistogramWeekResults {
   })
   total: number;
 
-  @ApiProperty({ type: Histogram, isArray: true })
-  results: Histogram[];
+  @ApiProperty({ type: HistogramTotalDatacap, isArray: true })
+  results: HistogramTotalDatacap[];
 
-  constructor(week: Date, total: number, results: Histogram[]) {
+  constructor(week: Date, total: number, results: HistogramTotalDatacap[]) {
     this.week = week;
     this.total = total;
     this.results = results;
   }
 }
 
-export class HistogramWeekFlat extends Histogram {
+export class HistogramWeekFlat extends HistogramTotalDatacap {
   week: Date;
 
   constructor(
@@ -91,7 +101,7 @@ export class RetrievabilityHistogramWeek extends HistogramWeekResults {
   constructor(
     week: Date,
     total: number,
-    results: Histogram[],
+    results: HistogramTotalDatacap[],
     averageHttpSuccessRate: number,
     averageUrlFinderSuccessRate: number,
   ) {
diff --git a/src/service/storage-provider-url-finder/storage-provider-url-finder.service.ts b/src/service/storage-provider-url-finder/storage-provider-url-finder.service.ts
diff --git a/src/service/storage-provider-url-finder/types.storage-provider-url-finder.service.ts b/src/service/storage-provider-url-finder/types.storage-provider-url-finder.service.ts
