diff --git a/lib/preprocess.js b/lib/preprocess.js index 7092bfb..0136f2a 100644 --- a/lib/preprocess.js +++ b/lib/preprocess.js @@ -17,6 +17,7 @@ export class Measurement { constructor (m, pointerize = (v) => v) { this.participantAddress = pointerize(parseParticipantAddress(m.participant_address)) this.retrievalResult = pointerize(getRetrievalResult(m)) + this.alternativeProviderRetrievalResult = pointerize(getAlternativeProviderRetrievalResult(m)) this.cid = pointerize(m.cid) this.minerId = pointerize(m.miner_id) // Note: providerId is recorded by spark-publish but we don't use it for evaluations yet @@ -41,6 +42,14 @@ export class Measurement { this.stationId = pointerize(m.station_id) this.carChecksum = pointerize(m.car_checksum) this.carTooLarge = m.car_too_large + this.alternativeProviderCheck = { + statusCode: m.alternative_provider_check?.status_code, + timeout: m.alternative_provider_check?.timeout, + carTooLarge: m.alternative_provider_check?.car_too_large, + endAt: parseDateTime(m.alternative_provider_check?.end_at), + protocol: m.alternative_provider_check?.protocol, + providerId: pointerize(m.alternative_provider_check?.provider_id) + } } } @@ -319,3 +328,31 @@ export const getRetrievalResult = (measurement) => { return ok ? 'OK' : 'UNKNOWN_ERROR' } + +/** + * Evaluates the alternative provider retrieval result. + * + * Alternative provider retrieval results are evaluated only if the indexer result is `NO_VALID_ADVERTISEMENT` + * and the network retrieval status code is set. + * + * @param {Partial} measurement + * @return {import('./typings.js').RetrievalResult} + */ +export const getAlternativeProviderRetrievalResult = (measurement) => { + if (measurement.indexer_result !== 'NO_VALID_ADVERTISEMENT' || !measurement.alternative_provider_check?.status_code) { + return getRetrievalResult(measurement) + } + + /** @type {Partial} */ + const alternativeProviderMeasurement = { + indexer_result: 'OK', + timeout: measurement.alternative_provider_check?.timeout, + status_code: measurement.alternative_provider_check?.status_code, + car_too_large: measurement.alternative_provider_check?.car_too_large, + end_at: measurement.alternative_provider_check?.end_at, + protocol: measurement.alternative_provider_check?.protocol, + provider_id: measurement.alternative_provider_check?.provider_id + } + + return getRetrievalResult(alternativeProviderMeasurement) +} diff --git a/lib/provider-retrieval-result-stats.js b/lib/provider-retrieval-result-stats.js index b2659de..8f31eb1 100644 --- a/lib/provider-retrieval-result-stats.js +++ b/lib/provider-retrieval-result-stats.js @@ -16,7 +16,7 @@ const withPgClient = fn => async ({ createPgClient, ...args }) => { } export const build = committees => { - /** @type {Map} */ + /** @type {Map} */ const providerRetrievalResultStats = new Map() for (const c of committees) { // IMPORTANT: include minority results in the calculation @@ -25,7 +25,7 @@ export const build = committees => { if (m.retrievalResult.match(/^IPNI_ERROR_5\d\d$/)) continue const minerId = m.minerId - const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0, successfulHttpHead: 0 } + const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0, successfulHttpHead: 0, successfulAltProvider: 0 } retrievalStats.total++ if (m.retrievalResult === 'OK') { retrievalStats.successful++ @@ -36,6 +36,9 @@ export const build = committees => { } } } + + if (m.alternativeProviderRetrievalResult === 'OK') retrievalStats.successfulAltProvider++ + providerRetrievalResultStats.set(minerId, retrievalStats) } } diff --git a/lib/public-stats.js b/lib/public-stats.js index d0a4197..2aa91f5 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -23,6 +23,7 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure try { for (const [minerId, retrievalResultStats] of stats.entries()) { await updateRetrievalStats(pgClient, minerId, retrievalResultStats) + await updateDailyAlternativeProviderRetrievalStats(pgClient, retrievalResultStats) } await updateIndexerQueryStats(pgClient, committees) await updateDailyDealsStats(pgClient, committees, findDealClients) @@ -407,3 +408,25 @@ function buildPerPartyStats (committees, perDealParty, partyName) { ) return flatStats } + +/** + * @param {pg.Client} pgClient + * @param {object} stats + * @param {number} stats.total + * @param {number} stats.successfulAltProvider + */ +export const updateDailyAlternativeProviderRetrievalStats = async (pgClient, { total, successfulAltProvider }) => { + debug('Updating public daily alternative provider retrieval stats: total += %s successful += %s', total, successfulAltProvider) + await pgClient.query(` + INSERT INTO daily_alternative_provider_retrieval_stats + (day, total, successful) + VALUES + (now(), $1, $2) + ON CONFLICT(day) DO UPDATE SET + total = daily_alternative_provider_retrieval_stats.total + $1, + successful = daily_alternative_provider_retrieval_stats.successful + $2 + `, [ + total, + successfulAltProvider + ]) +} diff --git a/lib/retrieval-stats.js b/lib/retrieval-stats.js index b56c44c..8e77544 100644 --- a/lib/retrieval-stats.js +++ b/lib/retrieval-stats.js @@ -60,6 +60,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { const sizeValues = [] let httpSuccesses = 0 let indexerServerErrorCount = 0 + let alternativeProviderRetrievalSuccess = 0 for (const m of measurements) { // `retrievalResult` should be always set by lib/preprocess.js, so we should never encounter @@ -106,6 +107,8 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { // A successful HTTP response is a response with result breakdown set to OK and the protocol being used is set to HTTP. if (m.retrievalResult === 'OK' && m.protocol === 'http') { httpSuccesses++ } + if (m.alternativeProviderRetrievalResult === 'OK') { alternativeProviderRetrievalSuccess++ } + if (m.retrievalResult.match(/^IPNI_ERROR_5\d\d$/)) { indexerServerErrorCount++ } @@ -114,6 +117,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { const totalForRSR = totalCount - indexerServerErrorCount const successRate = totalForRSR ? resultBreakdown.OK / totalForRSR : 0 const successRateHttp = totalForRSR ? httpSuccesses / totalForRSR : 0 + const altProviderRSR = totalForRSR ? alternativeProviderRetrievalSuccess / totalForRSR : 0 telemetryPoint.intField('total_for_success_rates', totalForRSR) telemetryPoint.intField('unique_tasks', uniqueTasksCount) telemetryPoint.floatField('success_rate', successRate) @@ -122,6 +126,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { telemetryPoint.intField('inet_groups', inetGroups.size) telemetryPoint.intField('measurements', totalCount) telemetryPoint.intField('download_bandwidth', downloadBandwidth) + telemetryPoint.floatField('alternative_provider_success_rate', altProviderRSR) addHistogramToPoint(telemetryPoint, ttfbValues, 'ttfb_') addHistogramToPoint(telemetryPoint, durationValues, 'duration_') diff --git a/lib/typings.d.ts b/lib/typings.d.ts index 1963e72..8bb9792 100644 --- a/lib/typings.d.ts +++ b/lib/typings.d.ts @@ -100,6 +100,15 @@ export interface RawMeasurement { | 'NO_VALID_ADVERTISEMENT' | 'ERROR_FETCH' | `ERROR_${number}`; + + alternative_provider_check: { + status_code: number | undefined | null; + timeout: boolean; + car_too_large: boolean; + end_at: string; + protocol: string; + provider_id: string; + }; } export type CreatePgClient = () => Promise; diff --git a/migrations/026.do.add-daily-alternative-provider-retrieval-stats-table.sql b/migrations/026.do.add-daily-alternative-provider-retrieval-stats-table.sql new file mode 100644 index 0000000..d1d0512 --- /dev/null +++ b/migrations/026.do.add-daily-alternative-provider-retrieval-stats-table.sql @@ -0,0 +1,5 @@ +CREATE TABLE daily_alternative_provider_retrieval_stats ( + day DATE NOT NULL PRIMARY KEY, + total INT NOT NULL, + successful INT NOT NULL +); diff --git a/test/helpers/test-data.js b/test/helpers/test-data.js index 7c9e748..1449f7e 100644 --- a/test/helpers/test-data.js +++ b/test/helpers/test-data.js @@ -43,7 +43,16 @@ export const VALID_MEASUREMENT = { retrievalResult: 'OK', indexerResult: 'OK', taskingEvaluation: null, - consensusEvaluation: null + consensusEvaluation: null, + alternativeProviderRetrievalResult: 'OK', + alternativeProviderCheck: { + statusCode: null, + timeout: false, + carTooLarge: false, + endAt: null, + protocol: 'http', + providerId: 'ALTPROVIDERID' + } } // Fraud detection is mutating the measurements parsed from JSON diff --git a/test/preprocess.js b/test/preprocess.js index d15d94b..3485754 100644 --- a/test/preprocess.js +++ b/test/preprocess.js @@ -4,7 +4,8 @@ import { preprocess, Measurement, parseMeasurements, - assertValidMeasurement + assertValidMeasurement, + getAlternativeProviderRetrievalResult } from '../lib/preprocess.js' import { Point } from '../lib/telemetry.js' import assert from 'node:assert' @@ -416,3 +417,97 @@ describe('assertValidMeasurement', () => { assert.throws(() => assertValidMeasurement(measurement), /first_byte_at must be greater than or equal to start_at/) }) }) + +describe('getAlternativeProviderRetrievalResult', () => { + /** @type {Partial} */ + const SUCCESSFUL_RETRIEVAL = { + spark_version: '1.5.2', + participant_address: 'f410fgkhpcrbmdvic52o3nivftrjxr7nzw47updmuzra', + station_id: VALID_STATION_ID, + finished_at: '2023-11-01T09:42:03.246Z', + timeout: false, + start_at: '2023-11-01T09:40:03.393Z', + status_code: 200, + first_byte_at: '1970-01-01T00:00:00.000Z', + end_at: '1970-01-01T00:00:00.000Z', + byte_length: 1234, + inet_group: 'ue49TX_JdYjI', + cid: 'bafkreihstuf2qcu3hs64ersidh46cjtilxcoipmzgu3pifwzmkqdjpraqq', + miner_id: 'f1abc', + provider_address: '/ip4/108.89.91.150/tcp/46717/p2p/12D3KooWSsaFCtzDJUEhLQYDdwoFtdCMqqfk562UMvccFz12kYxU', + provider_id: 'PROVIDERID', + protocol: 'http', + indexer_result: 'OK' + } + + it('successful retrieval', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL + }) + assert.strictEqual(result, 'OK') + }) + + it('TIMEOUT - no alternative provider retrieval measurements', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL, + timeout: true + }) + assert.strictEqual(result, 'TIMEOUT') + }) + + it('TIMEOUT - successful alternative provider retrieval measurements', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL, + timeout: true, + alternative_provider_check: { + car_too_large: false, + timeout: false, + status_code: 200, + end_at: '2023-11-01T09:42:03.246Z', + protocol: 'http', + provider_id: 'ALTPROVIDERID' + } + }) + assert.strictEqual(result, 'TIMEOUT') + }) + + it('NO_VALID_ADVERTISEMENT - no alternative provider retrieval measurements', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL, + indexer_result: 'NO_VALID_ADVERTISEMENT' + }) + assert.strictEqual(result, 'IPNI_NO_VALID_ADVERTISEMENT') + }) + + it('NO_VALID_ADVERTISEMENT - successful alternative provider retrieval measurements', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL, + indexer_result: 'NO_VALID_ADVERTISEMENT', + alternative_provider_check: { + car_too_large: false, + timeout: false, + status_code: 200, + end_at: '2023-11-01T09:42:03.246Z', + protocol: 'http', + provider_id: 'ALTPROVIDERID' + } + }) + assert.strictEqual(result, 'OK') + }) + + it('NO_VALID_ADVERTISEMENT - TIMEOUT alternative provider retrieval measurements', () => { + const result = getAlternativeProviderRetrievalResult({ + ...SUCCESSFUL_RETRIEVAL, + indexer_result: 'NO_VALID_ADVERTISEMENT', + alternative_provider_check: { + car_too_large: false, + timeout: true, + status_code: 500, + end_at: '2023-11-01T09:42:03.246Z', + protocol: 'http', + provider_id: 'ALTPROVIDERID' + } + }) + assert.strictEqual(result, 'TIMEOUT') + }) +}) diff --git a/test/provider-retrieval-result-stats.test.js b/test/provider-retrieval-result-stats.test.js index 302b2bb..fb6958b 100644 --- a/test/provider-retrieval-result-stats.test.js +++ b/test/provider-retrieval-result-stats.test.js @@ -63,8 +63,8 @@ describe('Provider Retrieval Result Stats', () => { } ]) assert.deepStrictEqual(stats, new Map([ - ['0', { total: 2, successful: 2, successfulHttp: 1, successfulHttpHead: 1 }], - ['1', { total: 2, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }] + ['0', { total: 2, successful: 2, successfulHttp: 1, successfulHttpHead: 1, successfulAltProvider: 0 }], + ['1', { total: 2, successful: 0, successfulHttp: 0, successfulHttpHead: 0, successfulAltProvider: 0 }] ])) }) }) @@ -178,8 +178,8 @@ describe('Provider Retrieval Result Stats', () => { contract_address: ieContractAddress, measurement_batches: round.measurementBatches, provider_retrieval_result_stats: { - 0: { successful: 2, total: 2, successfulHttp: 1, successfulHttpHead: 1 }, - 1: { successful: 0, total: 2, successfulHttp: 0, successfulHttpHead: 0 } + 0: { successful: 2, total: 2, successfulHttp: 1, successfulHttpHead: 1, successfulAltProvider: 0 }, + 1: { successful: 0, total: 2, successfulHttp: 0, successfulHttpHead: 0, successfulAltProvider: 0 } }, round_details: 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q', round_index: String(round.index), diff --git a/test/public-stats.test.js b/test/public-stats.test.js index 2b61d69..aaf2ea5 100644 --- a/test/public-stats.test.js +++ b/test/public-stats.test.js @@ -31,6 +31,7 @@ describe('public-stats', () => { await pgClient.query('DELETE FROM retrieval_timings') await pgClient.query('DELETE FROM daily_client_retrieval_stats') await pgClient.query('DELETE FROM daily_allocator_retrieval_stats') + await pgClient.query('DELETE FROM daily_alternative_provider_retrieval_stats') // Run all tests inside a transaction to ensure `now()` always returns the same value // See https://dba.stackexchange.com/a/63549/125312 @@ -724,7 +725,7 @@ describe('public-stats', () => { findDealClients ) const { rows } = await pgClient.query( - `SELECT + `SELECT day::TEXT, client_id, total, @@ -770,7 +771,7 @@ describe('public-stats', () => { findDealClients ) const { rows } = await pgClient.query( - `SELECT + `SELECT day::TEXT, client_id, total, @@ -965,7 +966,7 @@ describe('public-stats', () => { findDealAllocators ) const { rows } = await pgClient.query( - `SELECT + `SELECT day::TEXT, allocator_id, total, @@ -1011,7 +1012,7 @@ describe('public-stats', () => { findDealAllocators ) const { rows } = await pgClient.query( - `SELECT + `SELECT day::TEXT, allocator_id, total, @@ -1168,6 +1169,43 @@ describe('public-stats', () => { { day: today, allocator_id: 'f0allocator', total: 4, successful: 3 } ]) }) + + describe('daily_alternative_provider_retrieval_stats ', () => { + it('creates or updates the row for today', async () => { + /** @type {Measurement[]} */ + const allMeasurements = [ + { ...VALID_MEASUREMENT, alternativeProviderRetrievalResult: 'OK' }, + { ...VALID_MEASUREMENT, alternativeProviderRetrievalResult: 'OK' }, + + { ...VALID_MEASUREMENT, alternativeProviderRetrievalResult: 'TIMEOUT' }, + { ...VALID_MEASUREMENT, alternativeProviderRetrievalResult: 'CAR_TOO_LARGE' }, + { ...VALID_MEASUREMENT, alternativeProviderRetrievalResult: 'IPNI_ERROR_FETCH' } + ] + + const committees = buildEvaluatedCommitteesFromMeasurements(allMeasurements) + const { rows: created } = await pgClient.query( + 'SELECT * FROM daily_alternative_provider_retrieval_stats' + ) + assert.deepStrictEqual(created, []) + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + const { rows } = await pgClient.query( + `SELECT + day::TEXT, + total, + successful + FROM daily_alternative_provider_retrieval_stats`) + + assert.deepStrictEqual(rows, [ + { day: today, total: 5, successful: 2 } + ]) + }) + }) }) const getCurrentDate = async () => { diff --git a/test/retrieval-stats.test.js b/test/retrieval-stats.test.js index dba0350..564b7f5 100644 --- a/test/retrieval-stats.test.js +++ b/test/retrieval-stats.test.js @@ -324,6 +324,26 @@ describe('retrieval statistics', () => { // Only one of the successful measurements used http assertPointFieldValue(point, 'success_rate_http', '0.25') }) + + it('records alternative provider retrieval success rate', async () => { + /** @type {Measurement[]} */ + const measurements = [ + { + ...VALID_MEASUREMENT, + alternativeProviderRetrievalResult: 'OK' + }, + { + ...VALID_MEASUREMENT, + alternativeProviderRetrievalResult: 'HTTP_500' + } + ] + + const point = new Point('stats') + buildRetrievalStats(measurements, point) + debug('stats', point.fields) + + assertPointFieldValue(point, 'alternative_provider_success_rate', '0.5') + }) }) describe('getValueAtPercentile', () => {