From 7eecd81177424fa60c76c15d38c8222c8fdf549a Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 14:39:02 +0200
Subject: [PATCH 01/15] Add support for network wide check

---
 api/index.js                                       | 14 +++++++++++---
 api/test/test.js                                   |  6 +++++-
 .../067.do.measurement-network-wide-retrieval.sql  |  2 ++
 3 files changed, 18 insertions(+), 4 deletions(-)
 create mode 100644 migrations/067.do.measurement-network-wide-retrieval.sql

diff --git a/api/index.js b/api/index.js
index 8bc2e057..0f3e1628 100644
--- a/api/index.js
+++ b/api/index.js
@@ -96,6 +96,8 @@ const createMeasurement = async (req, res, client) => {
   validate(measurement, 'minerId', { type: 'string', required: false })
   validate(measurement, 'providerId', { type: 'string', required: false })
   validate(measurement, 'stationId', { type: 'string', required: true })
+  validate(measurement, 'networkWideRetrievalStatusCode', { type: 'number', required: false })
+  validate(measurement, 'networkWideRetrievalTimeout', { type: 'boolean', required: false })
   assert(measurement.stationId.match(/^[0-9a-fA-F]{88}$/), 400, 'Invalid Station ID')
 
   const inetGroup = await mapRequestToInetGroup(client, req)
@@ -124,10 +126,12 @@ const createMeasurement = async (req, res, client) => {
         indexer_result,
         miner_id,
         provider_id,
+        network_wide_retrieval_status_code,
+        network_wide_retrieval_timeout,
         completed_at_round
       )
       SELECT
-        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21,
+        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23,
         id as completed_at_round
       FROM spark_rounds
       ORDER BY id DESC
@@ -154,7 +158,9 @@ const createMeasurement = async (req, res, client) => {
     measurement.carChecksum,
     measurement.indexerResult,
     measurement.minerId,
-    measurement.providerId
+    measurement.providerId,
+    measurement.networkWideRetrievalStatusCode,
+    measurement.networkWideRetrievalTimeout
   ])
   json(res, { id: rows[0].id })
 }
@@ -190,7 +196,9 @@ const getMeasurement = async (req, res, client, measurementId) => {
     endAt: resultRow.end_at,
     byteLength: resultRow.byte_length,
     carTooLarge: resultRow.car_too_large,
-    attestation: resultRow.attestation
+    attestation: resultRow.attestation,
+    networkWideRetrievalStatusCode: resultRow.network_wide_retrieval_status_code,
+    networkWideRetrievalTimeout: resultRow.network_wide_retrieval_timeout
   })
 }
 
diff --git a/api/test/test.js b/api/test/test.js
index d730e94c..0b729086 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -38,7 +38,9 @@ const VALID_MEASUREMENT = {
   carChecksum: 'somehash',
   minerId: 'f02abc',
   providerId: 'provider-pubkey',
-  indexerResult: 'OK'
+  indexerResult: 'OK',
+  networkWideRetrievalStatusCode: null,
+  networkWideRetrievalTimeout: false
 }
 
 const assertResponseStatus = async (res, status) => {
@@ -194,6 +196,8 @@ describe('Routes', () => {
       assert.strictEqual(measurementRow.miner_id, measurement.minerId)
       assert.strictEqual(measurementRow.provider_id, measurement.providerId)
       assert.strictEqual(measurementRow.station_id, measurement.stationId)
+      assert.strictEqual(measurementRow.network_wide_retrieval_status_code, measurement.networkWideRetrievalStatusCode)
+      assert.strictEqual(measurementRow.network_wide_retrieval_timeout, measurement.networkWideRetrievalTimeout)
     })
 
     it('allows older format with walletAddress', async () => {
diff --git a/migrations/067.do.measurement-network-wide-retrieval.sql b/migrations/067.do.measurement-network-wide-retrieval.sql
new file mode 100644
index 00000000..20f75612
--- /dev/null
+++ b/migrations/067.do.measurement-network-wide-retrieval.sql
@@ -0,0 +1,2 @@
+ALTER TABLE measurements ADD COLUMN network_wide_retrieval_status_code INTEGER;
+ALTER TABLE measurements ADD COLUMN network_wide_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE;

From dd80cd0f469433f8ae5552810741944cf3a7d787 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 15:13:55 +0200
Subject: [PATCH 02/15] Simplify name for network wide measurements

---
 api/index.js                                             | 8 ++++----
 api/test/test.js                                         | 8 ++++----
 migrations/067.do.measurement-network-wide-retrieval.sql | 4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/api/index.js b/api/index.js
index 0f3e1628..c928043a 100644
--- a/api/index.js
+++ b/api/index.js
@@ -126,8 +126,8 @@ const createMeasurement = async (req, res, client) => {
         indexer_result,
         miner_id,
         provider_id,
-        network_wide_retrieval_status_code,
-        network_wide_retrieval_timeout,
+        network_retrieval_status_code,
+        network_retrieval_timeout,
         completed_at_round
       )
       SELECT
@@ -197,8 +197,8 @@ const getMeasurement = async (req, res, client, measurementId) => {
     byteLength: resultRow.byte_length,
     carTooLarge: resultRow.car_too_large,
     attestation: resultRow.attestation,
-    networkWideRetrievalStatusCode: resultRow.network_wide_retrieval_status_code,
-    networkWideRetrievalTimeout: resultRow.network_wide_retrieval_timeout
+    networkRetrievalStatusCode: resultRow.network_retrieval_status_code,
+    networkRetrievalTimeout: resultRow.network_retrieval_timeout
   })
 }
 
diff --git a/api/test/test.js b/api/test/test.js
index 0b729086..b997edc7 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -39,8 +39,8 @@ const VALID_MEASUREMENT = {
   minerId: 'f02abc',
   providerId: 'provider-pubkey',
   indexerResult: 'OK',
-  networkWideRetrievalStatusCode: null,
-  networkWideRetrievalTimeout: false
+  networkRetrievalStatusCode: null,
+  networkRetrievalTimeout: false
 }
 
 const assertResponseStatus = async (res, status) => {
@@ -196,8 +196,8 @@ describe('Routes', () => {
       assert.strictEqual(measurementRow.miner_id, measurement.minerId)
       assert.strictEqual(measurementRow.provider_id, measurement.providerId)
       assert.strictEqual(measurementRow.station_id, measurement.stationId)
-      assert.strictEqual(measurementRow.network_wide_retrieval_status_code, measurement.networkWideRetrievalStatusCode)
-      assert.strictEqual(measurementRow.network_wide_retrieval_timeout, measurement.networkWideRetrievalTimeout)
+      assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrievalStatusCode)
+      assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrievalTimeout)
     })
 
     it('allows older format with walletAddress', async () => {
diff --git a/migrations/067.do.measurement-network-wide-retrieval.sql b/migrations/067.do.measurement-network-wide-retrieval.sql
index 20f75612..dfa45392 100644
--- a/migrations/067.do.measurement-network-wide-retrieval.sql
+++ b/migrations/067.do.measurement-network-wide-retrieval.sql
@@ -1,2 +1,2 @@
-ALTER TABLE measurements ADD COLUMN network_wide_retrieval_status_code INTEGER;
-ALTER TABLE measurements ADD COLUMN network_wide_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE measurements ADD COLUMN network_retrieval_status_code INTEGER;
+ALTER TABLE measurements ADD COLUMN network_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE;

From 11af4fccac975b433c2428d7306b7af9f165faaa Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 15:39:57 +0200
Subject: [PATCH 03/15] Extend network measurement with more columns

---
 api/index.js                                  | 24 ++++++++++++++-----
 api/test/test.js                              |  5 +++-
 ....do.measurement-network-wide-retrieval.sql |  9 +++++--
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/api/index.js b/api/index.js
index c928043a..62c71807 100644
--- a/api/index.js
+++ b/api/index.js
@@ -96,8 +96,11 @@ const createMeasurement = async (req, res, client) => {
   validate(measurement, 'minerId', { type: 'string', required: false })
   validate(measurement, 'providerId', { type: 'string', required: false })
   validate(measurement, 'stationId', { type: 'string', required: true })
-  validate(measurement, 'networkWideRetrievalStatusCode', { type: 'number', required: false })
-  validate(measurement, 'networkWideRetrievalTimeout', { type: 'boolean', required: false })
+  validate(measurement, 'networkRetrievalStatusCode', { type: 'number', required: false })
+  validate(measurement, 'networkRetrievalTimeout', { type: 'boolean', required: false })
+  validate(measurement, 'networkRetrievalCarTooLarge', { type: 'boolean', required: false })
+  validate(measurement, 'networkRetrievalEndAt', { type: 'date', required: false })
+  validate(measurement, 'networkRetrievalProtocol', { type: 'string', required: false })
   assert(measurement.stationId.match(/^[0-9a-fA-F]{88}$/), 400, 'Invalid Station ID')
 
   const inetGroup = await mapRequestToInetGroup(client, req)
@@ -128,10 +131,13 @@ const createMeasurement = async (req, res, client) => {
         provider_id,
         network_retrieval_status_code,
         network_retrieval_timeout,
+        network_retrieval_car_too_large,
+        network_retrieval_end_at,
+        network_retrieval_protocol,
         completed_at_round
       )
       SELECT
-        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23,
+        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26
         id as completed_at_round
       FROM spark_rounds
       ORDER BY id DESC
@@ -159,8 +165,11 @@ const createMeasurement = async (req, res, client) => {
     measurement.indexerResult,
     measurement.minerId,
     measurement.providerId,
-    measurement.networkWideRetrievalStatusCode,
-    measurement.networkWideRetrievalTimeout
+    measurement.networkRetrievalStatusCode,
+    measurement.networkRetrievalTimeout,
+    measurement.networkRetrievalCarTooLarge ?? false,
+    measurement.networkRetrievalEndAt,
+    measurement.networkRetrievalProtocol
   ])
   json(res, { id: rows[0].id })
 }
@@ -198,7 +207,10 @@ const getMeasurement = async (req, res, client, measurementId) => {
     carTooLarge: resultRow.car_too_large,
     attestation: resultRow.attestation,
     networkRetrievalStatusCode: resultRow.network_retrieval_status_code,
-    networkRetrievalTimeout: resultRow.network_retrieval_timeout
+    networkRetrievalTimeout: resultRow.network_retrieval_timeout,
+    networkRetrievalCarTooLarge: resultRow.network_retrieval_car_too_large,
+    networkRetrievalEndAt: resultRow.network_retrieval_end_at,
+    networkRetrievalProtocol: resultRow.network_retrieval_protocol
   })
 }
 
diff --git a/api/test/test.js b/api/test/test.js
index b997edc7..e87057c7 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -40,7 +40,10 @@ const VALID_MEASUREMENT = {
   providerId: 'provider-pubkey',
   indexerResult: 'OK',
   networkRetrievalStatusCode: null,
-  networkRetrievalTimeout: false
+  networkRetrievalTimeout: false,
+  networkRetrievalCarTooLarge: false,
+  networkRetrievalEndAt: null,
+  networkRetrievalProtocol: null
 }
 
 const assertResponseStatus = async (res, status) => {
diff --git a/migrations/067.do.measurement-network-wide-retrieval.sql b/migrations/067.do.measurement-network-wide-retrieval.sql
index dfa45392..e194ef6e 100644
--- a/migrations/067.do.measurement-network-wide-retrieval.sql
+++ b/migrations/067.do.measurement-network-wide-retrieval.sql
@@ -1,2 +1,7 @@
-ALTER TABLE measurements ADD COLUMN network_retrieval_status_code INTEGER;
-ALTER TABLE measurements ADD COLUMN network_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE measurements 
+    ADD COLUMN network_retrieval_status_code INTEGER,
+    ADD COLUMN network_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE,
+    ADD COLUMN network_retrieval_car_too_large BOOLEAN NOT NULL DEFAULT FALSE,
+    ADD COLUMN network_retrieval_end_at TIMESTAMPTZ,
+    ADD COLUMN network_retrieval_protocol protocol;
+

From eae83a112898e5d9f3bfaa313401581b649bd253 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 15:40:19 +0200
Subject: [PATCH 04/15] Rename network retrieval table

---
 ...ide-retrieval.sql => 067.do.measurement-network-retrieval.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename migrations/{067.do.measurement-network-wide-retrieval.sql => 067.do.measurement-network-retrieval.sql} (100%)

diff --git a/migrations/067.do.measurement-network-wide-retrieval.sql b/migrations/067.do.measurement-network-retrieval.sql
similarity index 100%
rename from migrations/067.do.measurement-network-wide-retrieval.sql
rename to migrations/067.do.measurement-network-retrieval.sql

From 87d330e8f1c2915c76d3f4173be0358f95106328 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 17:30:50 +0200
Subject: [PATCH 05/15] Edit measurement test data

---
 api/test/test.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/test/test.js b/api/test/test.js
index e87057c7..c188e39c 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -39,11 +39,11 @@ const VALID_MEASUREMENT = {
   minerId: 'f02abc',
   providerId: 'provider-pubkey',
   indexerResult: 'OK',
-  networkRetrievalStatusCode: null,
+  networkRetrievalStatusCode: 200,
   networkRetrievalTimeout: false,
   networkRetrievalCarTooLarge: false,
-  networkRetrievalEndAt: null,
-  networkRetrievalProtocol: null
+  networkRetrievalEndAt: new Date(),
+  networkRetrievalProtocol: 'graphsync'
 }
 
 const assertResponseStatus = async (res, status) => {

From 2b52cd4fed71ebbf4239282943529929e00915ba Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Wed, 9 Apr 2025 18:30:47 +0200
Subject: [PATCH 06/15] Fix failing test

---
 api/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/index.js b/api/index.js
index 62c71807..e245454c 100644
--- a/api/index.js
+++ b/api/index.js
@@ -137,7 +137,7 @@ const createMeasurement = async (req, res, client) => {
         completed_at_round
       )
       SELECT
-        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26
+        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26,
         id as completed_at_round
       FROM spark_rounds
       ORDER BY id DESC

From 91f1a19e4ff8dcb6166f99db9eb9b87eef60c175 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Thu, 10 Apr 2025 11:16:07 +0200
Subject: [PATCH 07/15] Convert network measurement to object inside the
 measurement

---
 api/index.js     | 36 +++++++++++++++++++++---------------
 api/test/test.js | 19 ++++++++++++-------
 2 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/api/index.js b/api/index.js
index e245454c..4fc9c5ba 100644
--- a/api/index.js
+++ b/api/index.js
@@ -96,13 +96,17 @@ const createMeasurement = async (req, res, client) => {
   validate(measurement, 'minerId', { type: 'string', required: false })
   validate(measurement, 'providerId', { type: 'string', required: false })
   validate(measurement, 'stationId', { type: 'string', required: true })
-  validate(measurement, 'networkRetrievalStatusCode', { type: 'number', required: false })
-  validate(measurement, 'networkRetrievalTimeout', { type: 'boolean', required: false })
-  validate(measurement, 'networkRetrievalCarTooLarge', { type: 'boolean', required: false })
-  validate(measurement, 'networkRetrievalEndAt', { type: 'date', required: false })
-  validate(measurement, 'networkRetrievalProtocol', { type: 'string', required: false })
   assert(measurement.stationId.match(/^[0-9a-fA-F]{88}$/), 400, 'Invalid Station ID')
 
+  if (measurement.networkRetrieval) {
+    validate(measurement, 'networkRetrieval', { type: 'object', required: false })
+    validate(measurement.networkRetrieval, 'statusCode', { type: 'number', required: false })
+    validate(measurement.networkRetrieval, 'timeout', { type: 'boolean', required: false })
+    validate(measurement.networkRetrieval, 'carTooLarge', { type: 'boolean', required: false })
+    validate(measurement.networkRetrieval, 'endAt', { type: 'date', required: false })
+    validate(measurement.networkRetrieval, 'protocol', { type: 'string', required: false })
+  }
+
   const inetGroup = await mapRequestToInetGroup(client, req)
   logNetworkInfo(req.headers, measurement.stationId, recordNetworkInfoTelemetry)
 
@@ -165,11 +169,11 @@ const createMeasurement = async (req, res, client) => {
     measurement.indexerResult,
     measurement.minerId,
     measurement.providerId,
-    measurement.networkRetrievalStatusCode,
-    measurement.networkRetrievalTimeout,
-    measurement.networkRetrievalCarTooLarge ?? false,
-    measurement.networkRetrievalEndAt,
-    measurement.networkRetrievalProtocol
+    measurement.networkRetrieval?.statusCode,
+    measurement.networkRetrieval?.timeout,
+    measurement.networkRetrieval?.carTooLarge ?? false,
+    measurement.networkRetrieval?.endAt,
+    measurement.networkRetrieval?.protocol
   ])
   json(res, { id: rows[0].id })
 }
@@ -206,11 +210,13 @@ const getMeasurement = async (req, res, client, measurementId) => {
     byteLength: resultRow.byte_length,
     carTooLarge: resultRow.car_too_large,
     attestation: resultRow.attestation,
-    networkRetrievalStatusCode: resultRow.network_retrieval_status_code,
-    networkRetrievalTimeout: resultRow.network_retrieval_timeout,
-    networkRetrievalCarTooLarge: resultRow.network_retrieval_car_too_large,
-    networkRetrievalEndAt: resultRow.network_retrieval_end_at,
-    networkRetrievalProtocol: resultRow.network_retrieval_protocol
+    networkRetrieval: {
+      statusCode: resultRow.network_retrieval_status_code,
+      timeout: resultRow.network_retrieval_timeout,
+      carTooLarge: resultRow.network_retrieval_car_too_large,
+      endAt: resultRow.network_retrieval_end_at,
+      protocol: resultRow.network_retrieval_protocol
+    }
   })
 }
 
diff --git a/api/test/test.js b/api/test/test.js
index c188e39c..7d21b576 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -39,11 +39,13 @@ const VALID_MEASUREMENT = {
   minerId: 'f02abc',
   providerId: 'provider-pubkey',
   indexerResult: 'OK',
-  networkRetrievalStatusCode: 200,
-  networkRetrievalTimeout: false,
-  networkRetrievalCarTooLarge: false,
-  networkRetrievalEndAt: new Date(),
-  networkRetrievalProtocol: 'graphsync'
+  networkRetrieval: {
+    statusCode: 200,
+    timeout: false,
+    carTooLarge: false,
+    endAt: new Date(),
+    protocol: 'graphsync'
+  }
 }
 
 const assertResponseStatus = async (res, status) => {
@@ -199,8 +201,11 @@ describe('Routes', () => {
       assert.strictEqual(measurementRow.miner_id, measurement.minerId)
       assert.strictEqual(measurementRow.provider_id, measurement.providerId)
       assert.strictEqual(measurementRow.station_id, measurement.stationId)
-      assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrievalStatusCode)
-      assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrievalTimeout)
+      assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrieval.statusCode)
+      assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrieval.timeout)
+      assert.strictEqual(measurementRow.network_retrieval_car_too_large, measurement.networkRetrieval.carTooLarge)
+      assert.strictEqual(measurementRow.network_retrieval_end_at, measurement.networkRetrieval.endAt)
+      assert.strictEqual(measurementRow.network_retrieval_protocol, measurement.networkRetrieval.protocol)
     })
 
     it('allows older format with walletAddress', async () => {

From 4357648bba9fd50c70d04c14e4f3b52e71ef6097 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Thu, 10 Apr 2025 13:03:00 +0200
Subject: [PATCH 08/15] Fix failing test

---
 api/test/test.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/api/test/test.js b/api/test/test.js
index 7d21b576..37919a9c 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -204,7 +204,10 @@ describe('Routes', () => {
       assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrieval.statusCode)
       assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrieval.timeout)
       assert.strictEqual(measurementRow.network_retrieval_car_too_large, measurement.networkRetrieval.carTooLarge)
-      assert.strictEqual(measurementRow.network_retrieval_end_at, measurement.networkRetrieval.endAt)
+      assert.strictEqual(
+        measurementRow.network_retrieval_end_at.toJSON(),
+        measurement.networkRetrieval.endAt.toJSON()
+      )
       assert.strictEqual(measurementRow.network_retrieval_protocol, measurement.networkRetrieval.protocol)
     })
 

From b2bcb2bec10fd8a9077a5b15ac5c003a7c3cb4de Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Tue, 15 Apr 2025 18:36:55 +0200
Subject: [PATCH 09/15] Rename network check to alternative provider check

---
 api/index.js                                  | 46 +++++++++----------
 ...measurement-alternative-provider-check.sql |  7 +++
 .../067.do.measurement-network-retrieval.sql  |  7 ---
 3 files changed, 30 insertions(+), 30 deletions(-)
 create mode 100644 migrations/067.do.measurement-alternative-provider-check.sql
 delete mode 100644 migrations/067.do.measurement-network-retrieval.sql

diff --git a/api/index.js b/api/index.js
index 4fc9c5ba..da926f9d 100644
--- a/api/index.js
+++ b/api/index.js
@@ -98,13 +98,13 @@ const createMeasurement = async (req, res, client) => {
   validate(measurement, 'stationId', { type: 'string', required: true })
   assert(measurement.stationId.match(/^[0-9a-fA-F]{88}$/), 400, 'Invalid Station ID')
 
-  if (measurement.networkRetrieval) {
-    validate(measurement, 'networkRetrieval', { type: 'object', required: false })
-    validate(measurement.networkRetrieval, 'statusCode', { type: 'number', required: false })
-    validate(measurement.networkRetrieval, 'timeout', { type: 'boolean', required: false })
-    validate(measurement.networkRetrieval, 'carTooLarge', { type: 'boolean', required: false })
-    validate(measurement.networkRetrieval, 'endAt', { type: 'date', required: false })
-    validate(measurement.networkRetrieval, 'protocol', { type: 'string', required: false })
+  if (measurement.alternativeProviderCheck) {
+    validate(measurement, 'alternativeProviderCheck', { type: 'object', required: false })
+    validate(measurement.alternativeProviderCheck, 'statusCode', { type: 'number', required: false })
+    validate(measurement.alternativeProviderCheck, 'timeout', { type: 'boolean', required: false })
+    validate(measurement.alternativeProviderCheck, 'carTooLarge', { type: 'boolean', required: false })
+    validate(measurement.alternativeProviderCheck, 'endAt', { type: 'date', required: false })
+    validate(measurement.alternativeProviderCheck, 'protocol', { type: 'string', required: false })
   }
 
   const inetGroup = await mapRequestToInetGroup(client, req)
@@ -133,11 +133,11 @@ const createMeasurement = async (req, res, client) => {
         indexer_result,
         miner_id,
         provider_id,
-        network_retrieval_status_code,
-        network_retrieval_timeout,
-        network_retrieval_car_too_large,
-        network_retrieval_end_at,
-        network_retrieval_protocol,
+        alternative_provider_check_status_code,
+        alternative_provider_check_timeout,
+        alternative_provider_check_car_too_large,
+        alternative_provider_check_end_at,
+        alternative_provider_check_protocol,
         completed_at_round
       )
       SELECT
@@ -169,11 +169,11 @@ const createMeasurement = async (req, res, client) => {
     measurement.indexerResult,
     measurement.minerId,
     measurement.providerId,
-    measurement.networkRetrieval?.statusCode,
-    measurement.networkRetrieval?.timeout,
-    measurement.networkRetrieval?.carTooLarge ?? false,
-    measurement.networkRetrieval?.endAt,
-    measurement.networkRetrieval?.protocol
+    measurement.alternativeProviderCheck?.statusCode,
+    measurement.alternativeProviderCheck?.timeout,
+    measurement.alternativeProviderCheck?.carTooLarge ?? false,
+    measurement.alternativeProviderCheck?.endAt,
+    measurement.alternativeProviderCheck?.protocol
   ])
   json(res, { id: rows[0].id })
 }
@@ -210,12 +210,12 @@ const getMeasurement = async (req, res, client, measurementId) => {
     byteLength: resultRow.byte_length,
     carTooLarge: resultRow.car_too_large,
     attestation: resultRow.attestation,
-    networkRetrieval: {
-      statusCode: resultRow.network_retrieval_status_code,
-      timeout: resultRow.network_retrieval_timeout,
-      carTooLarge: resultRow.network_retrieval_car_too_large,
-      endAt: resultRow.network_retrieval_end_at,
-      protocol: resultRow.network_retrieval_protocol
+    alternativeProviderCheck: {
+      statusCode: resultRow.alternative_provider_check_status_code,
+      timeout: resultRow.alternative_provider_check_timeout,
+      carTooLarge: resultRow.alternative_provider_check_car_too_large,
+      endAt: resultRow.alternative_provider_check_end_at,
+      protocol: resultRow.alternative_provider_check_protocol
     }
   })
 }
diff --git a/migrations/067.do.measurement-alternative-provider-check.sql b/migrations/067.do.measurement-alternative-provider-check.sql
new file mode 100644
index 00000000..02ec6875
--- /dev/null
+++ b/migrations/067.do.measurement-alternative-provider-check.sql
@@ -0,0 +1,7 @@
+ALTER TABLE measurements 
+    ADD COLUMN alternative_provider_check_status_code INTEGER,
+    ADD COLUMN alternative_provider_check_timeout BOOLEAN NOT NULL DEFAULT FALSE,
+    ADD COLUMN alternative_provider_check_car_too_large BOOLEAN NOT NULL DEFAULT FALSE,
+    ADD COLUMN alternative_provider_check_end_at TIMESTAMPTZ,
+    ADD COLUMN alternative_provider_check_protocol protocol;
+
diff --git a/migrations/067.do.measurement-network-retrieval.sql b/migrations/067.do.measurement-network-retrieval.sql
deleted file mode 100644
index e194ef6e..00000000
--- a/migrations/067.do.measurement-network-retrieval.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-ALTER TABLE measurements 
-    ADD COLUMN network_retrieval_status_code INTEGER,
-    ADD COLUMN network_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE,
-    ADD COLUMN network_retrieval_car_too_large BOOLEAN NOT NULL DEFAULT FALSE,
-    ADD COLUMN network_retrieval_end_at TIMESTAMPTZ,
-    ADD COLUMN network_retrieval_protocol protocol;
-

From 0ede95e280f0fab0c3cdfd421afb35a386cbede2 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Tue, 15 Apr 2025 18:42:34 +0200
Subject: [PATCH 10/15] Fix failing tests

---
 api/test/test.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/api/test/test.js b/api/test/test.js
index 37919a9c..bd493944 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -39,7 +39,7 @@ const VALID_MEASUREMENT = {
   minerId: 'f02abc',
   providerId: 'provider-pubkey',
   indexerResult: 'OK',
-  networkRetrieval: {
+  alternativeProviderCheck: {
     statusCode: 200,
     timeout: false,
     carTooLarge: false,
@@ -201,14 +201,14 @@ describe('Routes', () => {
       assert.strictEqual(measurementRow.miner_id, measurement.minerId)
       assert.strictEqual(measurementRow.provider_id, measurement.providerId)
       assert.strictEqual(measurementRow.station_id, measurement.stationId)
-      assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrieval.statusCode)
-      assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrieval.timeout)
-      assert.strictEqual(measurementRow.network_retrieval_car_too_large, measurement.networkRetrieval.carTooLarge)
+      assert.strictEqual(measurementRow.alternative_provider_check_status_code, measurement.alternativeProviderCheck.statusCode)
+      assert.strictEqual(measurementRow.alternative_provider_check_timeout, measurement.alternativeProviderCheck.timeout)
+      assert.strictEqual(measurementRow.alternative_provider_check_car_too_large, measurement.alternativeProviderCheck.carTooLarge)
       assert.strictEqual(
-        measurementRow.network_retrieval_end_at.toJSON(),
-        measurement.networkRetrieval.endAt.toJSON()
+        measurementRow.alternative_provider_check_end_at.toJSON(),
+        measurement.alternativeProviderCheck.endAt.toJSON()
       )
-      assert.strictEqual(measurementRow.network_retrieval_protocol, measurement.networkRetrieval.protocol)
+      assert.strictEqual(measurementRow.alternative_provider_check_protocol, measurement.alternativeProviderCheck.protocol)
     })
 
     it('allows older format with walletAddress', async () => {

From bba43e4387a7d91150b80ff309d81713da3a9ee4 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Mon, 28 Apr 2025 12:11:55 +0200
Subject: [PATCH 11/15] Add alt provider id

---
 api/index.js                                           | 10 +++++++---
 api/test/test.js                                       |  4 +++-
 .../067.do.measurement-alternative-provider-check.sql  |  6 +++---
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/api/index.js b/api/index.js
index da926f9d..7068fc0a 100644
--- a/api/index.js
+++ b/api/index.js
@@ -105,6 +105,7 @@ const createMeasurement = async (req, res, client) => {
     validate(measurement.alternativeProviderCheck, 'carTooLarge', { type: 'boolean', required: false })
     validate(measurement.alternativeProviderCheck, 'endAt', { type: 'date', required: false })
     validate(measurement.alternativeProviderCheck, 'protocol', { type: 'string', required: false })
+    validate(measurement.alternativeProviderCheck, 'providerId', { type: 'string', required: false })
   }
 
   const inetGroup = await mapRequestToInetGroup(client, req)
@@ -138,10 +139,11 @@ const createMeasurement = async (req, res, client) => {
         alternative_provider_check_car_too_large,
         alternative_provider_check_end_at,
         alternative_provider_check_protocol,
+        alternative_provider_check_provider_id,
         completed_at_round
       )
       SELECT
-        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26,
+        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27
         id as completed_at_round
       FROM spark_rounds
       ORDER BY id DESC
@@ -173,7 +175,8 @@ const createMeasurement = async (req, res, client) => {
     measurement.alternativeProviderCheck?.timeout,
     measurement.alternativeProviderCheck?.carTooLarge ?? false,
     measurement.alternativeProviderCheck?.endAt,
-    measurement.alternativeProviderCheck?.protocol
+    measurement.alternativeProviderCheck?.protocol,
+    measurement.alternativeProviderCheck?.provider_id
   ])
   json(res, { id: rows[0].id })
 }
@@ -215,7 +218,8 @@ const getMeasurement = async (req, res, client, measurementId) => {
       timeout: resultRow.alternative_provider_check_timeout,
       carTooLarge: resultRow.alternative_provider_check_car_too_large,
       endAt: resultRow.alternative_provider_check_end_at,
-      protocol: resultRow.alternative_provider_check_protocol
+      protocol: resultRow.alternative_provider_check_protocol,
+      providerId: resultRow.alternative_provider_check_provider_id
     }
   })
 }
diff --git a/api/test/test.js b/api/test/test.js
index bd493944..00b81419 100644
--- a/api/test/test.js
+++ b/api/test/test.js
@@ -44,7 +44,8 @@ const VALID_MEASUREMENT = {
     timeout: false,
     carTooLarge: false,
     endAt: new Date(),
-    protocol: 'graphsync'
+    protocol: 'graphsync',
+    providerId: 'alt-provider-pubkey'
   }
 }
 
@@ -209,6 +210,7 @@ describe('Routes', () => {
         measurement.alternativeProviderCheck.endAt.toJSON()
       )
       assert.strictEqual(measurementRow.alternative_provider_check_protocol, measurement.alternativeProviderCheck.protocol)
+      assert.strictEqual(measurementRow.alternative_provider_check_provider_id, measurement.alternativeProviderCheck.providerId)
     })
 
     it('allows older format with walletAddress', async () => {
diff --git a/migrations/067.do.measurement-alternative-provider-check.sql b/migrations/067.do.measurement-alternative-provider-check.sql
index 02ec6875..6d13319c 100644
--- a/migrations/067.do.measurement-alternative-provider-check.sql
+++ b/migrations/067.do.measurement-alternative-provider-check.sql
@@ -1,7 +1,7 @@
-ALTER TABLE measurements 
+ALTER TABLE measurements
     ADD COLUMN alternative_provider_check_status_code INTEGER,
     ADD COLUMN alternative_provider_check_timeout BOOLEAN NOT NULL DEFAULT FALSE,
     ADD COLUMN alternative_provider_check_car_too_large BOOLEAN NOT NULL DEFAULT FALSE,
     ADD COLUMN alternative_provider_check_end_at TIMESTAMPTZ,
-    ADD COLUMN alternative_provider_check_protocol protocol;
-
+    ADD COLUMN alternative_provider_check_protocol protocol,
+    ADD COLUMN alternative_provider_check_provider_id TEXT;

From 3ea8dae9b5a2947e6505ab377114c24c102594a2 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Mon, 28 Apr 2025 12:22:16 +0200
Subject: [PATCH 12/15] Fix insert query

---
 api/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/index.js b/api/index.js
index 7068fc0a..a431e9e9 100644
--- a/api/index.js
+++ b/api/index.js
@@ -176,7 +176,7 @@ const createMeasurement = async (req, res, client) => {
     measurement.alternativeProviderCheck?.carTooLarge ?? false,
     measurement.alternativeProviderCheck?.endAt,
     measurement.alternativeProviderCheck?.protocol,
-    measurement.alternativeProviderCheck?.provider_id
+    measurement.alternativeProviderCheck?.providerId
   ])
   json(res, { id: rows[0].id })
 }

From 85f497e845961e3173f541114959f69d49c81f3c Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Mon, 28 Apr 2025 16:34:06 +0200
Subject: [PATCH 13/15] Publish new measurement fields

---
 api/bin/spark.js     |  2 +-
 publish/index.js     |  8 +++++++-
 publish/test/test.js | 13 +++++++++++--
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/api/bin/spark.js b/api/bin/spark.js
index cd9ea082..823553dd 100644
--- a/api/bin/spark.js
+++ b/api/bin/spark.js
@@ -27,7 +27,7 @@ assert(DEAL_INGESTER_TOKEN, 'DEAL_INGESTER_TOKEN is required')
 const client = new pg.Pool({
   connectionString: DATABASE_URL,
   // allow the pool to close all connections and become empty
-  min: 0,
+  // min: 0,
   // this values should correlate with service concurrency hard_limit configured in fly.toml
   // and must take into account the connection limit of our PG server, see
   // https://fly.io/docs/postgres/managing/configuration-tuning/
diff --git a/publish/index.js b/publish/index.js
index 72068d2a..9e62482e 100644
--- a/publish/index.js
+++ b/publish/index.js
@@ -37,7 +37,13 @@ export const publish = async ({
       provider_id,
       cid,
       provider_address,
-      protocol
+      protocol,
+      alternative_provider_check_status_code,
+      alternative_provider_check_timeout,
+      alternative_provider_check_car_too_large,
+      alternative_provider_check_end_at,
+      alternative_provider_check_protocol,
+      alternative_provider_check_provider_id
     FROM measurements
     LIMIT $1
   `, [
diff --git a/publish/test/test.js b/publish/test/test.js
index fff95127..a7cfd2a5 100644
--- a/publish/test/test.js
+++ b/publish/test/test.js
@@ -164,8 +164,17 @@ describe('integration', () => {
       carTooLarge: true,
       minerId: 'f02abc',
       providerId: 'provider-pubkey',
-      round: 42
-    }]
+      round: 42,
+      alternativeProviderCheck: {
+        statusCode: 200,
+        timeout: false,
+        carTooLarge: false,
+        endAt: new Date(),
+        protocol: 'graphsync',
+        providerId: 'alt-provider-pubkey'
+      }
+    }
+    ]
 
     for (const measurement of measurements) {
       await insertMeasurement(client, measurement)

From 2bba71cc03a4a82481265c3710faf822e9228c96 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Tue, 29 Apr 2025 11:23:35 +0200
Subject: [PATCH 14/15] Group alternative provider check fields into single
 object

---
 api/bin/spark.js |  2 +-
 publish/index.js | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/api/bin/spark.js b/api/bin/spark.js
index 823553dd..cd9ea082 100644
--- a/api/bin/spark.js
+++ b/api/bin/spark.js
@@ -27,7 +27,7 @@ assert(DEAL_INGESTER_TOKEN, 'DEAL_INGESTER_TOKEN is required')
 const client = new pg.Pool({
   connectionString: DATABASE_URL,
   // allow the pool to close all connections and become empty
-  // min: 0,
+  min: 0,
   // this values should correlate with service concurrency hard_limit configured in fly.toml
   // and must take into account the connection limit of our PG server, see
   // https://fly.io/docs/postgres/managing/configuration-tuning/
diff --git a/publish/index.js b/publish/index.js
index 9e62482e..c77b8bb6 100644
--- a/publish/index.js
+++ b/publish/index.js
@@ -38,12 +38,14 @@ export const publish = async ({
       cid,
       provider_address,
       protocol,
-      alternative_provider_check_status_code,
-      alternative_provider_check_timeout,
-      alternative_provider_check_car_too_large,
-      alternative_provider_check_end_at,
-      alternative_provider_check_protocol,
-      alternative_provider_check_provider_id
+      jsonb_build_object(
+        'status_code', alternative_provider_check_status_code,
+        'timeout', alternative_provider_check_timeout,
+        'car_too_large', alternative_provider_check_car_too_large,
+        'end_at', alternative_provider_check_end_at,
+        'protocol', alternative_provider_check_protocol,
+        'provider_id', alternative_provider_check_provider_id
+      ) AS alternative_provider_check
     FROM measurements
     LIMIT $1
   `, [

From 083bf60e84c13768146d2bb5ebd9d5e2d2d46af9 Mon Sep 17 00:00:00 2001
From: Srdjan S <stankovic.srdjo@gmail.com>
Date: Tue, 29 Apr 2025 12:01:35 +0200
Subject: [PATCH 15/15] Change alternativeProviderCheck validation message

---
 api/index.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/api/index.js b/api/index.js
index a431e9e9..04f7d074 100644
--- a/api/index.js
+++ b/api/index.js
@@ -100,12 +100,12 @@ const createMeasurement = async (req, res, client) => {
 
   if (measurement.alternativeProviderCheck) {
     validate(measurement, 'alternativeProviderCheck', { type: 'object', required: false })
-    validate(measurement.alternativeProviderCheck, 'statusCode', { type: 'number', required: false })
-    validate(measurement.alternativeProviderCheck, 'timeout', { type: 'boolean', required: false })
-    validate(measurement.alternativeProviderCheck, 'carTooLarge', { type: 'boolean', required: false })
-    validate(measurement.alternativeProviderCheck, 'endAt', { type: 'date', required: false })
-    validate(measurement.alternativeProviderCheck, 'protocol', { type: 'string', required: false })
-    validate(measurement.alternativeProviderCheck, 'providerId', { type: 'string', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.statusCode', { type: 'number', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.timeout', { type: 'boolean', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.carTooLarge', { type: 'boolean', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.endAt', { type: 'date', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.protocol', { type: 'string', required: false })
+    validate(measurement.alternativeProviderCheck, 'alternativeProviderCheck.providerId', { type: 'string', required: false })
   }
 
   const inetGroup = await mapRequestToInetGroup(client, req)