fix: authorize gateway in integration tests (#464)

Includes switch to pnpm (#465)

Author: alanshaw

.env.tpl

@@ -71,3 +71,7 @@ PRINCIPAL_MAPPING = '{"did:web:example.com":"did:key:z6MktkCXwNmpqejQxYd7JHPcw7d
 INTEGRATION_TESTS_STORAGE_PROVIDER_DID = ''
 INTEGRATION_TESTS_STORAGE_PROVIDER_ENDPOINT = ''
 INTEGRATION_TESTS_STORAGE_PROVIDER_PROOF = ''
+
+# Optional - custom gateway to authorize in integration tests (defaults to staging)
+INTEGRATION_TESTS_GATEWAY_DID = ''
+INTEGRATION_TESTS_GATEWAY_ENDPOINT = ''

.github/workflows/fetch-metrics-for-space.yaml

@@ -21,13 +21,13 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
-          node-version: 18
-          cache: 'npm'
-      - run: npm ci
+          node-version: 20
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile
       - name: Run job
         env:
           DEBUG: '*'

.github/workflows/follow-filecoin-receipt-chain.yaml

@@ -21,13 +21,13 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
-          node-version: 18
-          cache: 'npm'
-      - run: npm ci
+          node-version: 20
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile
       - name: Run job
         env:
           DEBUG: '*'

.github/workflows/get-oldest-pieces-pending-deal.yml

@@ -20,13 +20,13 @@ jobs:
         env: ['staging', 'production']
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
           node-version: 20
-          cache: 'npm'
-      - run: npm ci
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile
       - name: Run job
         env:
           DEBUG: '*'

.github/workflows/test.yaml

@@ -11,15 +11,15 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
-          node-version: 18
-          cache: 'npm'
-      - run: npm ci
-      - run: npm run lint
-      - run: npm test
+          node-version: 20
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm lint
+      - run: pnpm test
         env:
           AWS_REGION: 'us-west-2'
           AWS_ACCESS_KEY_ID: 'NOSUCH'

billing/package.json

@@ -8,34 +8,38 @@
     "coverage": "c8 -r text -r html npm test"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "3.563.0",
-    "@aws-sdk/client-sqs": "3.563.0",
-    "@aws-sdk/util-dynamodb": "3.563.0",
-    "@ipld/dag-json": "^10.2.3",
-    "@ipld/dag-ucan": "^3.4.5",
-    "@sentry/serverless": "^7.74.1",
-    "@storacha/capabilities": "^1.2.0",
-    "@ucanto/interface": "^10.2.0",
-    "@ucanto/server": "^10.2.0",
-    "big.js": "^6.2.1",
-    "lru-cache": "^11.0.0",
-    "multiformats": "^13.1.0",
-    "p-retry": "^6.2.0",
-    "stripe": "^14.2.0",
-    "uint8arrays": "^5.1.0"
+    "@aws-sdk/client-dynamodb": "catalog:",
+    "@aws-sdk/client-sqs": "catalog:",
+    "@aws-sdk/util-dynamodb": "catalog:",
+    "@ipld/dag-json": "catalog:",
+    "@ipld/dag-ucan": "catalog:",
+    "@sentry/serverless": "catalog:",
+    "@storacha/capabilities": "catalog:",
+    "@ucanto/interface": "catalog:",
+    "@ucanto/server": "catalog:",
+    "big.js": "catalog:",
+    "lru-cache": "catalog:",
+    "multiformats": "catalog:",
+    "p-retry": "catalog:",
+    "sst": "catalog:",
+    "stripe": "catalog:",
+    "uint8arrays": "catalog:"
   },
   "devDependencies": {
-    "@aws-sdk/client-glue": "3.563.0",
-    "@dsnp/parquetjs": "^1.8.6",
-    "@types/big.js": "^6.2.1",
-    "aws-lambda": "^1.0.7",
-    "c8": "^8.0.1",
-    "csv-parser": "^3.0.0",
-    "csv-stringify": "^6.4.6",
-    "dotenv": "^16.4.5",
-    "entail": "^2.1.1",
-    "p-all": "^5.0.0",
-    "testcontainers": "^10.7.1"
+    "@aws-sdk/client-glue": "catalog:",
+    "@aws-sdk/client-s3": "catalog:",
+    "@dsnp/parquetjs": "catalog:",
+    "@storacha/did-mailto": "catalog:",
+    "@types/aws-lambda": "catalog:",
+    "@types/big.js": "catalog:",
+    "aws-lambda": "catalog:",
+    "c8": "catalog:",
+    "csv-parser": "catalog:",
+    "csv-stringify": "catalog:",
+    "dotenv": "catalog:",
+    "entail": "catalog:",
+    "p-all": "catalog:",
+    "testcontainers": "catalog:"
   },
   "engines": {
     "node": ">=16.15"

carpark/package.json

@@ -6,16 +6,16 @@
     "test": "ava --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
-    "@aws-sdk/client-eventbridge": "3.563.0",
-    "@aws-sdk/client-sqs": "3.563.0",
-    "@sentry/serverless": "^7.74.1"
+    "@aws-sdk/client-eventbridge": "catalog:",
+    "@aws-sdk/client-sqs": "catalog:",
+    "@sentry/serverless": "catalog:"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "3.563.0",
-    "constructs": "*",
-    "ava": "^4.3.3",
-    "nanoid": "^4.0.0",
-    "p-retry": "6.2.0",
-    "testcontainers": "^10.7.1"
+    "@aws-sdk/client-s3": "catalog:",
+    "@types/aws-lambda": "catalog:",
+    "ava": "catalog:",
+    "nanoid": "catalog:",
+    "p-retry": "catalog:",
+    "testcontainers": "catalog:"
   }
 }

filecoin/metrics.js

@@ -4,7 +4,7 @@ import * as Block from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { Piece } from '@web3-storage/data-segment'
 
-import { hasOkReceipt } from '@storacha/upload-service-infra-upload-api/utils.js'
+import { hasOkReceipt } from '../upload-api/utils.js'
 import { AGGREGATE_ACCEPT, AGGREGATE_OFFER, METRICS_NAMES } from './constants.js'
 import { DecodeBlockOperationError, NotFoundWorkflowError } from './errors.js'
 
@@ -24,7 +24,7 @@ import { DecodeBlockOperationError, NotFoundWorkflowError } from './errors.js'
  * Metrics:
  * - AGGREGATE_ACCEPT_TOTAL: increment number of `aggregate/accept` success receipts
  * 
- * @param {import('@storacha/upload-service-infra-upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
+ * @param {import('../upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
  * @param {import('./types.js').FilecoinMetricsCtx} ctx
  */
 export async function updateAggregateAcceptTotal (ucanInvocations, ctx) {
@@ -50,7 +50,7 @@ export async function updateAggregateAcceptTotal (ucanInvocations, ctx) {
  * - AGGREGATE_OFFER_PIECES_TOTAL: increment number of pieces included in `aggregate/offer` success receipts
  * - AGGREGATE_OFFER_PIECES_SIZE_TOTAL: increment size of pieces included of `aggregate/offer` success receipts
  *
- * @param {import('@storacha/upload-service-infra-upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
+ * @param {import('../upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
  * @param {import('./types.js').FilecoinAggregateOfferMetricsCtx} ctx
  */
 export async function updateAggregateOfferTotal (ucanInvocations, ctx) {
@@ -104,7 +104,7 @@ export async function updateAggregateOfferTotal (ucanInvocations, ctx) {
 /**
  * Get a map of workflows that include given capability.
  *
- * @param {import('@storacha/upload-service-infra-upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
+ * @param {import('../upload-api/types.js').UcanStreamInvocation[]} ucanInvocations
  * @param {string} capability
  * @param {import('./types.js').FilecoinAggregateOfferMetricsCtx} ctx
  */

filecoin/package.json

@@ -6,34 +6,38 @@
     "test": "ava --serial --no-worker-threads --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "3.563.0",
-    "@aws-sdk/client-s3": "3.563.0",
-    "@aws-sdk/client-sqs": "3.563.0",
-    "@ipld/car": "^5.2.6",
-    "@ipld/dag-json": "^10.2.3",
-    "@sentry/serverless": "^7.74.1",
-    "@storacha/filecoin-api": "^1.1.6",
-    "@storacha/filecoin-client": "^1.0.1",
-    "@ucanto/client": "^9.0.1",
-    "@ucanto/core": "^10.3.1",
-    "@ucanto/interface": "^10.2.0",
-    "@ucanto/principal": "^9.0.1",
-    "@ucanto/transport": "^9.1.1",
-    "@web3-storage/data-segment": "^5.3.0",
-    "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0",
-    "multiformats": "^13.1.0",
-    "p-retry": "^6.2.0",
-    "uint8arrays": "^5.1.0"
+    "@aws-sdk/client-dynamodb": "catalog:",
+    "@aws-sdk/client-s3": "catalog:",
+    "@aws-sdk/client-sqs": "catalog:",
+    "@aws-sdk/util-dynamodb": "catalog:",
+    "@ipld/car": "catalog:",
+    "@ipld/dag-json": "catalog:",
+    "@sentry/serverless": "catalog:",
+    "@storacha/filecoin-api": "catalog:",
+    "@storacha/filecoin-client": "catalog:",
+    "@ucanto/client": "catalog:",
+    "@ucanto/core": "catalog:",
+    "@ucanto/interface": "catalog:",
+    "@ucanto/principal": "catalog:",
+    "@ucanto/server": "catalog:",
+    "@ucanto/transport": "catalog:",
+    "@web3-storage/data-segment": "catalog:",
+    "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "catalog:",
+    "multiformats": "catalog:",
+    "p-retry": "catalog:",
+    "sst": "catalog:",
+    "uint8arrays": "catalog:"
   },
   "devDependencies": {
-    "@web-std/blob": "3.0.5",
-    "ava": "^4.3.3",
-    "constructs": "*",
-    "delay": "^6.0.0",
-    "nanoid": "^4.0.0",
-    "p-defer": "^4.0.0",
-    "sqs-consumer": "^7.4.0",
-    "testcontainers": "^10.7.1"
+    "@ipld/dag-pb": "catalog:",
+    "@web-std/blob": "catalog:",
+    "@types/aws-lambda": "catalog:",
+    "ava": "catalog:",
+    "delay": "catalog:",
+    "nanoid": "catalog:",
+    "p-defer": "catalog:",
+    "sqs-consumer": "catalog:",
+    "testcontainers": "catalog:"
   },
   "eslintConfig": {
     "rules": {

filecoin/test/metrics-aggregate-accept-total.test.js

@@ -9,7 +9,7 @@ import { randomAggregate } from '@storacha/filecoin-api/test'
 import { updateAggregateAcceptTotal } from '../metrics.js'
 import { METRICS_NAMES, STREAM_TYPE } from '../constants.js'
 
-import { adminMetricsTableProps } from '@storacha/upload-service-infra-upload-api/tables/index.js'
+import { adminMetricsTableProps } from '../../upload-api/tables/index.js'
 import { createFilecoinMetricsTable } from '../store/metrics.js'
 import { createWorkflowStore } from '../store/workflow.js'
 

filecoin/test/metrics-aggregate-offer-total.test.js

@@ -14,7 +14,7 @@ import { Piece } from '@web3-storage/data-segment'
 import { updateAggregateOfferTotal } from '../metrics.js'
 import { METRICS_NAMES, STREAM_TYPE } from '../constants.js'
 
-import { adminMetricsTableProps } from '@storacha/upload-service-infra-upload-api/tables/index.js'
+import { adminMetricsTableProps } from '../../upload-api/tables/index.js'
 import { createFilecoinMetricsTable } from '../store/metrics.js'
 import { createWorkflowStore } from '../store/workflow.js'
 import { createInvocationStore } from '../store/invocation.js'

indexer/package.json

@@ -6,18 +6,21 @@
     "test": "entail '**/*.spec.js'"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "3.563.0",
-    "@aws-sdk/client-sqs": "3.563.0",
-    "@aws-sdk/util-dynamodb": "3.563.0",
-    "@ipld/dag-json": "^10.2.1",
-    "@storacha/one-webcrypto": "^1.0.1",
-    "@ucanto/interface": "^10.2.0",
-    "multiformats": "^13.1.1",
-    "p-map": "^7.0.2",
-    "p-retry": "^6.2.0"
+    "@aws-sdk/client-dynamodb": "catalog:",
+    "@aws-sdk/client-sqs": "catalog:",
+    "@aws-sdk/util-dynamodb": "catalog:",
+    "@ipld/dag-json": "catalog:",
+    "@sentry/serverless": "catalog:",
+    "@storacha/one-webcrypto": "catalog:",
+    "@ucanto/interface": "catalog:",
+    "multiformats": "catalog:",
+    "p-map": "catalog:",
+    "p-retry": "catalog:"
   },
   "devDependencies": {
-    "@ucanto/server": "^10.2.0",
-    "entail": "^2.1.2"
+    "@types/aws-lambda": "catalog:",
+    "@ucanto/server": "catalog:",
+    "entail": "catalog:",
+    "testcontainers": "catalog:"
   }
 }