refactor: remove pinata (#2078)

This PR removes the pinata cron job (attempting to pin things), and removes the pinata default pin that gets inserted when an upload is created.

The pinata entry in the database service enum remains because a) we have loads of rows referencing that value and b) you can't remove a postgres enum value - you need to drop the whole enum and recreate (which would be really difficult considering the number of rows we have referencing it).

This effectively stops NFT.Storage pinning to Pinata.

closes #260
closes #391

Author: Alan Shaw

.github/workflows/cron-pinata.yml

@@ -66,7 +66,6 @@ The vars below are used by the API and/or GitHub cron actions and should be set
 | STAGING_DATABASE_TOKEN      | Auth token for staging Postgrest API                   |                                          |
 | PROD_DATABASE_CONNECTION    | Connection string for prodction db (used by cron jobs) |                                          |
 | STAGING_DATABASE_CONNECTION | Connection string for staging db (used by cron jobs)   |                                          |
-| PINATA_JWT                  | Auth token for pinning to Pinata cluster               |                                          |
 | DAG_CARGO_HOST              | Endpoint for dag cargo API                             |                                          |
 | DAG_CARGO_DATABASE          | dag cargo database name                                |                                          |
 | DAG_CARGO_USER              | dag cargo db username                                  |                                          |

DEVELOPMENT.md

@@ -64,7 +64,6 @@ Go to `/packages/api/src/constants.js` _uncomment_ the first line and run `wrang
 # production secrets
 wrangler secret put MAGIC_SECRET_KEY --env production # Get from magic.link account
 wrangler secret put SALT --env production # open `https://csprng.xyz/v1/api` in the browser and use the value of `Data`
-wrangler secret put PINATA_JWT --env production # Get from Pinata
 wrangler secret put SENTRY_DSN --env USER # Get from Sentry
 wrangler secret put DATABASE_TOKEN --env production # Get from database account
 wrangler secret put CLUSTER_BASIC_AUTH_TOKEN --env production # Get from nft.storage vault in 1password

packages/api/README.md

@@ -39,13 +39,13 @@ CREATE TYPE pin_status_type AS ENUM (
 
 -- Service type is the place/location/organisation that is pinning the content.
 CREATE TYPE service_type AS ENUM (
-    -- The NFT.Storage cluster in Pinata.
+    -- The NFT.Storage cluster in Pinata (LEGACY).
     'Pinata',
     -- The original NFT.Storage cluster.
     'IpfsCluster',
-    -- The current cluster, originally commissioned for niftysave.
+    -- An IPFS Cluster originally commissioned for niftysave.
     'IpfsCluster2',
-    -- New cluster with flatfs and better DHT
+    -- New cluster with flatfs and better DHT.
     'IpfsCluster3'
     );
 

packages/api/db/tables.sql

@@ -143,7 +143,7 @@ export type NFT = {
    */
   files: Array<{ name?: string; type?: string } | undefined>
   /**
-   * Pinata pin name and meta.
+   * Pin name and meta.
    */
   pin?: { name?: string; meta?: Record<string, string> }
   /**

packages/api/src/bindings.d.ts

@@ -267,10 +267,6 @@ export class DBClient {
         status: 'PinQueued',
         service: 'IpfsCluster3',
       },
-      {
-        status: 'PinQueued',
-        service: 'Pinata',
-      },
     ]
 
     const now = new Date().toISOString()

packages/api/src/utils/db-client.js

@@ -80,12 +80,6 @@ describe('NFT store', () => {
         status: 'PinQueued',
         service: 'IpfsCluster3',
       },
-      {
-        content_cid:
-          'bafyreicnwbboevx6g6fykitf4nebz2kqgkqz35qvlnlcgfulhrris66m6i',
-        status: 'PinQueued',
-        service: 'Pinata',
-      },
     ])
   })
 
@@ -145,12 +139,6 @@ describe('NFT store', () => {
         status: 'PinQueued',
         service: 'IpfsCluster3',
       },
-      {
-        content_cid:
-          'bafyreibubvuqeh5lm2ccose7uq54wweeajtjuhcj4pciogeq2lhbw2a23y',
-        status: 'PinQueued',
-        service: 'Pinata',
-      },
     ])
   })
 })

packages/api/test/nfts-store.spec.js

@@ -249,8 +249,6 @@ export interface FinalizedDeal extends DealInfo {
 }
 
 export interface Pin {
-  // Pinata does not provide this
-  // requestid: string
   cid: CIDString
   name?: string
   status: PinStatus

packages/client/src/lib/interface.ts

@@ -7,8 +7,7 @@
   "type": "module",
   "bin": {
     "nftstorage-metrics": "src/bin/metrics.js",
-    "nftstorage-pins": "src/bin/pins.js",
-    "nftstorage-pinata": "src/bin/pinata.js"
+    "nftstorage-pins": "src/bin/pins.js"
   },
   "scripts": {
     "lint": "eslint 'src/**/*.js'",
@@ -17,7 +16,6 @@
     "start:nft-ttr": "node src/bin/nft-ttr.js",
     "start:pins": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/pins.js",
     "start:pins-failed": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/pins-failed.js",
-    "start:pinata": "node src/bin/pinata.js",
     "start:dagcargo:sizes": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/dagcargo-sizes.js",
     "start:metrics": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/metrics.js",
     "test": "c8 ava",
@@ -32,7 +30,6 @@
     "dotenv": "^10.0.0",
     "form-data": "^4.0.0",
     "it-to-buffer": "^2.0.2",
-    "limiter": "2.0.1",
     "nft.storage": "^6.0.0",
     "p-retry": "^4.6.1",
     "p-settle": "^5.0.0",
@@ -128,7 +125,6 @@
       {
         "files": [
           "./src/jobs/dagcargo.js",
-          "./src/jobs/pinata.js",
           "./src/jobs/pins.js",
           "./src/lib/ipfs.js"
         ],
@@ -140,8 +136,7 @@
         "files": [
           "./src/jobs/dagcargo.js",
           "./src/jobs/pins.js",
-          "./src/lib/fetch.js",
-          "./src/lib/pinata.js"
+          "./src/lib/fetch.js"
         ],
         "rules": {
           "@typescript-eslint/no-unsafe-return": "off"
@@ -151,7 +146,6 @@
         "files": [
           "./src/bin/dagcargo-sizes.js",
           "./src/bin/metrics.js",
-          "./src/bin/pinata.js",
           "./src/bin/pins-failed.js",
           "./src/bin/pins.js"
         ],
@@ -163,13 +157,10 @@
         "files": [
           "./src/bin/dagcargo-sizes.js",
           "./src/bin/metrics.js",
-          "./src/bin/pinata.js",
           "./src/bin/pins-failed.js",
           "./src/bin/pins.js",
           "./src/jobs/dagcargo.js",
-          "./src/jobs/pinata.js",
-          "./src/jobs/pins.js",
-          "./src/lib/pinata.js"
+          "./src/jobs/pins.js"
         ],
         "rules": {
           "@typescript-eslint/naming-convention": "off"