Skip to content

Commit e5f8b73

Browse files
WaDadidouWaDadidou
authored
fix(media): restore broken IPFS media and route images through Next.js optimizer (#1032)
## Summary Fixes #1031 - **Backend**: adds a one-time CLI subcommand `pin-ipfs-cids` to re-pin all IPFS CIDs stored in the database to the current Pinata account, restoring access to media that was pinned to a previous account. Also includes the hardcoded default avatar CID (not stored in DB). - **Frontend**: removes the custom Pinata image-transform loader from `Web3Image`; IPFS URIs are now resolved to HTTPS gateway URLs before being handed to Next.js `Image`, so all media flows through the standard `_next/image` optimizer - Deletes the now-unused `lib/web3-img-loader.ts` - Adds `.github/workflows/run-ipfs-migration.yml` -- a manual `workflow_dispatch` to run the migration against staging or prod via Turso (libsql), using per-environment Pinata JWT secrets (`STAGING_PINATA_JWT` / `PROD_PINATA_JWT`) ## How to run the migration The migration is triggered manually via GitHub Actions after deploying: **GitHub Actions (staging or prod):** Go to Actions -> "Run IPFS Migration" -> Run workflow -> pick environment + dry_run. > Requires `STAGING_PINATA_JWT` and `PROD_PINATA_JWT` to be added as GitHub secrets. **Local (dev.db or any SQLite file):** ```sh # Preview CIDs without pinning PINATA_JWT=<jwt> go run ./backend pin-ipfs-cids -dry-run -db <path-to.db> # Pin all CIDs PINATA_JWT=<jwt> go run ./backend pin-ipfs-cids -db <path-to.db> ``` The `-db` flag also accepts a libsql DSN: `libsql://<host>?authToken=<token>`. `ZENAO_DB` env var is accepted as a fallback. ## Affected files | File | Change | |---|---| | `backend/pin_ipfs_cids.go` | New -- migration subcommand (supports SQLite + libsql/Turso) | | `backend/main.go` | Register new subcommand | | `components/widgets/images/web3-image.tsx` | Remove custom Pinata loader, resolve IPFS URI via `web2URL` | | `lib/web3-img-loader.ts` | Deleted -- no longer used | | `.github/workflows/run-ipfs-migration.yml` | New -- manual workflow to run the migration against staging/prod | ## Test plan **Pre-merge (local):** - [x] Run `pin-ipfs-cids -dry-run -db dev.db` and verify the expected CIDs are listed (includes the default avatar CID) - [x] Run `pin-ipfs-cids -db dev.db` and confirm all CIDs are queued on Pinata - [x] Verify images display on `/discover`, event pages, community pages, and profile pages - [ ] Verify audio posts play in social feeds - [x] Verify "Open image in new tab" displays inline instead of downloading **Post-merge (staging then prod):** - [ ] Add `STAGING_PINATA_JWT` and `PROD_PINATA_JWT` secrets to GitHub - [ ] Trigger "Run IPFS Migration" workflow on staging with `dry_run: true` -- verify expected CIDs are listed - [ ] Trigger "Run IPFS Migration" workflow on staging with `dry_run: false` -- confirm all CIDs are queued on Pinata - [ ] Verify images display correctly on staging - [ ] Repeat on prod
1 parent 0558c75 commit e5f8b73

6 files changed

Lines changed: 267 additions & 35 deletions

File tree

‎.github/workflows/run-ipfs-migration.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Run IPFS Migration
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
environment:
7+
description: "Target environment"
8+
required: true
9+
type: choice
10+
options:
11+
- staging
12+
- prod
13+
dry_run:
14+
description: "Dry run (list CIDs without pinning)"
15+
required: false
16+
type: boolean
17+
default: true
18+
19+
jobs:
20+
pin-ipfs-cids:
21+
runs-on: ubuntu-latest
22+
steps:
23+
- uses: actions/checkout@v4
24+
25+
- name: Setup Golang with cache
26+
uses: magnetikonline/action-golang-cache@v5
27+
with:
28+
go-version-file: go.mod
29+
cache-key-suffix: ipfs-migration
30+
31+
- name: Set DB URL (staging)
32+
if: inputs.environment == 'staging'
33+
env:
34+
TOKEN: ${{ secrets.STAGING_TURSO_TOKEN }}
35+
run: echo "ZENAO_DB=libsql://zenao-staging-3-samourai-coop.turso.io?authToken=${TOKEN}" >> $GITHUB_ENV
36+
37+
- name: Set DB URL (prod)
38+
if: inputs.environment == 'prod'
39+
env:
40+
TOKEN: ${{ secrets.PROD_TURSO_TOKEN }}
41+
run: echo "ZENAO_DB=libsql://zenao-prod-samourai-coop.turso.io?authToken=${TOKEN}" >> $GITHUB_ENV
42+
43+
- name: Set Pinata JWT (staging)
44+
if: inputs.environment == 'staging'
45+
env:
46+
TOKEN: ${{ secrets.STAGING_PINATA_JWT }}
47+
run: echo "PINATA_JWT=${TOKEN}" >> $GITHUB_ENV
48+
49+
- name: Set Pinata JWT (prod)
50+
if: inputs.environment == 'prod'
51+
env:
52+
TOKEN: ${{ secrets.PROD_PINATA_JWT }}
53+
run: echo "PINATA_JWT=${TOKEN}" >> $GITHUB_ENV
54+
55+
- name: Run pin-ipfs-cids (dry run)
56+
if: inputs.dry_run == true
57+
run: go run ./backend pin-ipfs-cids -dry-run
58+
59+
- name: Run pin-ipfs-cids
60+
if: inputs.dry_run == false
61+
run: go run ./backend pin-ipfs-cids

‎backend/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func main() {
4747
newGenticketCmd(),
4848
newGenPdfTicketCmd(),
4949
newConvertEvtToComCmd(),
50+
newPinIPFSCIDsCmd(),
5051
)
5152

5253
cmd.Execute(context.Background(), os.Args[1:])

‎backend/pin_ipfs_cids.go‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"flag"
8+
"fmt"
9+
"io"
10+
"net/http"
11+
"os"
12+
"strings"
13+
"time"
14+
15+
"github.com/gnolang/gno/tm2/pkg/commands"
16+
"go.uber.org/zap"
17+
"gorm.io/driver/sqlite"
18+
"gorm.io/gorm"
19+
)
20+
21+
func newPinIPFSCIDsCmd() *commands.Command {
22+
return commands.NewCommand(
23+
commands.Metadata{
24+
Name: "pin-ipfs-cids",
25+
ShortUsage: "pin-ipfs-cids [flags]",
26+
ShortHelp: "pin all IPFS CIDs stored in the database to the current Pinata account",
27+
},
28+
&pinIPFSCIDsConf,
29+
func(ctx context.Context, args []string) error {
30+
return pinIPFSCIDs()
31+
},
32+
)
33+
}
34+
35+
var pinIPFSCIDsConf = pinIPFSCIDsConfig{}
36+
37+
type pinIPFSCIDsConfig struct {
38+
dbPath string
39+
pinataJWT string
40+
dryRun bool
41+
}
42+
43+
func (conf *pinIPFSCIDsConfig) RegisterFlags(flset *flag.FlagSet) {
44+
flset.StringVar(&conf.dbPath, "db", "dev.db", "path to the SQLite database or a libsql DSN (libsql://...?authToken=...)")
45+
flset.StringVar(&conf.pinataJWT, "pinata-jwt", "", "Pinata API JWT token (or set PINATA_JWT env var)")
46+
flset.BoolVar(&conf.dryRun, "dry-run", false, "list CIDs without actually pinning them")
47+
}
48+
49+
func pinIPFSCIDs() error {
50+
if val := os.Getenv("ZENAO_DB"); val != "" && pinIPFSCIDsConf.dbPath == "dev.db" {
51+
pinIPFSCIDsConf.dbPath = val
52+
}
53+
if val := os.Getenv("PINATA_JWT"); val != "" {
54+
pinIPFSCIDsConf.pinataJWT = val
55+
}
56+
if !pinIPFSCIDsConf.dryRun && pinIPFSCIDsConf.pinataJWT == "" {
57+
return fmt.Errorf("-pinata-jwt or PINATA_JWT env var is required")
58+
}
59+
60+
logger, err := zap.NewDevelopment()
61+
if err != nil {
62+
return err
63+
}
64+
65+
var db *gorm.DB
66+
if strings.HasPrefix(pinIPFSCIDsConf.dbPath, "libsql") {
67+
db, err = gorm.Open(sqlite.New(sqlite.Config{
68+
DriverName: "libsql",
69+
DSN: pinIPFSCIDsConf.dbPath,
70+
}), &gorm.Config{})
71+
} else {
72+
db, err = gorm.Open(sqlite.Open(pinIPFSCIDsConf.dbPath), &gorm.Config{})
73+
}
74+
if err != nil {
75+
return fmt.Errorf("open database: %w", err)
76+
}
77+
78+
cids, err := collectAllIPFSCIDs(db)
79+
if err != nil {
80+
return fmt.Errorf("collect CIDs: %w", err)
81+
}
82+
logger.Info("collected unique IPFS CIDs", zap.Int("count", len(cids)))
83+
84+
if pinIPFSCIDsConf.dryRun {
85+
for _, cid := range cids {
86+
fmt.Println(cid)
87+
}
88+
return nil
89+
}
90+
91+
client := &http.Client{Timeout: 30 * time.Second}
92+
pinned, failed := 0, 0
93+
for _, cid := range cids {
94+
logger.Info("pinning", zap.String("cid", cid))
95+
if err := callPinByHash(client, pinIPFSCIDsConf.pinataJWT, cid); err != nil {
96+
logger.Error("failed to pin", zap.String("cid", cid), zap.Error(err))
97+
failed++
98+
} else {
99+
pinned++
100+
}
101+
}
102+
103+
logger.Info("done", zap.Int("pinned", pinned), zap.Int("failed", failed))
104+
if failed > 0 {
105+
return fmt.Errorf("%d CID(s) failed to pin", failed)
106+
}
107+
return nil
108+
}
109+
110+
// allIPFSCIDsQuery collects every distinct ipfs:// URI across all tables that store
111+
// user-uploaded media, so we can pin them all to the current Pinata account.
112+
const allIPFSCIDsQuery = `
113+
SELECT DISTINCT uri FROM (
114+
SELECT avatar_uri AS uri FROM users WHERE avatar_uri LIKE 'ipfs://%'
115+
UNION
116+
SELECT image_uri FROM events WHERE image_uri LIKE 'ipfs://%'
117+
UNION
118+
SELECT avatar_uri FROM communities WHERE avatar_uri LIKE 'ipfs://%'
119+
UNION
120+
SELECT banner_uri FROM communities WHERE banner_uri LIKE 'ipfs://%'
121+
UNION
122+
SELECT preview_image_uri FROM posts WHERE preview_image_uri LIKE 'ipfs://%'
123+
UNION
124+
SELECT image_uri FROM posts WHERE image_uri LIKE 'ipfs://%'
125+
UNION
126+
SELECT audio_uri FROM posts WHERE audio_uri LIKE 'ipfs://%'
127+
UNION
128+
SELECT video_uri FROM posts WHERE video_uri LIKE 'ipfs://%'
129+
UNION
130+
SELECT thumbnail_image_uri FROM posts WHERE thumbnail_image_uri LIKE 'ipfs://%'
131+
)`
132+
133+
func collectAllIPFSCIDs(db *gorm.DB) ([]string, error) {
134+
rows, err := db.Raw(allIPFSCIDsQuery).Rows()
135+
if err != nil {
136+
return nil, err
137+
}
138+
defer rows.Close()
139+
140+
var cids []string
141+
for rows.Next() {
142+
var uri string
143+
if err := rows.Scan(&uri); err != nil {
144+
return nil, err
145+
}
146+
cids = append(cids, strings.TrimPrefix(uri, "ipfs://"))
147+
}
148+
return cids, rows.Err()
149+
}
150+
151+
type pinByHashBody struct {
152+
HashToPin string `json:"hashToPin"`
153+
PinataOptions pinataOpts `json:"pinataOptions"`
154+
}
155+
156+
type pinataOpts struct {
157+
CIDVersion int `json:"cidVersion"`
158+
}
159+
160+
// callPinByHash queues a CID to be pinned to the Pinata account associated with
161+
// jwt. The operation is asynchronous: Pinata fetches the content from the IPFS
162+
// network in the background after this call returns.
163+
func callPinByHash(client *http.Client, jwt, cid string) error {
164+
body, err := json.Marshal(pinByHashBody{
165+
HashToPin: cid,
166+
PinataOptions: pinataOpts{CIDVersion: 1},
167+
})
168+
if err != nil {
169+
return err
170+
}
171+
172+
req, err := http.NewRequest(http.MethodPost, "https://api.pinata.cloud/pinning/pinByHash", bytes.NewReader(body))
173+
if err != nil {
174+
return err
175+
}
176+
req.Header.Set("Content-Type", "application/json")
177+
req.Header.Set("Authorization", "Bearer "+jwt)
178+
179+
resp, err := client.Do(req)
180+
if err != nil {
181+
return err
182+
}
183+
defer resp.Body.Close()
184+
185+
if resp.StatusCode >= 400 {
186+
respBody, _ := io.ReadAll(resp.Body)
187+
return fmt.Errorf("pinata returned status %d: %s", resp.StatusCode, string(respBody))
188+
}
189+
return nil
190+
}

‎components/widgets/images/web3-image.tsx‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,21 @@
22

33
import React from "react";
44
import Image, { ImageProps } from "next/image";
5-
import withWeb3ImgLoader from "@/lib/web3-img-loader";
5+
import { web2URL } from "@/lib/uris";
66
import { cn } from "@/lib/tailwind";
77

88
export const Web3Image = React.forwardRef<
99
HTMLImageElement,
10-
Omit<ImageProps, "loader"> & {
11-
imgFit?: "scale-down" | "contain" | "cover" | "crop" | "pad";
12-
}
13-
>(({ alt, src, className, imgFit, ...props }, ref) => {
14-
const isWeb3 = typeof src === "string" && src.startsWith("ipfs://");
10+
Omit<ImageProps, "loader">
11+
>(({ alt, src, className, ...props }, ref) => {
12+
const resolvedSrc =
13+
typeof src === "string" && src.startsWith("ipfs://") ? web2URL(src) : src;
1514

1615
return (
1716
<Image
1817
ref={ref}
19-
src={src}
18+
src={resolvedSrc}
2019
alt={alt}
21-
loader={isWeb3 ? withWeb3ImgLoader({ imgFit }) : undefined}
2220
className={cn("bg-primary/10", className)}
2321
{...props}
2422
/>

‎lib/web3-img-loader.ts‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎next.config.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ const nextConfig: NextConfig = {
7373
].join("; ");
7474

7575
return [
76+
{
77+
source: "/_next/image",
78+
headers: [
79+
{
80+
key: "Content-Disposition",
81+
value: "inline",
82+
},
83+
],
84+
},
7685
{
7786
source: "/(.*)",
7887
headers: [

0 commit comments

Comments
 (0)