Skip to content

Commit 980447e

Browse files
hacdiashacdias
authored
Merge pull request #613 from ipfs/release-v0.20.0
Release v0.20.0
2 parents b50fcfd + 797e6b4 commit 980447e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+5361
-993
lines changed

‎.github/workflows/gateway-conformance.yml‎

Lines changed: 158 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
name: Gateway Conformance
2+
# This workflow runs https://github.com/ipfs/gateway-conformance
3+
# against different backend implementations of boxo/gateway
24

35
on:
46
push:
57
branches:
68
- main
79
pull_request:
10+
workflow_dispatch:
811

912
concurrency:
1013
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
1114
cancel-in-progress: true
1215

1316
jobs:
14-
gateway-conformance:
17+
# This test uses a static CAR file as a local blockstore,
18+
# allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go)
19+
# which is used by implementations like Kubo
20+
local-block-backend:
1521
runs-on: ubuntu-latest
1622
steps:
1723
# 1. Download the gateway-conformance fixtures
@@ -21,49 +27,185 @@ jobs:
2127
output: fixtures
2228
merged: true
2329

24-
# 2. Build the car-gateway
30+
# 2. Build the gateway binary
31+
- name: Checkout boxo
32+
uses: actions/checkout@v4
33+
with:
34+
path: boxo
35+
- name: Setup Go
36+
uses: actions/setup-go@v5
37+
with:
38+
go-version-file: 'boxo/examples/go.mod'
39+
cache-dependency-path: "boxo/**/*.sum"
40+
- name: Build test-gateway
41+
run: go build -o test-gateway
42+
working-directory: boxo/examples/gateway/car-file
43+
44+
# 3. Start the gateway binary
45+
- name: Start test-gateway
46+
run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 &
47+
48+
# 4. Run the gateway-conformance tests
49+
- name: Run gateway-conformance tests
50+
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
51+
with:
52+
gateway-url: http://127.0.0.1:8040
53+
json: output.json
54+
xml: output.xml
55+
html: output.html
56+
markdown: output.md
57+
subdomain-url: http://example.net
58+
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
59+
60+
# 5. Upload the results
61+
- name: Upload MD summary
62+
if: failure() || success()
63+
run: cat output.md >> $GITHUB_STEP_SUMMARY
64+
- name: Upload HTML report
65+
if: failure() || success()
66+
uses: actions/upload-artifact@v4
67+
with:
68+
name: gateway-conformance_local-block-backend.html
69+
path: output.html
70+
- name: Upload JSON report
71+
if: failure() || success()
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: gateway-conformance_local-block-backend.json
75+
path: output.json
76+
77+
# This test uses remote block gateway (?format=raw) as a remote blockstore,
78+
# allowing us to test conformance against RemoteBlocksBackend
79+
# (gateway/backend_blocks.go) which is used by implementations like
80+
# rainbow configured to use with remote block backend
81+
# Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
82+
remote-block-backend:
83+
runs-on: ubuntu-latest
84+
steps:
85+
# 1. Download the gateway-conformance fixtures
86+
- name: Download gateway-conformance fixtures
87+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
88+
with:
89+
output: fixtures
90+
merged: true
91+
92+
# 2. Build the gateway binaries
93+
- name: Checkout boxo
94+
uses: actions/checkout@v4
95+
with:
96+
path: boxo
2597
- name: Setup Go
26-
uses: actions/setup-go@v4
98+
uses: actions/setup-go@v5
99+
with:
100+
go-version-file: 'boxo/examples/go.mod'
101+
cache-dependency-path: "boxo/**/*.sum"
102+
- name: Build remote-block-backend # it will act as a trustless CAR gateway
103+
run: go build -o remote-block-backend
104+
working-directory: boxo/examples/gateway/car-file
105+
- name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend
106+
run: go build -o test-gateway
107+
working-directory: boxo/examples/gateway/proxy-blocks
108+
109+
# 3. Start the gateway binaries
110+
- name: Start remote HTTP backend that serves application/vnd.ipld.raw
111+
run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests
112+
- name: Start gateway that uses the remote block backend
113+
run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 &
114+
115+
# 4. Run the gateway-conformance tests
116+
- name: Run gateway-conformance tests
117+
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
118+
with:
119+
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
120+
json: output.json
121+
xml: output.xml
122+
html: output.html
123+
markdown: output.md
124+
subdomain-url: http://example.net
125+
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
126+
args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
127+
128+
# 5. Upload the results
129+
- name: Upload MD summary
130+
if: failure() || success()
131+
run: cat output.md >> $GITHUB_STEP_SUMMARY
132+
- name: Upload HTML report
133+
if: failure() || success()
134+
uses: actions/upload-artifact@v4
27135
with:
28-
go-version: 1.21.x
136+
name: gateway-conformance_remote-block-backend.html
137+
path: output.html
138+
- name: Upload JSON report
139+
if: failure() || success()
140+
uses: actions/upload-artifact@v4
141+
with:
142+
name: gateway-conformance_remote-block-backend.json
143+
path: output.json
144+
145+
# This test uses remote CAR gateway (?format=car, IPIP-402)
146+
# allowing us to test conformance against remote CarFetcher backend.
147+
# (gateway/backend_car_fetcher.go) which is used by implementations like
148+
# rainbow configured to use with remote car backend
149+
# Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car
150+
remote-car-backend:
151+
runs-on: ubuntu-latest
152+
steps:
153+
# 1. Download the gateway-conformance fixtures
154+
- name: Download gateway-conformance fixtures
155+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
156+
with:
157+
output: fixtures
158+
merged: true
159+
160+
# 2. Build the gateway binaries
29161
- name: Checkout boxo
30162
uses: actions/checkout@v4
31163
with:
32164
path: boxo
33-
- name: Build car-gateway
34-
run: go build -o car-gateway
35-
working-directory: boxo/examples/gateway/car
165+
- name: Setup Go
166+
uses: actions/setup-go@v5
167+
with:
168+
go-version-file: 'boxo/examples/go.mod'
169+
cache-dependency-path: "boxo/**/*.sum"
170+
- name: Build remote-car-backend # it will act as a trustless CAR gateway
171+
run: go build -o remote-car-backend
172+
working-directory: boxo/examples/gateway/car-file
173+
- name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend
174+
run: go build -o test-gateway
175+
working-directory: boxo/examples/gateway/proxy-car
36176

37-
# 3. Start the car-gateway
38-
- name: Start car-gateway
39-
run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 &
177+
# 3. Start the gateway binaries
178+
- name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402)
179+
run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests
180+
- name: Start gateway that uses the remote CAR backend
181+
run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 &
40182

41183
# 4. Run the gateway-conformance tests
42184
- name: Run gateway-conformance tests
43185
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
44186
with:
45-
gateway-url: http://127.0.0.1:8040
187+
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
46188
json: output.json
47189
xml: output.xml
48190
html: output.html
49191
markdown: output.md
50192
subdomain-url: http://example.net
51193
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
52-
args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
194+
args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
53195

54196
# 5. Upload the results
55197
- name: Upload MD summary
56198
if: failure() || success()
57199
run: cat output.md >> $GITHUB_STEP_SUMMARY
58200
- name: Upload HTML report
59201
if: failure() || success()
60-
uses: actions/upload-artifact@v3
202+
uses: actions/upload-artifact@v4
61203
with:
62-
name: gateway-conformance.html
204+
name: gateway-conformance_remote-car-backend.html
63205
path: output.html
64206
- name: Upload JSON report
65207
if: failure() || success()
66-
uses: actions/upload-artifact@v3
208+
uses: actions/upload-artifact@v4
67209
with:
68-
name: gateway-conformance.json
210+
name: gateway-conformance_remote-car-backend.json
69211
path: output.json

‎.github/workflows/gateway-sharness.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
- name: Setup Go
1919
uses: actions/setup-go@v4
2020
with:
21-
go-version: 1.21.x
21+
go-version: 1.22.x
2222
- name: Checkout boxo
2323
uses: actions/checkout@v3
2424
with:

‎CHANGELOG.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,22 @@ The following emojis are used to highlight certain changes:
2424

2525
### Security
2626

27+
## [v0.20.0]
28+
29+
### Added
30+
31+
* ✨ `gateway` has new backend possibilities:
32+
* `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
33+
* `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
34+
* `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys.
35+
* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `application/vnd.ipld.car` parameters sent via `Accept` header. The parameters in the `Accept` header have always priority, but including them in URL simplifies HTTP caching and allows use in `Content-Location` header on CAR responses to maximize interoperability with wide array of HTTP caches.
36+
* `bitswap/server` now allows to override the default peer ledger with `WithPeerLedger`.
37+
38+
### Fixed
39+
40+
* `routing/http/server` now returns 404 Status Not Found when no records can be found.
41+
* `routing/http/server` now supports legacy RSA PeerIDs encoded as Base58 Multihash
42+
2743
## [v0.19.0]
2844

2945
### Added

‎bitswap/network/ipfs_impl.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,17 @@ import (
1111
bsmsg "github.com/ipfs/boxo/bitswap/message"
1212
"github.com/ipfs/boxo/bitswap/network/internal"
1313

14-
cid "github.com/ipfs/go-cid"
14+
"github.com/ipfs/go-cid"
1515
logging "github.com/ipfs/go-log/v2"
1616
"github.com/libp2p/go-libp2p/core/connmgr"
1717
"github.com/libp2p/go-libp2p/core/host"
1818
"github.com/libp2p/go-libp2p/core/network"
1919
"github.com/libp2p/go-libp2p/core/peer"
20-
peerstore "github.com/libp2p/go-libp2p/core/peerstore"
20+
"github.com/libp2p/go-libp2p/core/peerstore"
2121
"github.com/libp2p/go-libp2p/core/protocol"
2222
"github.com/libp2p/go-libp2p/core/routing"
2323
"github.com/libp2p/go-libp2p/p2p/protocol/ping"
24-
msgio "github.com/libp2p/go-msgio"
24+
"github.com/libp2p/go-msgio"
2525
ma "github.com/multiformats/go-multiaddr"
2626
"github.com/multiformats/go-multistream"
2727
)
@@ -452,7 +452,7 @@ func (nn *netNotifiee) impl() *impl {
452452

453453
func (nn *netNotifiee) Connected(n network.Network, v network.Conn) {
454454
// ignore transient connections
455-
if v.Stat().Transient {
455+
if v.Stat().Limited {
456456
return
457457
}
458458

‎bitswap/options.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ func WithScoreLedger(scoreLedger server.ScoreLedger) Option {
5959
return Option{server.WithScoreLedger(scoreLedger)}
6060
}
6161

62+
func WithPeerLedger(peerLedger server.PeerLedger) Option {
63+
return Option{server.WithPeerLedger(peerLedger)}
64+
}
65+
6266
func WithTargetMessageSize(tms int) Option {
6367
return Option{server.WithTargetMessageSize(tms)}
6468
}

‎bitswap/server/forward.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,6 @@ type (
1111
TaskInfo = decision.TaskInfo
1212
ScoreLedger = decision.ScoreLedger
1313
ScorePeerFunc = decision.ScorePeerFunc
14+
PeerLedger = decision.PeerLedger
15+
PeerEntry = decision.PeerEntry
1416
)

‎bitswap/server/internal/decision/engine.go‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,42 @@ type ScoreLedger interface {
123123
Stop()
124124
}
125125

126+
type PeerEntry struct {
127+
Peer peer.ID
128+
Priority int32
129+
WantType pb.Message_Wantlist_WantType
130+
}
131+
132+
// PeerLedger is an external ledger dealing with peers and their want lists.
133+
type PeerLedger interface {
134+
// Wants informs the ledger that [peer.ID] wants [wl.Entry].
135+
Wants(p peer.ID, e wl.Entry)
136+
137+
// CancelWant returns true if the [cid.Cid] is present in the wantlist of [peer.ID].
138+
CancelWant(p peer.ID, k cid.Cid) bool
139+
140+
// CancelWantWithType will not cancel WantBlock if we sent a HAVE message.
141+
CancelWantWithType(p peer.ID, k cid.Cid, typ pb.Message_Wantlist_WantType)
142+
143+
// Peers returns all peers that want [cid.Cid].
144+
Peers(k cid.Cid) []PeerEntry
145+
146+
// CollectPeerIDs returns all peers that the ledger has an active session with.
147+
CollectPeerIDs() []peer.ID
148+
149+
// WantlistSizeForPeer returns the size of the wantlist for [peer.ID].
150+
WantlistSizeForPeer(p peer.ID) int
151+
152+
// WantlistForPeer returns the wantlist for [peer.ID].
153+
WantlistForPeer(p peer.ID) []wl.Entry
154+
155+
// ClearPeerWantlist clears the wantlist for [peer.ID].
156+
ClearPeerWantlist(p peer.ID)
157+
158+
// PeerDisconnected informs the ledger that [peer.ID] is no longer connected.
159+
PeerDisconnected(p peer.ID)
160+
}
161+
126162
// Engine manages sending requested blocks to peers.
127163
type Engine struct {
128164
// peerRequestQueue is a priority queue of requests received from peers.
@@ -150,7 +186,7 @@ type Engine struct {
150186
lock sync.RWMutex // protects the fields immediately below
151187

152188
// peerLedger saves which peers are waiting for a Cid
153-
peerLedger *peerLedger
189+
peerLedger PeerLedger
154190

155191
// an external ledger dealing with peer scores
156192
scoreLedger ScoreLedger
@@ -240,6 +276,13 @@ func WithScoreLedger(scoreledger ScoreLedger) Option {
240276
}
241277
}
242278

279+
// WithPeerLedger sets a custom [PeerLedger] to be used with this [Engine].
280+
func WithPeerLedger(peerLedger PeerLedger) Option {
281+
return func(e *Engine) {
282+
e.peerLedger = peerLedger
283+
}
284+
}
285+
243286
// WithBlockstoreWorkerCount sets the number of worker threads used for
244287
// blockstore operations in the decision engine
245288
func WithBlockstoreWorkerCount(count int) Option {
@@ -359,7 +402,7 @@ func newEngine(
359402
taskWorkerCount: defaults.BitswapEngineTaskWorkerCount,
360403
sendDontHaves: true,
361404
self: self,
362-
peerLedger: newPeerLedger(),
405+
peerLedger: NewDefaultPeerLedger(),
363406
pendingGauge: bmetrics.PendingEngineGauge(ctx),
364407
activeGauge: bmetrics.ActiveEngineGauge(ctx),
365408
targetMessageSize: defaultTargetMessageSize,

0 commit comments

Comments
 (0)