Skip to content

Commit 6b09374

Browse files
hsanjuanhsanjuan
authored
Merge pull request #268 from ipfs/branch-v0.23.0
Update to Kubo v0.23.0 - add prebuilt-binary pipeline
2 parents 07312ee + 8ca41f8 commit 6b09374

File tree

11 files changed

+616
-1550
lines changed

11 files changed

+616
-1550
lines changed

‎.github/workflows/plugin-test.yml‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎.github/workflows/release.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: Kubo plugin
2+
3+
on:
4+
push:
5+
6+
7+
env:
8+
# must be same as official ipfs builds. See distributions/.tool-versions
9+
GO_VERSION: "1.21.1"
10+
11+
jobs:
12+
build-artifacts:
13+
name: "Build"
14+
strategy:
15+
matrix:
16+
os:
17+
- ubuntu-20.04
18+
- macos-latest
19+
runs-on: ${{ matrix.os }}
20+
steps:
21+
- uses: actions/checkout@v4
22+
with:
23+
fetch-depth: 2
24+
- name: Set up Go
25+
uses: actions/setup-go@v4
26+
with:
27+
go-version: ${{ env.GO_VERSION }}
28+
- name: Extract release name from tag
29+
run: |
30+
RELEASE=$(basename ${{ github.ref_name }})
31+
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
32+
- name: Package plugin
33+
run: make dist-plugin
34+
- name: Record Go environment
35+
run: |
36+
echo "GOHOSTOS=$(go env GOHOSTOS)" >> "$GITHUB_ENV"
37+
echo "GOHOSTARCH=$(go env GOHOSTARCH)" >> "$GITHUB_ENV"
38+
- name: Rename package
39+
run: |
40+
NAME="go-ds-s3-plugin_${{ env.RELEASE }}_${{ env.GOHOSTOS }}_${{ env.GOHOSTARCH }}.tar.gz"
41+
mv go-ds-s3-plugin/go-ds-s3-plugin.tar.gz "$NAME"
42+
echo "ARTIFACT_NAME=$NAME" >> "$GITHUB_ENV"
43+
- name: Archive artifacts
44+
uses: actions/upload-artifact@v3
45+
with:
46+
name: ${{ env.ARTIFACT_NAME }}
47+
path: go-ds-s3-plugin_*.tar.gz
48+
49+
test-artifacts:
50+
name: "Test"
51+
needs: build-artifacts
52+
strategy:
53+
matrix:
54+
os:
55+
- ubuntu-20.04
56+
# macos test fail with dist-built ipfs.
57+
# - macos-latest
58+
runs-on: ${{ matrix.os }}
59+
steps:
60+
- uses: actions/checkout@v4
61+
with:
62+
fetch-depth: 2
63+
- name: Set up Go
64+
uses: actions/setup-go@v4
65+
with:
66+
go-version: ${{ env.GO_VERSION }}
67+
- name: Record variables
68+
run: |
69+
V=`cat go.mod | grep github.com/ipfs/kubo | grep -o 'v.*'`
70+
echo "KUBO_VERSION=$V" >> "$GITHUB_ENV"
71+
GOHOSTOS=`go env GOHOSTOS`
72+
echo "GOHOSTOS=$GOHOSTOS" >> "$GITHUB_ENV"
73+
RELEASE=$(basename ${{ github.ref_name }})
74+
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
75+
echo PLUGIN_ARTIFACT=go-ds-s3-plugin_${RELEASE}_${GOHOSTOS}_amd64.tar.gz >> "$GITHUB_ENV"
76+
- name: Download artifact
77+
id: download
78+
uses: actions/download-artifact@v3
79+
with:
80+
name: ${{ env.PLUGIN_ARTIFACT }}
81+
- name: Download unpack Kubo
82+
run: |
83+
wget -nv https://dist.ipfs.tech/kubo/${{ env.KUBO_VERSION }}/kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
84+
tar -xf kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
85+
tar -xf go-ds-s3-plugin_*.tar.gz
86+
chmod +x kubo/ipfs
87+
- name: Initialize IPFS and copy plugin
88+
run: |
89+
export IPFS_PATH=$(pwd)/ipfs-config
90+
echo "IPFS_PATH=$IPFS_PATH" >> "$GITHUB_ENV"
91+
./kubo/ipfs init
92+
mkdir -p ipfs-config/plugins
93+
cp go-ds-s3-plugin/go-ds-s3-plugin ipfs-config/plugins/
94+
- name: Check IPFS works with the plugin
95+
run: ./kubo/ipfs version --all
96+
97+
release:
98+
name: "Release"
99+
needs: test-artifacts
100+
runs-on: ubuntu-20.04
101+
steps:
102+
- name: Download artifacts
103+
id: download
104+
uses: actions/download-artifact@v3
105+
- name: Extract release name from tag
106+
run: |
107+
RELEASE=$(basename ${{ github.ref_name }})
108+
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
109+
- name: Release
110+
uses: softprops/action-gh-release@v1
111+
if: startsWith(github.ref, 'refs/tags/go-ds-s3-plugin/v')
112+
with:
113+
files: go-ds-s3-plugin_*.tar.gz/*
114+
body: |
115+
This is a binary build of the go-ds-s3 Kubo plugin targeting Kubo version ${{ env.RELEASE }}.
116+
117+
To install, download the relevant asset for your platform, unpack the plugin file (`go-ds-s3-plugin`) and place it in `~/.ipfs/plugins`. MacOS users will need to compile Kubo themselves, as the official releases have no CGO support.
118+
119+
See the included README.md for more information.
120+
name: ${{ github.ref_name }}

‎Makefile‎

Lines changed: 11 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,15 @@
1-
# Force Go Modules
2-
GO111MODULE = on
1+
plugin:
2+
$(MAKE) -C go-ds-s3-plugin all
33

4-
GOCC ?= go
5-
GOFLAGS ?=
4+
install-plugin:
5+
$(MAKE) -C go-ds-s3-plugin install
66

7-
# If set, override the install location for plugins
8-
IPFS_PATH ?= $(HOME)/.ipfs
7+
dist-plugin:
8+
$(MAKE) -C go-ds-s3-plugin dist
99

10-
# If set, override the IPFS version to build against. This _modifies_ the local
11-
# go.mod/go.sum files and permanently sets this version.
12-
IPFS_VERSION ?= $(lastword $(shell $(GOCC) list -m github.com/ipfs/kubo))
10+
check:
11+
go vet ./...
12+
staticcheck --checks all ./...
13+
misspell -error -locale US .
1314

14-
# make reproducible
15-
ifneq ($(findstring /,$(IPFS_VERSION)),)
16-
# Locally built kubo
17-
GOFLAGS += -asmflags=all=-trimpath="$(GOPATH)" -gcflags=all=-trimpath="$(GOPATH)"
18-
else
19-
# Remote version of kubo (e.g. via `go get -trimpath` or official distribution)
20-
GOFLAGS += -trimpath
21-
endif
22-
23-
.PHONY: install build
24-
25-
go.mod: FORCE
26-
./set-target.sh $(IPFS_VERSION)
27-
28-
FORCE:
29-
30-
s3plugin.so: plugin/main/main.go go.mod
31-
CGO_ENABLED=1 $(GOCC) build $(GOFLAGS) -buildmode=plugin -o "$@" "$<"
32-
chmod +x "$@"
33-
34-
build: s3plugin.so
35-
@echo "Built against" $(IPFS_VERSION)
36-
37-
install: build
38-
install -Dm700 s3plugin.so "$(IPFS_PATH)/plugins/go-ds-s3.so"
15+
.PHONY: plugin install-plugin check

‎README.md‎

Lines changed: 28 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,38 @@ This is an implementation of the datastore interface backed by amazon s3.
44

55
**NOTE:** Plugins only work on Linux and MacOS at the moment. You can track the progress of this issue here: https://github.com/golang/go/issues/19282
66

7-
## Building and Installing
7+
## Quickstart
88

9-
You must build the plugin with the *exact* version of go used to build the kubo binary you will use it with. You can find the go version for kubo builds from dist.ipfs.io in the build-info file, e.g. https://dist.ipfs.io/kubo/v0.4.22/build-info or by running `ipfs version --all`.
9+
1. Grab a plugin release from the [releases](https://github.com/ipfs/go-ds-s3/releases) section matching your Kubo version and install the plugin file in `~/.ipfs/plugins`.
10+
2. Follow the instructions in the plugin's [README.md](go-ds-s3-plugin/README.md)
1011

11-
In addition to needing the exact version of go, you need to build the correct version of this plugin.
1212

13-
* To build against a released version of kubo, checkout the `release/v$VERSION` branch and build.
14-
* To build against a custom (local) build of kubo, run `make IPFS_VERSION=/path/to/kubo/source`.
13+
## Building and installing
1514

16-
You can then install it into your local IPFS repo by running `make install`.
15+
16+
The plugin can be manually built/installed for different versions of Kubo (starting with 0.23.0) with:
17+
18+
```
19+
git checkout go-ds-s3-plugin/v<kubo-version>
20+
make plugin
21+
make install-plugin
22+
```
23+
24+
## Updating to a new version
25+
26+
1. `go get` the Kubo release you want to build for. Make sure any other
27+
dependencies are aligned to what Kubo uses.
28+
2. `make install` and test.
29+
30+
31+
If you are building against dist-released versions of Kubo, you need to build using the same version of go that was used to build the release ([here](https://github.com/ipfs/distributions/blob/master/.tool-versions)).
32+
33+
If you are building against your own build of Kubo you must align your plugin to use it.
34+
35+
If you are updating this repo to produce a new version of the plugin:
36+
37+
1. Submit a PR so that integration tests run
38+
2. Make a new tag `go-ds-s3-plugin/v<kubo_version>` and push it. This will build and release the plugin prebuilt binaries.
1739

1840
## Bundling
1941

@@ -46,74 +68,6 @@ As go plugins can be finicky to correctly compile and install, you may want to c
4668
> make install
4769
```
4870

49-
## Detailed Installation
50-
51-
For a brand new ipfs instance (no data stored yet):
52-
53-
1. Copy s3plugin.so $IPFS_DIR/plugins/go-ds-s3.so (or run `make install` if you are installing locally).
54-
2. Run `ipfs init`.
55-
3. Edit $IPFS_DIR/config to include s3 details (see Configuration below).
56-
4. Overwrite `$IPFS_DIR/datastore_spec` as specified below (*Don't do this on an instance with existing data - it will be lost*).
57-
58-
### Configuration
59-
60-
The config file should include the following:
61-
```json
62-
{
63-
"Datastore": {
64-
...
65-
66-
"Spec": {
67-
"mounts": [
68-
{
69-
"child": {
70-
"type": "s3ds",
71-
"region": "us-east-1",
72-
"bucket": "$bucketname",
73-
"rootDirectory": "$bucketsubdirectory",
74-
"accessKey": "",
75-
"secretKey": ""
76-
},
77-
"mountpoint": "/blocks",
78-
"prefix": "s3.datastore",
79-
"type": "measure"
80-
},
81-
```
82-
83-
If the access and secret key are blank they will be loaded from the usual ~/.aws/.
84-
If you are on another S3 compatible provider, e.g. Linode, then your config should be:
85-
86-
```json
87-
{
88-
"Datastore": {
89-
...
90-
91-
"Spec": {
92-
"mounts": [
93-
{
94-
"child": {
95-
"type": "s3ds",
96-
"region": "us-east-1",
97-
"bucket": "$bucketname",
98-
"rootDirectory": "$bucketsubdirectory",
99-
"regionEndpoint": "us-east-1.linodeobjects.com",
100-
"accessKey": "",
101-
"secretKey": ""
102-
},
103-
"mountpoint": "/blocks",
104-
"prefix": "s3.datastore",
105-
"type": "measure"
106-
},
107-
```
108-
109-
If you are configuring a brand new ipfs instance without any data, you can overwrite the datastore_spec file with:
110-
111-
```
112-
{"mounts":[{"bucket":"$bucketname","mountpoint":"/blocks","region":"us-east-1","rootDirectory":"$bucketsubdirectory"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}
113-
```
114-
115-
Otherwise, you need to do a datastore migration.
116-
11771
## Contribute
11872

11973
Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-example-plugin/issues)!

‎go-ds-s3-plugin/Makefile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
SRC := $(shell find ../.. -type f -name '*.go')
2+
IPFS_PATH ?= $(HOME)/.ipfs
3+
4+
export CGO_ENABLED := 1
5+
6+
all: build
7+
8+
build: $(SRC)
9+
go build -buildmode=plugin -trimpath -o go-ds-s3-plugin
10+
11+
install: build
12+
mkdir -p "$(IPFS_PATH)/.ipfs/plugins"
13+
install -Dm700 go-ds-s3-plugin "$(IPFS_PATH)/plugins/go-ds-s3-plugin"
14+
15+
dist: build
16+
mkdir -p dist/go-ds-s3-plugin
17+
cp go-ds-s3-plugin README.md dist/go-ds-s3-plugin/
18+
chmod +x dist/go-ds-s3-plugin/go-ds-s3-plugin
19+
tar -C dist -zcf go-ds-s3-plugin.tar.gz go-ds-s3-plugin
20+
rm -rf go-ds-s3-plugin
21+
echo "Packaged as go-ds-s3-plugin.tar.gz"
22+
23+
.PHONY: install

0 commit comments

Comments
 (0)