feat: devnet with market and deal client (#379)

* devnet with market

* update circleci check

* please the linter gods

* add ipni node

* fix PieceLocator url

.circleci/config.yml

@@ -7,12 +7,12 @@ executors:
   golang:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.22.8
+      - image: cimg/go:1.23
     resource_class: medium+
   golang-2xl:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.22.8
+      - image: cimg/go:1.23
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -90,12 +90,39 @@ commands:
   check-go-version:
     steps:
       - run: |
-          v=`go version | { read _ _ v _; echo ${v#go}; }`
-          if [[ $v != `cat GO_VERSION_MIN` ]]; then
-            echo "GO_VERSION_MIN file does not match the go version being used."
-            echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
-            exit 1
+          v=$(go version | { read _ _ v _; echo ${v#go}; })
+          go_min=$(cat GO_VERSION_MIN | tr -d ' \n')  # Remove spaces/newlines
+          
+          # Extract major, minor, and full versions
+          v_major=$(echo "$v" | cut -d. -f1)
+          v_major_minor=$(echo "$v" | cut -d. -f1,2)
+          v_full=$(echo "$v" | cut -d. -f1,2,3)
+
+          go_min_major=$(echo "$go_min" | cut -d. -f1)
+          go_min_major_minor=$(echo "$go_min" | cut -d. -f1,2)
+          go_min_full=$(echo "$go_min" | cut -d. -f1,2,3)
+
+          # Ensure major version is allowed
+          if [[ "$v_major" != "1" && "$v_major" != "23" && "$v_major" != "6" ]]; then
+          echo "Go version mismatch! Allowed major versions: 1, 23, 6, but found Go $v."
+          exit 1
+          fi
+
+          # If GO_VERSION_MIN specifies a patch version, check full match
+          if [[ "$go_min" == *.*.* ]]; then
+          if [[ "$v_full" != "$go_min_full" ]]; then
+          echo "Go version mismatch! Expected Go $go_min but found $v."
+          exit 1
           fi
+          # If GO_VERSION_MIN specifies only major.minor, check that exactly
+          elif [[ "$go_min" == *.* ]]; then
+          if [[ "$v_major_minor" != "$go_min_major_minor" ]]; then
+          echo "Go version mismatch! Expected Go $go_min but found $v."
+          exit 1
+          fi
+          fi
+
+          echo "Go version matches: $v"
 
 jobs:
   build-debug:
@@ -189,7 +216,7 @@ jobs:
         default: unit
         description: Test suite name to report to CircleCI.
     docker:
-      - image: cimg/go:1.22
+      - image: cimg/go:1.23
         environment:
           CURIO_HARMONYDB_HOSTS: yugabyte
           LOTUS_HARMONYDB_HOSTS: yugabyte
@@ -244,7 +271,7 @@ jobs:
         description: |
           Arguments to pass to golangci-lint
     docker:
-      - image: cimg/go:1.21
+      - image: cimg/go:1.23
     resource_class: medium+
     steps:
       - checkout

.github/image/Dockerfile

@@ -1,6 +1,6 @@
 ## This image is used to github action runner for this repo
 # Use the official Golang image as the base image
-FROM golang:1.22-bullseye AS builder
+FROM golang:1.23-bullseye AS builder
 
 # Set the working directory inside the container
 WORKDIR /app
@@ -23,7 +23,7 @@ RUN git submodule update --init
 RUN go mod download
 
 # Stage 2: Install Lotus binary
-FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.28.1-devnet AS lotus-test
+FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.32.0-rc1-devnet AS lotus-test
 
 # Stage 3: Build the final image
 FROM myoung34/github-runner AS curio-github-runner

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
       - main
 
 env:
-  GO_VERSION: 1.22.8
+  GO_VERSION: 1.23
 
 jobs:
   ci-lint:
@@ -257,7 +257,7 @@ jobs:
 
       - name: Install golangci-lint
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.58.1
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.60.1
         shell: bash
 
       - name: Lint

Dockerfile

@@ -1,8 +1,8 @@
 #####################################
 ARG LOTUS_TEST_IMAGE=curio/lotus-all-in-one:latest
 FROM ${LOTUS_TEST_IMAGE} AS lotus-test
-FROM golang:1.22.8-bullseye AS curio-builder
-LABEL Maintainer = "Curio Development Team"
+FROM golang:1.23-bullseye AS curio-builder
+LABEL Maintainer="Curio Development Team"
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
 
@@ -47,6 +47,18 @@ ARG CURIO_TAGS=""
 
 RUN make build
 
+####################################
+FROM golang:1.23-bullseye AS piece-server-builder
+
+RUN go install github.com/ipld/go-car/cmd/car@latest \
+ && cp $GOPATH/bin/car /usr/local/bin/
+
+RUN go install github.com/LexLuthr/piece-server@latest \
+ && cp $GOPATH/bin/piece-server /usr/local/bin/
+
+RUN go install github.com/ipni/storetheindex@latest \
+ && cp $GOPATH/bin/storetheindex /usr/local/bin/
+
 #####################################
 FROM ubuntu:22.04 AS curio-all-in-one
 
@@ -69,10 +81,12 @@ RUN useradd -r -u 532 -U fc && \
     echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
 
 # Environment setup
-ENV FILECOIN_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters \
+ENV FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters \
     LOTUS_MINER_PATH=/var/lib/lotus-miner \
     LOTUS_PATH=/var/lib/lotus \
-    CURIO_REPO_PATH=/var/lib/curio
+    CURIO_REPO_PATH=/var/lib/curio \
+    CURIO_MK12_CLIENT_REPO=/var/lib/curio-client \
+    STORETHEINDEX_PATH=/var/lib/indexer
 
 # Copy binaries and scripts
 COPY --from=lotus-test /usr/local/bin/lotus /usr/local/bin/
@@ -81,18 +95,23 @@ COPY --from=lotus-test /usr/local/bin/lotus-shed /usr/local/bin/
 COPY --from=lotus-test /usr/local/bin/lotus-miner /usr/local/bin/
 COPY --from=curio-builder /opt/curio/curio /usr/local/bin/
 COPY --from=curio-builder /opt/curio/sptool /usr/local/bin/
+COPY --from=piece-server-builder /usr/local/bin/piece-server /usr/local/bin/
+COPY --from=piece-server-builder /usr/local/bin/car /usr/local/bin/
+COPY --from=piece-server-builder /usr/local/bin/storetheindex /usr/local/bin/
 
 # Set up directories and permissions
 RUN mkdir /var/tmp/filecoin-proof-parameters \
            /var/lib/lotus \
            /var/lib/lotus-miner \
-           /var/lib/curio && \
-    chown fc: /var/tmp/filecoin-proof-parameters /var/lib/lotus /var/lib/lotus-miner /var/lib/curio
+           /var/lib/curio \
+           /var/lib/curio-client \
+           /var/lib/indexer && \
+    chown fc: /var/tmp/filecoin-proof-parameters /var/lib/lotus /var/lib/lotus-miner /var/lib/curio /var/lib/curio-client /var/lib/indexer
 
 # Define volumes
-VOLUME ["/var/tmp/filecoin-proof-parameters", "/var/lib/lotus", "/var/lib/lotus-miner", "/var/lib/curio"]
+VOLUME ["/var/tmp/filecoin-proof-parameters", "/var/lib/lotus", "/var/lib/lotus-miner", "/var/lib/curio", "/var/lib/curio-client", "/var/lib/indexer"]
 
 # Expose necessary ports
-EXPOSE 1234 2345 12300 4701 32100
+EXPOSE 1234 2345 12300 4701 32100 12310 12320 3000 3001 3002 3003
 
 CMD ["/bin/bash"]

GO_VERSION_MIN

@@ -1 +1 @@
-1.22.8
+1.23

Makefile

@@ -319,7 +319,17 @@ docker/curio:
 		--build-arg BUILD_VERSION=dev .
 .PHONY: docker/curio
 
-docker/devnet: $(lotus_build_cmd) docker/curio-all-in-one docker/lotus docker/lotus-miner docker/curio docker/yugabyte
+docker/piece-server:
+	cd docker/piece-server && DOCKER_BUILDKIT=1 $(curio_docker_build_cmd) -t $(curio_docker_user)/piece-server-dev:dev \
+		--build-arg BUILD_VERSION=dev .
+.PHONY: docker/piece-server
+
+docker/indexer:
+	cd docker/indexer && DOCKER_BUILDKIT=1 $(curio_docker_build_cmd) -t $(curio_docker_user)/indexer-dev:dev \
+		--build-arg BUILD_VERSION=dev .
+.PHONY: docker/indexer
+
+docker/devnet: $(lotus_build_cmd) docker/curio-all-in-one docker/lotus docker/lotus-miner docker/curio docker/piece-server docker/indexer
 .PHONY: docker/devnet
 
 devnet/up:

cmd/sptool/main.go

@@ -90,5 +90,6 @@ var toolboxCmd = &cli.Command{
 	Usage: "some tools to fix some problems",
 	Subcommands: []*cli.Command{
 		sparkCmd,
+		mk12Clientcmd,
 	},
 }