add ipni node

Dockerfile

@@ -56,6 +56,9 @@ RUN go install github.com/ipld/go-car/cmd/car@latest \
 RUN go install github.com/LexLuthr/piece-server@latest \
  && cp $GOPATH/bin/piece-server /usr/local/bin/
 
+RUN go install github.com/ipni/storetheindex@latest \
+ && cp $GOPATH/bin/storetheindex /usr/local/bin/
+
 #####################################
 FROM ubuntu:22.04 AS curio-all-in-one
 
@@ -82,7 +85,8 @@ ENV FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters \
     LOTUS_MINER_PATH=/var/lib/lotus-miner \
     LOTUS_PATH=/var/lib/lotus \
     CURIO_REPO_PATH=/var/lib/curio \
-    CURIO_MK12_CLIENT_REPO=/var/lib/curio-client
+    CURIO_MK12_CLIENT_REPO=/var/lib/curio-client \
+    STORETHEINDEX_PATH=/var/lib/indexer
 
 # Copy binaries and scripts
 COPY --from=lotus-test /usr/local/bin/lotus /usr/local/bin/
@@ -93,19 +97,21 @@ COPY --from=curio-builder /opt/curio/curio /usr/local/bin/
 COPY --from=curio-builder /opt/curio/sptool /usr/local/bin/
 COPY --from=piece-server-builder /usr/local/bin/piece-server /usr/local/bin/
 COPY --from=piece-server-builder /usr/local/bin/car /usr/local/bin/
+COPY --from=piece-server-builder /usr/local/bin/storetheindex /usr/local/bin/
 
 # Set up directories and permissions
 RUN mkdir /var/tmp/filecoin-proof-parameters \
            /var/lib/lotus \
            /var/lib/lotus-miner \
            /var/lib/curio \
-           /var/lib/curio-client && \
-    chown fc: /var/tmp/filecoin-proof-parameters /var/lib/lotus /var/lib/lotus-miner /var/lib/curio /var/lib/curio-client
+           /var/lib/curio-client \
+           /var/lib/indexer && \
+    chown fc: /var/tmp/filecoin-proof-parameters /var/lib/lotus /var/lib/lotus-miner /var/lib/curio /var/lib/curio-client /var/lib/indexer
 
 # Define volumes
-VOLUME ["/var/tmp/filecoin-proof-parameters", "/var/lib/lotus", "/var/lib/lotus-miner", "/var/lib/curio", "/var/lib/curio-client"]
+VOLUME ["/var/tmp/filecoin-proof-parameters", "/var/lib/lotus", "/var/lib/lotus-miner", "/var/lib/curio", "/var/lib/curio-client", "/var/lib/indexer"]
 
 # Expose necessary ports
-EXPOSE 1234 2345 12300 4701 32100 12310 12320
+EXPOSE 1234 2345 12300 4701 32100 12310 12320 3000 3001 3002 3003
 
 CMD ["/bin/bash"]

Makefile

@@ -324,7 +324,12 @@ docker/piece-server:
 		--build-arg BUILD_VERSION=dev .
 .PHONY: docker/piece-server
 
-docker/devnet: $(lotus_build_cmd) docker/curio-all-in-one docker/lotus docker/lotus-miner docker/curio docker/piece-server
+docker/indexer:
+	cd docker/indexer && DOCKER_BUILDKIT=1 $(curio_docker_build_cmd) -t $(curio_docker_user)/indexer-dev:dev \
+		--build-arg BUILD_VERSION=dev .
+.PHONY: docker/indexer
+
+docker/devnet: $(lotus_build_cmd) docker/curio-all-in-one docker/lotus docker/lotus-miner docker/curio docker/piece-server docker/indexer
 .PHONY: docker/devnet
 
 devnet/up:

docker/.env

@@ -3,4 +3,5 @@ LOTUS_IMAGE=${DOCKER_USER}/lotus-dev:dev
 LOTUS_MINER_IMAGE=${DOCKER_USER}/lotus-miner-dev:dev
 CURIO_IMAGE=${DOCKER_USER}/curio-dev:dev
 PIECE_SERVER_IMAGE=${DOCKER_USER}/piece-server-dev:dev
+INDEXER_IMAGE=${DOCKER_USER}/indexer-dev:dev
 FIL_PROOFS_PARAMETER_CACHE=${HOME}/.cache/filecoin-proof-parameters

docker/curio/entrypoint.sh

@@ -47,6 +47,9 @@ if [ ! -f $CURIO_REPO_PATH/.init.curio ]; then
 
   [Market]
     [Market.StorageMarketConfig]
+      [Market.StorageMarketConfig.IPNI]
+        DirectAnnounceURLs = ["http://indexer:3001"]
+        ServiceURL = ["http://indexer:3000"]
       [Market.StorageMarketConfig.MK12]
           ExpectedPoRepSealDuration = "0h1m0s"
           ExpectedSnapSealDuration = "0h1m0s"

docker/docker-compose.yaml

@@ -121,3 +121,26 @@ services:
     networks:
       curio-net:
         ipv4_address: 172.20.0.6
+
+  indexer:
+    container_name: indexer
+    image: ${INDEXER_IMAGE}
+    init: true
+    ports:
+      - "3000:3000"
+      - "3001:3001"
+      - "3002:3002"
+      - "3003:3003"
+    environment:
+      - STORETHEINDEX_PATH=/var/lib/indexer
+    restart: unless-stopped
+    logging: *default-logging
+    volumes:
+      - ./data/indexer:/var/lib/indexer:rw
+      - ./data/curio:/var/lib/curio:ro
+      - ./data/lotus:/var/lib/lotus:ro
+      - ./data/lotus-miner:/var/lib/lotus-miner:ro
+      - ${FIL_PROOFS_PARAMETER_CACHE}:/var/tmp/filecoin-proof-parameters:rw
+    networks:
+      curio-net:
+        ipv4_address: 172.20.0.7

docker/indexer/Dockerfile

@@ -0,0 +1,30 @@
+ARG CURIO_TEST_IMAGE=curio/curio-all-in-one:latest
+#############################################################################
+FROM ${CURIO_TEST_IMAGE}
+
+ARG BUILD_VERSION=0.1
+
+LABEL org.opencontainers.image.version=$BUILD_VERSION \
+      org.opencontainers.image.authors="Curio Dev Team" \
+      name="indexer-dev" \
+      maintainer="Curio Dev Team" \
+      vendor="Curio Dev Team" \
+      version=$BUILD_VERSION \
+      release=$BUILD_VERSION \
+      summary="This image is used to host the curio dev service" \
+      description="This image is used to host the curio dev service"
+
+EXPOSE 12320
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/genesis
+VOLUME /var/lib/builtin-actors
+
+WORKDIR /app
+RUN mkdir -p /app
+
+COPY entrypoint.sh /app
+
+USER root
+
+ENTRYPOINT ["./entrypoint.sh"]

docker/indexer/entrypoint.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -e
+STORETHEINDEX_PATH=$STORETHEINDEX_PATH
+echo Wait for lotus is ready ...
+lotus wait-api
+head=0
+# Loop until the head is greater than 9
+while [[ $head -le 9 ]]; do
+    head=$(lotus chain list | awk '{print $1}' | awk -F':' '{print $1}' | tail -1)
+    if [[ $head -le 9 ]]; then
+        echo "Current head: $head, which is not greater than 9. Waiting..."
+        sleep 1  # Wait for 4 seconds before checking again
+    else
+        echo "The head is now greater than 9: $head"
+    fi
+done
+
+myip=`nslookup piece-server | grep -v "#" | grep Address | awk '{print $2}'`
+
+echo $LOTUS_PATH
+
+# Init Indexer
+ if [ ! -f $STORETHEINDEX_PATH/.init ]; then
+  echo "Initialising indexer"
+  storetheindex init --listen-admin /ip4/0.0.0.0/tcp/3002 --listen-finder /ip4/0.0.0.0/tcp/3000 --listen-ingest /ip4/0.0.0.0/tcp/3001 --cachesize 10485760 --no-bootstrap
+  touch $STORETHEINDEX_PATH/.init
+fi
+
+# Start piece-server in HTTP mode (no HTTPS)
+echo "Starting indexer"
+storetheindex daemon

docker/piece-server/sample/random-deal.sh

@@ -17,8 +17,8 @@ miner_actor=$(lotus state list-miners | grep -v t01000)
 
 ###################################################################################
 printf "${ci}sptool --actor t01000 toolbox mk12-client deal --verified=false --provider=$miner_actor \
---http-url=http://demo-http-server/$FILE \
+--http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID \
 --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \
 --payload-cid=$PAYLOAD_CID --storage-price 20000000000\n\n${cn}"
 
-sptool --actor t01000 toolbox mk12-client deal --verified=false --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --payload-cid=$PAYLOAD_CID --storage-price 20000000000
+sptool --actor t01000 toolbox mk12-client deal --verified=false --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --payload-cid=$PAYLOAD_CID --storage-price 20000000000

market/ipni/ipni-provider/ipni-provider.go

@@ -50,7 +50,7 @@ const IPNIPath = "/ipni/v1/ad/"
 // publishInterval represents the time interval between each publishing operation.
 // It is set to 10 minutes.
 const publishInterval = 10 * time.Minute
-const publishProviderSpacing = 5 * time.Minute
+const publishProviderSpacing = 10 * time.Second
 
 var (
 	log = logging.Logger("ipni-provider")
@@ -161,7 +161,7 @@ func NewProvider(d *deps.Deps) (*Provider, error) {
 
 		if build.BuildType != build.BuildMainnet && build.BuildType != build.BuildCalibnet {
 			ls := strings.Split(d.Cfg.HTTP.ListenAddress, ":")
-			u, err = url.Parse(fmt.Sprintf("https://%s:%s", d.Cfg.HTTP.DomainName, ls[1]))
+			u, err = url.Parse(fmt.Sprintf("http://%s:%s", d.Cfg.HTTP.DomainName, ls[1]))
 			if err != nil {
 				return nil, xerrors.Errorf("parsing announce address domain: %w", err)
 			}
@@ -476,18 +476,23 @@ func RemoveCidContact(slice []*url.URL) []*url.URL {
 
 // StartPublishing starts a poller which publishes the head for each provider every 10 minutes.
 func (p *Provider) StartPublishing(ctx context.Context) {
+	// A poller which publishes head for each provider
+	// every 10 minutes
+	ticker := time.NewTicker(publishInterval)
+
 	// Do not publish for any network except mainnet to cid.contact
 	if build.BuildType != build.BuildMainnet {
 		// Check if there are any other URLs except cid.contact
 		urls := RemoveCidContact(p.announceURLs)
 		if len(urls) == 0 {
+			log.Warn("Not starting IPNI provider publishing as there are no other URLs except cid.contact for testnet build")
 			return
 		}
+		if build.BuildType != build.BuildCalibnet {
+			ticker = time.NewTicker(time.Second * 10)
+		}
 	}
 
-	// A poller which publishes head for each provider
-	// every 10 minutes
-	ticker := time.NewTicker(publishInterval)
 	go func() {
 		for {
 			select {
@@ -530,13 +535,14 @@ func (p *Provider) publishHead(ctx context.Context) {
 	var i int
 	for provider := range p.keys {
 		if i > 0 {
-			time.Sleep(publishProviderSpacing)
+			p.publishProviderSpacingWait()
 		}
 		c, err := p.getHeadCID(ctx, provider)
 		if err != nil {
 			log.Errorw("failed to get head CID", "provider", provider, "error", err)
 			continue
 		}
+		log.Infow("Publishing head for provider", "provider", provider, "cid", c.String())
 		err = p.publishhttp(ctx, c, provider)
 		if err != nil {
 			log.Errorw("failed to publish head for provide", "provider", provider, "error", err)
@@ -546,6 +552,14 @@ func (p *Provider) publishHead(ctx context.Context) {
 	}
 }
 
+func (p *Provider) publishProviderSpacingWait() {
+	if build.BuildType != build.BuildMainnet && build.BuildType != build.BuildCalibnet {
+		time.Sleep(time.Second)
+		return
+	}
+	time.Sleep(publishProviderSpacing)
+}
+
 // publishhttp sends an HTTP announce message for the given advertisement CID and peer ID.
 // It creates an HTTP announce sender using the provided announce URLs and the private key of the peer.
 // It obtains the HTTP addresses for the peer and sends the announce message to those addresses.

web/static/pages/market/deal-pipelines.mjs

@@ -288,7 +288,7 @@ class DealPipelines extends LitElement {
             return '(#######..) Indexed';
         } else if (deal.sector) {
             return '(######...) Sealing And Indexing';
-        } else if (deal.after_find_deal) {
+        } else if (deal.after_find_deal && !deal.sector) {
             return '(#####....) Assigning Sector';
         } else if (deal.after_psd) {
             return '(####.....) Waiting for DealID';