feat: libstorage support in build system (#7332)

* feat: support libstorage in the build system

* build: make sure linting includes logos-storage

* build: tweaking native build of libstorage that avoids calling make update (allowing to build on RYZEN processors)

* build: removes redundant scripts

* build: removes printing libstorage version on nix env

* test: more uniform naming (USE_LOGOS_STORAGE)

* build: bumps vendor hash

* test: include logosstorage tests in the coverage on the CI

* build: remove passing lib paths to the generate target

Co-authored-by: Jakub <jakub@status.im>

* build: Update Dockerfile - remove indentation

Co-authored-by: Jakub <jakub@status.im>

* build: remove redundant check in Makefile

Co-authored-by: Jakub <jakub@status.im>

* build: remove noise from Makefile

Co-authored-by: Jakub <jakub@status.im>

* build: remove redundant IFs from Makefile

* build: adds README to be used when printing logosstorage help messages

* build: move some Makefile vars to be better visible in context

---------

Co-authored-by: Jakub <jakub@status.im>

Author: marcinczenko

.github/workflows/pr.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Setup Protocol Buffer Compiler
         uses: arduino/setup-protoc@v3
         with:
-          version: "29.3"
+          version: '29.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Cache go-generate-fast
@@ -57,11 +57,11 @@ jobs:
         uses: golangci/golangci-lint-action@v8
         with:
           version: v2.3.1
-          args: --build-tags=gowaku_no_rln,lint
+          args: --build-tags=gowaku_no_rln,lint,use_logos_storage
 
       - name: lint-panics
         run: |
-          make lint-panics
+          make lint-panics USE_LOGOS_STORAGE=true
 
       - name: go mod tidy
         run: go mod tidy && git diff --exit-code

.gitignore

@@ -121,5 +121,8 @@ go.work.sum
 # Token deployment cache files
 tests-functional/snt_addresses.json*
 
-# Codex AI (chatGPT)
+# libstorage
+libs
+
+# CODEX (ChatGPT plugin)
 .codex

.vscode/settings.json

@@ -1,9 +1,11 @@
 {
-  "go.testTags": "gowaku_skip_migrations,gowaku_no_rln",
+  "go.testTags": "gowaku_skip_migrations,gowaku_no_rln,use_logos_storage",
   "cSpell.words": ["unmarshalling"],
   "gopls": {
     "formatting.local": "github.com/status-im/status-go",
-    "build.buildFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"]
+    "build.buildFlags": [
+      "-tags=gowaku_skip_migrations,gowaku_no_rln,use_logos_storage"
+    ]
   },
   // format all files on save if a formatter is available
   "editor.formatOnSave": true,
@@ -14,9 +16,13 @@
     "CGO_ENABLED": "1"
   },
   "go.testEnvVars": {
-    "CGO_CFLAGS": "-I${workspaceFolder}/../nim-sds/library",
-    "CGO_LDFLAGS": "-L${workspaceFolder}/../nim-sds/build -lsds -Wl,-rpath,${workspaceFolder}/../nim-sds/build"
+    "CGO_CFLAGS": "-I${workspaceFolder}/../logos-storage-nim/library -I${workspaceFolder}/../nim-sds/library",
+    "CGO_LDFLAGS": "-L${workspaceFolder}/../logos-storage-nim/build -lstorage -Wl,-rpath,${workspaceFolder}/../logos-storage-nim/build -L${workspaceFolder}/../nim-sds/build -lsds"
   },
-  "go.buildFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"],
-  "go.testFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"]
+  "go.buildFlags": [
+    "-tags=gowaku_skip_migrations,gowaku_no_rln,use_logos_storage"
+  ],
+  "go.testFlags": [
+    "-tags=gowaku_skip_migrations,gowaku_no_rln,use_logos_storage"
+  ]
 }

Dockerfile

@@ -1,12 +1,12 @@
 # Build status-go in a Go builder container
-FROM golang:1.24.7-bookworm AS builder
+FROM golang:1.24.7-trixie AS builder
 
 # Set environment variables to use Clang
 ENV CC=clang
 ENV CXX=clang++
 
 RUN apt-get update \
-    && apt-get install -y git bash make llvm clang protobuf-compiler build-essential pkg-config curl xz-utils jq \
+    && apt-get install -y git bash make cmake llvm clang protobuf-compiler build-essential pkg-config curl xz-utils jq unzip \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -15,16 +15,16 @@ ARG NIM_VERSION=2.2.4
 RUN set -eu && \
     DPKG_ARCH="$(dpkg --print-architecture)" && \
     case "$DPKG_ARCH" in \
-        amd64) NIM_ARCH="x64" ;; \
-        arm64) NIM_ARCH="arm64" ;; \
-        *) echo "Unsupported architecture: $DPKG_ARCH" >&2; exit 1 ;; \
+    amd64) NIM_ARCH="x64" ;; \
+    arm64) NIM_ARCH="arm64" ;; \
+    *) echo "Unsupported architecture: $DPKG_ARCH" >&2; exit 1 ;; \
     esac && \
     NIM_URL=$(curl -sSf https://nim-lang.org/releases.json \
-        | jq -r --arg ver "$NIM_VERSION" --arg arch "$NIM_ARCH" \
-            '.[$ver]["linux_" + $arch].github_url // empty') && \
+    | jq -r --arg ver "$NIM_VERSION" --arg arch "$NIM_ARCH" \
+    '.[$ver]["linux_" + $arch].github_url // empty') && \
     if [ -z "$NIM_URL" ]; then \
-        echo "ERROR: No download URL found for Nim $NIM_VERSION linux_$NIM_ARCH" >&2; \
-        exit 1; \
+    echo "ERROR: No download URL found for Nim $NIM_VERSION linux_$NIM_ARCH" >&2; \
+    exit 1; \
     fi && \
     curl -sSfL "$NIM_URL" -o /tmp/nim.tar.xz && \
     mkdir -p /opt/nim && \
@@ -37,6 +37,7 @@ ENV PATH="/opt/nim/bin:${PATH}"
 ARG build_tags='gowaku_no_rln'
 ARG build_flags=''
 ARG build_target='cmd'
+ARG use_logos_storage='false'
 
 RUN mkdir -p /go/src/github.com/status-im/status-go
 WORKDIR /go/src/github.com/status-im/status-go
@@ -50,20 +51,27 @@ ARG cache_id='local'
 ARG enable_go_cache=true
 
 RUN if [ "$enable_go_cache" = "true" ]; then \
-      go env -w GOCACHE=/root/.cache/go-build; \
+    go env -w GOCACHE=/root/.cache/go-build; \
     fi
 RUN --mount=type=cache,target="/root/.cache/go-build",id=statusgo-build-$cache_id \
-    make $build_target BUILD_TAGS="$build_tags" BUILD_FLAGS="$build_flags"
+    make $build_target BUILD_TAGS="$build_tags" BUILD_FLAGS="$build_flags" USE_LOGOS_STORAGE="$use_logos_storage"
+
+# Stage runtime shared libraries required by built binaries.
+RUN mkdir -p /tmp/status-runtime-libs \
+    && cp /go/src/github.com/status-im/nim-sds/build/libsds.so /tmp/status-runtime-libs/ \
+    && if [ -f /go/src/github.com/status-im/logos-storage-nim/build/libstorage.so ]; then \
+    cp /go/src/github.com/status-im/logos-storage-nim/build/libstorage.so /tmp/status-runtime-libs/; \
+    fi
 
 # Copy binaries to the second image
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
 LABEL maintainer="support@status.im"
 LABEL source="https://github.com/status-im/status-go"
 LABEL description="status-go is an underlying part of Status - a browser, messenger, and gateway to a decentralized world."
 
 RUN apt-get update \
- && apt-get install -y ca-certificates bash curl python3 \
+ && apt-get install -y ca-certificates bash curl python3 libgomp1 \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
@@ -73,10 +81,10 @@ RUN mkdir -p /static/configs
 COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/status-backend /usr/local/bin/
 COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/push-notification-server /usr/local/bin/
 COPY --from=builder /go/src/github.com/status-im/status-go/tests-functional/waku_configs/* /static/configs/
-COPY --from=builder /go/src/github.com/status-im/nim-sds/build/libsds.so /usr/local/lib/
+COPY --from=builder /tmp/status-runtime-libs/ /usr/local/lib/
 
 ENV LD_LIBRARY_PATH=/usr/local/lib/
 
 EXPOSE 8080 8545 30303 30303/udp 30304/udp
 
-CMD ["status-backend", "--help"]
+CMD ["status-backend", "--help"]

Makefile

@@ -1,6 +1,8 @@
 .PHONY: statusgo all test clean help
 .PHONY: statusgo-ios-library statusgo-android-library
 .PHONY: build-libsds clean-libsds rebuild-libsds
+.PHONY: clone-storage build-storage clean-storage test-storage
+.PHONY: storage-help
 
 # Clear any GOROOT set outside of the Nix shell
 export GOROOT=
@@ -133,6 +135,70 @@ LIBSDS ?= $(NIM_SDS_LIB_DIR)/libsds.$(LIB_EXT)
 CGO_CFLAGS+=-I$(NIM_SDS_INC_DIR)
 CGO_LDFLAGS+=-L$(NIM_SDS_LIB_DIR) -lsds
 
+# `logos-storage` variables (opt-in)
+USE_LOGOS_STORAGE ?= false
+LOGOS_STORAGE_VERSION ?= 3c09f008bb5266a669fd19f18368f9e8b861b664
+LOGOS_STORAGE_SOURCE_DIR ?= $(GIT_ROOT)/../logos-storage-nim
+
+# Option 1: Provide LOGOS_STORAGE_SOURCE_DIR. Make clones it if missing.
+# Option 2: Provide LOGOS_STORAGE_LIB_DIR and LOGOS_STORAGE_INC_DIR.
+ifdef LOGOS_STORAGE_LIB_DIR
+ifdef LOGOS_STORAGE_INC_DIR
+    LOGOS_STORAGE_BUILD_FROM_SOURCE := false
+else
+    $(error LOGOS_STORAGE_INC_DIR must be provided when LOGOS_STORAGE_LIB_DIR is set)
+endif
+else
+    LOGOS_STORAGE_LIB_DIR := $(LOGOS_STORAGE_SOURCE_DIR)/build
+    LOGOS_STORAGE_INC_DIR := $(LOGOS_STORAGE_SOURCE_DIR)/library
+    LOGOS_STORAGE_BUILD_FROM_SOURCE := true
+endif
+
+LIBSTORAGE ?= $(LOGOS_STORAGE_LIB_DIR)/libstorage.$(LIB_EXT)
+
+RUNTIME_LIB_DIRS := $(NIM_SDS_LIB_DIR)
+LOGOS_STORAGE_BUILD_DEPS :=
+ifeq ($(USE_LOGOS_STORAGE),true)
+	BUILD_TAGS += use_logos_storage
+	CGO_CFLAGS += -I$(LOGOS_STORAGE_INC_DIR)
+	CGO_LDFLAGS += -L$(LOGOS_STORAGE_LIB_DIR) -lstorage -Wl,-rpath,$(LOGOS_STORAGE_LIB_DIR)
+	RUNTIME_LIB_DIRS := $(LOGOS_STORAGE_LIB_DIR):$(RUNTIME_LIB_DIRS)
+	LOGOS_STORAGE_BUILD_DEPS += $(LIBSTORAGE)
+endif
+
+clone-storage: ##@build Clone or update logos-storage-nim
+ifeq ($(LOGOS_STORAGE_BUILD_FROM_SOURCE),true)
+	@echo "Cloning or updating logos-storage-nim ..."
+	if [ ! -d "$(LOGOS_STORAGE_SOURCE_DIR)" ]; then \
+		git clone --recurse-submodules https://github.com/logos-storage/logos-storage-nim.git $(LOGOS_STORAGE_SOURCE_DIR); \
+	else \
+		cd $(LOGOS_STORAGE_SOURCE_DIR) && git fetch --tags; \
+	fi
+	cd $(LOGOS_STORAGE_SOURCE_DIR) && git checkout $(LOGOS_STORAGE_VERSION) && git submodule update --init --recursive
+endif
+
+$(LIBSTORAGE): clone-storage
+ifeq ($(LOGOS_STORAGE_BUILD_FROM_SOURCE),true)
+	@echo "Building logos-storage: $(LIBSTORAGE)"
+	$(MAKE) -C $(LOGOS_STORAGE_SOURCE_DIR) libstorage
+endif
+
+build-storage: $(LIBSTORAGE)
+
+clean-storage: ##@other Remove built native libstorage artifacts
+ifeq ($(LOGOS_STORAGE_BUILD_FROM_SOURCE),true)
+	@rm -f "$(LOGOS_STORAGE_LIB_DIR)"/libstorage.*
+endif
+
+test-storage: build-storage $(LIBSDS) generate ##@tests Run logosstorage package tests via gotestsum
+	LD_LIBRARY_PATH="$(LOGOS_STORAGE_LIB_DIR):$(RUNTIME_LIB_DIRS)" \
+	CGO_LDFLAGS="$(CGO_LDFLAGS) -L$(LOGOS_STORAGE_LIB_DIR) -lstorage -Wl,-rpath,$(LOGOS_STORAGE_LIB_DIR)" \
+	CGO_CFLAGS="$(CGO_CFLAGS) -I$(LOGOS_STORAGE_INC_DIR)" \
+	gotestsum --packages="./services/logosstorage" -f testname -- -count 1 -tags "use_logos_storage $(BUILD_TAGS) gowaku_skip_migrations"
+
+storage-help: ##@build Show logos-storage build/test toggles and env vars
+	@cat services/logosstorage/README.md
+
 # mbedtls configuration for go-sqlcipher
 ifeq ($(detected_OS),Windows)
  # On Windows, use portable C implementations and add -Werror=implicit-function-declaration workaround
@@ -211,7 +277,7 @@ nix-purge: ##@nix Completely remove Nix setup, including /nix directory
 all: $(GO_CMD_NAMES)
 
 .PHONY: $(GO_CMD_NAMES) $(GO_CMD_PATHS) $(GO_CMD_BUILDS)
-$(GO_CMD_BUILDS): generate $(LIBSDS)
+$(GO_CMD_BUILDS): generate $(LOGOS_STORAGE_BUILD_DEPS) $(LIBSDS)
 $(GO_CMD_BUILDS): ##@build Build any Go project from cmd folder
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
 	go build -v \
@@ -400,14 +466,14 @@ clean-generated: ##@generate Remove orphaned generated files
 	  	echo "Skipping cleanup of generated files"; \
 	fi
 
-generate: PACKAGES ?= $$(go list -e ./... | grep -v "/contracts/")
 generate: PACKAGES ?= $$(go list -e ./... | grep -v "/contracts/")
 generate: GO_GENERATE_CMD ?= go tool go-generate-fast
 generate: export GO_GENERATE_FAST_DEBUG ?= false
 generate: export GO_GENERATE_FAST_RECACHE ?= false
 generate: clean-generated
 generate: ##@ Run generate for all given packages using go-generate-fast, fallback to `go generate` (e.g. for docker)
 	@GOROOT=$$(go env GOROOT) $(GO_GENERATE_CMD) $(PACKAGES)
+	@go generate -tags "use_logos_storage $(BUILD_TAGS)" ./services/logosstorage
 
 generate-contracts:
 	go generate ./contracts
@@ -433,7 +499,7 @@ docker-test: ##@tests Run tests in a docker container with golang.
 
 test: test-unit ##@tests Run basic, short tests during development
 
-test-unit-prep: $(LIBSDS)
+test-unit-prep: $(LOGOS_STORAGE_BUILD_DEPS) $(LIBSDS)
 test-unit-prep: generate
 test-unit-prep: export BUILD_TAGS ?=
 test-unit-prep: export UNIT_TEST_DRY_RUN ?= false
@@ -444,17 +510,17 @@ test-unit-prep: export UNIT_TEST_REPORT_CODECOV ?= false
 
 test-unit: test-unit-prep
 test-unit: export UNIT_TEST_RERUN_FAILS ?= true
-test-unit: export UNIT_TEST_PACKAGES ?= $(call sh, go list ./... | \
+test-unit: export UNIT_TEST_PACKAGES ?= $(call sh, go list -tags '$(BUILD_TAGS)' ./... | \
 	grep -v /t/e2e | \
 	grep -v /t/benchmarks | \
 	grep -v /transactions/fake | \
 	grep -v /tests-unit-network)
 test-unit: ##@tests Run unit and integration tests
-	LD_LIBRARY_PATH="$(NIM_SDS_LIB_DIR)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
+	LD_LIBRARY_PATH="$(RUNTIME_LIB_DIRS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
 	./scripts/run_unit_tests.sh
 
 test-single: test-unit-prep
-	LD_LIBRARY_PATH="$(NIM_SDS_LIB_DIR)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
+	LD_LIBRARY_PATH="$(RUNTIME_LIB_DIRS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
 	go test -v $(PKG) -testify.m $(TEST)
 
 test-unit-network: test-unit-prep
@@ -469,15 +535,20 @@ test-unit-race: test-unit ##@tests Run unit and integration tests with -race fla
 test-functional: generate
 test-functional: export FUNCTIONAL_TESTS_DOCKER_UID ?= $(call sh, id -u)
 test-functional: export FUNCTIONAL_TESTS_REPORT_CODECOV ?= false
+test-functional: export USE_LOGOS_STORAGE := $(USE_LOGOS_STORAGE)
 test-functional:
 	@./scripts/run_functional_tests.sh
 
 benchmark: export FUNCTIONAL_TESTS_DOCKER_UID ?= $(call sh, id -u)
 benchmark:
 	@./scripts/run_benchmark.sh
 
+empty :=
+space := $(empty) $(empty)
+comma := ,
+
 lint-panics: generate
-	GOFLAGS=-tags='$(BUILD_TAGS),lint' \
+	GOFLAGS=-tags='$(subst $(space),$(comma),$(strip $(BUILD_TAGS) lint))' \
 	go tool goroutine-defer-guard -test=false -target github.com/status-im/status-go/common.LogOnPanic ./...
 
 lint: generate lint-panics
@@ -487,7 +558,7 @@ lint:
 lint-fix: generate
 	golangci-lint --build-tags '$(BUILD_TAGS) lint' run --fix ./...
 
-clean: ##@other Cleanup
+clean: clean-storage ##@other Cleanup
 	rm -fr build/bin/*
 
 git-clean:
@@ -559,5 +630,5 @@ pytest-lint:
 	$(MAKE) -C tests-functional lint
 
 generate-db: ##@build Generate fake sqlite DBs in ./build directory for IDE SQL inspections
-	LD_LIBRARY_PATH="$(NIM_SDS_LIB_DIR)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
+	LD_LIBRARY_PATH="$(RUNTIME_LIB_DIRS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \
 	go run tools/generate-db/main.go -out-dir build/db

_assets/ci/Jenkinsfile.tests

@@ -123,7 +123,7 @@ pipeline {
               variable: 'CODECOV_TOKEN'
             ),
           ]) {
-            nix.develop('make test-unit V=1', pure: false)
+            nix.develop('make test-unit V=1 USE_LOGOS_STORAGE=true', pure: false)
           }
           archiveArtifacts('report_*.xml, test_*.log, test-coverage.html')
         }