feat: Persistent state across restarts, incremental rescans, and logging

Author: m0wer

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **Persistent state across restarts** (`rescan_state.db`): Watched addresses, UTXO set, and rescan metadata (last scanned tip, start height) are now persisted to a separate bbolt database. On restart, the server restores its previous state so UTXOs are available immediately without re-scanning. The state store uses three buckets (`watched_addrs`, `utxo_set`, `rescan_meta`) and is optional (nil = no persistence) for backward compatibility with tests.
+- **Incremental rescan**: When a rescan is requested with a `start_height` within the already-scanned range, the scan starts from `LastScannedTip+1` instead of re-scanning the entire range. If already up-to-date, returns immediately. This avoids redundant 52k+ block rescans after restarts.
+- **Rescan fallback to watched addresses**: `POST /v1/rescan` with an empty `addresses` field now falls back to all previously watched addresses instead of silently doing nothing.
+- **HTTP request/response logging middleware**: Every API call is logged with method, path, status code, and duration. 4xx/5xx responses log at WARN level; 2xx at INFO level.
+- **Rescan progress logging**: During block scanning, progress is logged every 10 seconds with blocks scanned/total, percentage, current height, blocks/sec, estimated time remaining, and filter match count. A final summary includes total blocks, duration, speed, filter matches, and detailed UTXO accounting (found/added/removed).
+- **Rescan handler request logging**: `POST /v1/rescan` now logs start_height, address count, and outpoint count at INFO level.
+
 ## [0.8.0] - 2026-03-17
 
 ### Added

README.md

@@ -41,7 +41,7 @@ docker run -d \
   -p 8334:8334 \
   -v neutrino-data:/data/neutrino \
   -e NETWORK=regtest \
-  -e CONNECT_PEERS=bitcoin-node:18444 \
+  -e ADD_PEERS=bitcoin-node:18444 \
   -e LOG_LEVEL=debug \
   ghcr.io/m0wer/neutrino-api
 ```
@@ -92,7 +92,7 @@ Anyone can reproduce and verify a release locally with one command:
 | `LISTEN_ADDR` | `0.0.0.0:8334` | REST API listen address |
 | `DATA_DIR` | `/data/neutrino` | Data directory for headers and filters |
 | `LOG_LEVEL` | `info` | Log level (trace, debug, info, warn, error) |
-| `CONNECT_PEERS` | | Comma-separated list of peers (e.g., `node1:8333,node2:8333`) |
+| `ADD_PEERS` | | Comma-separated list of preferred peers (e.g., `node1:8333,node2:8333`) while still allowing peer discovery |
 | `TOR_PROXY` | | Tor SOCKS5 proxy address (e.g., `127.0.0.1:9050`) |
 | `MAX_PEERS` | `8` | Maximum number of peers to connect to |
 
@@ -104,7 +104,7 @@ Anyone can reproduce and verify a release locally with one command:
   --listen=0.0.0.0:8334 \
   --datadir=/data/neutrino \
   --loglevel=info \
-  --connect=peer1:8333,peer2:8333 \
+  --addpeer=peer1:8333,peer2:8333 \
   --torproxy=127.0.0.1:9050 \
   --maxpeers=8
 ```

docker-compose.yml

@@ -48,7 +48,7 @@ services:
       - LISTEN_ADDR=0.0.0.0:8334
       - DATA_DIR=/data/neutrino
       - LOG_LEVEL=debug
-      - CONNECT_PEERS=bitcoin:18444
+      - ADD_PEERS=bitcoin:18444
       - MAX_PEERS=8
     ports:
       - "8334:8334"

neutrino_server/cmd/neutrinod/main.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"time"
 
@@ -32,9 +33,11 @@ func main() {
 	listen := flag.String("listen", getEnv("LISTEN_ADDR", "0.0.0.0:8334"), "REST API listen address")
 	dataDir := flag.String("datadir", getEnv("DATA_DIR", "/data/neutrino"), "Data directory for headers and filters")
 	logLevel := flag.String("loglevel", getEnv("LOG_LEVEL", "info"), "Log level (trace, debug, info, warn, error)")
-	connectPeers := flag.String("connect", getEnv("CONNECT_PEERS", ""), "Comma-separated list of peers to connect to")
 	addPeers := flag.String("addpeer", getEnv("ADD_PEERS", ""), "Comma-separated list of peers to add while still allowing discovery")
 	torProxy := flag.String("torproxy", getEnv("TOR_PROXY", ""), "Tor SOCKS5 proxy address (e.g., 127.0.0.1:9050)")
+	prefetchFilters := flag.Bool("prefetchfilters", getEnvBool("PREFETCH_FILTERS", true), "Enable background compact filter prefetch")
+	prefetchWorkers := flag.Int("prefetchworkers", getEnvInt("PREFETCH_WORKERS", 0), "Number of workers for background filter prefetch (0=auto)")
+	prefetchStart := flag.Int("prefetchstart", getEnvInt("PREFETCH_START", 0), "Start height for background filter prefetch")
 	showVersion := flag.Bool("version", false, "Show version and exit")
 	flag.Parse()
 
@@ -65,14 +68,17 @@ func main() {
 
 	// Create neutrino node
 	nodeConfig := &neutrino.Config{
-		Network:      *network,
-		DataDir:      *dataDir,
-		TorProxy:     *torProxy,
-		ConnectPeers: *connectPeers,
-		AddPeers:     *addPeers,
-		MaxPeers:     8,
-		Logger:       backend,
-		LogLevel:     *logLevel,
+		Network:         *network,
+		DataDir:         *dataDir,
+		TorProxy:        *torProxy,
+		AddPeers:        *addPeers,
+		MaxPeers:        8,
+		FilterCacheSize: 100 * 1024 * 1024,
+		PrefetchFilters: *prefetchFilters,
+		PrefetchWorkers: *prefetchWorkers,
+		PrefetchStart:   int32(*prefetchStart),
+		Logger:          backend,
+		LogLevel:        *logLevel,
 	}
 
 	node, err := neutrino.NewNode(nodeConfig)
@@ -142,3 +148,31 @@ func getEnv(key, defaultValue string) string {
 	}
 	return defaultValue
 }
+
+// getEnvBool returns a bool env var or a default value.
+func getEnvBool(key string, defaultValue bool) bool {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+
+	parsed, err := strconv.ParseBool(value)
+	if err != nil {
+		return defaultValue
+	}
+	return parsed
+}
+
+// getEnvInt returns an int env var or a default value.
+func getEnvInt(key string, defaultValue int) int {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+
+	parsed, err := strconv.Atoi(value)
+	if err != nil {
+		return defaultValue
+	}
+	return parsed
+}

neutrino_server/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/btcsuite/btcwallet/walletdb v1.3.5
 	github.com/gorilla/mux v1.8.1
 	github.com/lightninglabs/neutrino v0.16.0
+	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
 	golang.org/x/net v0.48.0
 )
 
@@ -28,7 +29,6 @@ require (
 	github.com/lightningnetwork/lnd/clock v1.0.1 // indirect
 	github.com/lightningnetwork/lnd/queue v1.0.1 // indirect
 	github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect
-	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 // indirect
 	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
 )

neutrino_server/internal/api/handler.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/wire"
@@ -49,6 +50,9 @@ func NewHandler(node NodeInterface, logger btclog.Logger) *Handler {
 
 // RegisterRoutes registers all API routes.
 func (h *Handler) RegisterRoutes(r *mux.Router) {
+	// Add request logging middleware.
+	r.Use(h.loggingMiddleware)
+
 	// Status
 	r.HandleFunc("/v1/status", h.handleGetStatus).Methods("GET")
 
@@ -76,6 +80,34 @@ func (h *Handler) RegisterRoutes(r *mux.Router) {
 	r.HandleFunc("/v1/peers", h.handleGetPeers).Methods("GET")
 }
 
+// statusRecorder wraps http.ResponseWriter to capture the status code.
+type statusRecorder struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func (rec *statusRecorder) WriteHeader(code int) {
+	rec.statusCode = code
+	rec.ResponseWriter.WriteHeader(code)
+}
+
+// loggingMiddleware logs every HTTP request with method, path, status, and duration.
+func (h *Handler) loggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		rec := &statusRecorder{ResponseWriter: w, statusCode: http.StatusOK}
+		next.ServeHTTP(rec, r)
+		duration := time.Since(start)
+
+		// Use Warn level for errors (4xx/5xx), Info for everything else.
+		if rec.statusCode >= 400 {
+			h.logger.Warnf("%s %s -> %d (%s)", r.Method, r.URL.Path, rec.statusCode, duration.Round(time.Millisecond))
+		} else {
+			h.logger.Infof("%s %s -> %d (%s)", r.Method, r.URL.Path, rec.statusCode, duration.Round(time.Millisecond))
+		}
+	})
+}
+
 // Response helpers
 
 func (h *Handler) jsonResponse(w http.ResponseWriter, data any) {
@@ -316,6 +348,9 @@ func (h *Handler) handleRescan(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	h.logger.Infof("Rescan requested: start_height=%d, addresses=%d, outpoints=%d",
+		req.StartHeight, len(req.Addresses), len(req.Outpoints))
+
 	// Start rescan in background goroutine to not block HTTP response
 	go func() {
 		if err := h.node.Rescan(req.StartHeight, req.Addresses); err != nil {

neutrino_server/internal/api/handler_test.go

@@ -585,3 +585,54 @@ func TestHandleGetRescanStatus_NotInProgress(t *testing.T) {
 		t.Error("expected last_scanned_tip in response")
 	}
 }
+
+func TestLoggingMiddleware_Success(t *testing.T) {
+	backend := btclog.NewBackend(os.Stdout)
+	logger := backend.Logger("TEST")
+
+	handler := NewHandler(&mockNode{}, logger)
+
+	router := mux.NewRouter()
+	handler.RegisterRoutes(router)
+
+	req, err := http.NewRequest("GET", "/v1/status", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	router.ServeHTTP(rr, req)
+
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
+	}
+
+	// Verify the response is valid JSON (middleware should not interfere)
+	var response neutrino.Status
+	if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+		t.Fatalf("could not decode response: %v", err)
+	}
+}
+
+func TestLoggingMiddleware_Error(t *testing.T) {
+	backend := btclog.NewBackend(os.Stdout)
+	logger := backend.Logger("TEST")
+
+	handler := NewHandler(&mockNode{}, logger)
+
+	router := mux.NewRouter()
+	handler.RegisterRoutes(router)
+
+	// Invalid body should trigger 400
+	req, err := http.NewRequest("POST", "/v1/rescan", bytes.NewBufferString("invalid json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	router.ServeHTTP(rr, req)
+
+	if status := rr.Code; status != http.StatusBadRequest {
+		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusBadRequest)
+	}
+}