From f7414f57c7a9eba4a234292b1a6fb3507de454fc Mon Sep 17 00:00:00 2001 From: litbot-9000 <58193817+litbot-9000@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:08:14 +0000 Subject: [PATCH 1/3] lwwallet: bound the response body quoted back in an error Every Esplora error path interpolated the whole response body. That is fine for the short status line a REST API returns, but a base URL that names a web frontend instead of the API root answers with a full HTML document, and the resulting error carried three kilobytes of markup up through the wallet, the daemon and out to the RPC client, where it is neither readable nor useful. Collapse the body onto one line and bound it, so a single-line log record stays a single line. The cut lands on a rune boundary: the body is arbitrary bytes from an endpoint whose shape we have just decided not to trust, and slicing it blindly would emit a half-encoded rune. --- lwwallet/esplora.go | 36 +++++++++++++++++++++++--- lwwallet/esplora_test.go | 56 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/lwwallet/esplora.go b/lwwallet/esplora.go index 9c58ac536..974bfa4fb 100644 --- a/lwwallet/esplora.go +++ b/lwwallet/esplora.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" @@ -689,6 +690,33 @@ func (c *EsploraClient) GetOutspend(ctx context.Context, txid chainhash.Hash, return &outspend, nil } +// maxErrBodyBytes bounds how much of a response body an error quotes back. +// A base URL pointing at a web frontend answers with a full HTML page, and +// the unbounded form of these errors carried three kilobytes of markup +// through every layer up to the RPC client. +const maxErrBodyBytes = 256 + +// truncateBody renders a response body for an error message: collapsed onto +// one line, since a multi-line body is illegible inside a single-line log +// record, and bounded to maxErrBodyBytes. +// +// The cut lands on a rune boundary. A body is arbitrary bytes from an +// endpoint we have just decided we do not trust the shape of, so slicing it +// blindly would emit a half-encoded rune into the log. +func truncateBody(body []byte) string { + collapsed := strings.Join(strings.Fields(string(body)), " ") + if len(collapsed) <= maxErrBodyBytes { + return collapsed + } + + cut := maxErrBodyBytes + for cut > 0 && !utf8.RuneStart(collapsed[cut]) { + cut-- + } + + return collapsed[:cut] + "... (truncated)" +} + // get performs an HTTP GET request and returns the response body. func (c *EsploraClient) get(ctx context.Context, path string) ([]byte, error) { req, err := http.NewRequestWithContext( @@ -713,7 +741,7 @@ func (c *EsploraClient) get(ctx context.Context, path string) ([]byte, error) { if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, - string(body)) + truncateBody(body)) } return body, nil @@ -773,7 +801,7 @@ func (c *EsploraClient) TestMempoolAccept(ctx context.Context, if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, - string(respBody)) + truncateBody(respBody)) } var results []testMempoolAcceptResult @@ -868,7 +896,7 @@ func (c *EsploraClient) SubmitPackage(ctx context.Context, if resp.StatusCode != http.StatusOK { return fmt.Errorf("submit package HTTP %d: %s", resp.StatusCode, - string(respBody)) + truncateBody(respBody)) } if len(respBody) > 0 { @@ -959,7 +987,7 @@ func (c *EsploraClient) post(ctx context.Context, path string, body string) ( if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, - string(respBody)) + truncateBody(respBody)) } return respBody, nil diff --git a/lwwallet/esplora_test.go b/lwwallet/esplora_test.go index c9e52e372..e01fb8daa 100644 --- a/lwwallet/esplora_test.go +++ b/lwwallet/esplora_test.go @@ -7,7 +7,9 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" + "unicode/utf8" "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" @@ -456,3 +458,57 @@ func TestScriptHashEncoding(t *testing.T) { t, hex.EncodeToString(reversed), scriptHashHex(pkScript), ) } + +// TestEsploraHTTPErrorBodyTruncated verifies a non-200 body is collapsed +// and bounded in the error. A misconfigured base URL naming a web frontend +// answers with a full HTML page rather than a short status line, so the +// error path needs a bound to stay readable. +func TestEsploraHTTPErrorBodyTruncated(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + + _, err := w.Write([]byte(strings.Repeat("x", 4096))) + require.NoError(t, err) + }), + ) + defer srv.Close() + + client := NewEsploraClient(srv.URL, btclog.Disabled) + _, err := client.GetTipHeight(t.Context()) + require.Error(t, err) + require.Contains(t, err.Error(), "404") + require.Contains(t, err.Error(), "truncated") + require.Less(t, len(err.Error()), 512) +} + +// TestTruncateBody covers the shapes the helper has to handle: a short body +// passes through unchanged, a multi-line body is collapsed onto one line so +// it stays legible inside a single-line log record, and a long body is cut. +func TestTruncateBody(t *testing.T) { + t.Parallel() + + require.Equal( + t, "already short", + truncateBody( + []byte("already short"), + ), + ) + + require.Equal(t, "a b c", truncateBody([]byte("a\n b\n\tc\n"))) + + long := truncateBody([]byte(strings.Repeat("y", maxErrBodyBytes+10))) + require.Len(t, long, maxErrBodyBytes+len("... (truncated)")) + require.Contains(t, long, "truncated") + + // A cut that would land inside a multi-byte rune backs up to the + // rune boundary rather than emitting a half-encoded rune. "€" is + // three bytes, so a body of them has no boundary at maxErrBodyBytes + // unless the helper looks for one. + runes := truncateBody( + []byte(strings.Repeat("€", maxErrBodyBytes)), + ) + require.True(t, utf8.ValidString(runes)) +} From bfa316926a1562f7c4a3efe5c7c1f6dae5dca4ce Mon Sep 17 00:00:00 2001 From: litbot-9000 <58193817+litbot-9000@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:09:42 +0000 Subject: [PATCH 2/3] lwwallet: reject an HTML response as a misconfigured endpoint A mempool.space instance serves its Esplora-compatible REST API under /api, and serves a single-page web app everywhere else. That app answers every unrouted path with index.html and a 200, so a base URL missing the /api suffix passes the status check and hands markup to a parser. The operator sees the failure as parse tip height: strconv.ParseInt: parsing "... raised from inside wallet startup, several layers away from the setting that caused it, with the whole document inlined. No Esplora endpoint answers with HTML, so a response that does did not come from the API. Reject it before the body reaches a parser and name the likely fix in the error. The check keys on the content type alone. Body sniffing would be wrong here: /tx/:txid/raw and /block/:hash/raw return arbitrary binary, which can legitimately begin with the same '<' byte an HTML document does. A test pins that case with a version-60 transaction, whose serialization starts with 0x3c. ErrNotEsploraAPI is exported so a caller can tell a misconfiguration apart from a transient failure and decline to retry it. --- lwwallet/esplora.go | 60 +++++++++++++++++++++ lwwallet/esplora_test.go | 114 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/lwwallet/esplora.go b/lwwallet/esplora.go index 974bfa4fb..b7b9bdc93 100644 --- a/lwwallet/esplora.go +++ b/lwwallet/esplora.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "mime" "net/http" "strconv" "strings" @@ -690,6 +691,11 @@ func (c *EsploraClient) GetOutspend(ctx context.Context, txid chainhash.Hash, return &outspend, nil } +// ErrNotEsploraAPI is returned when an endpoint answers a request with an +// HTML page. It signals a misconfigured base URL rather than a transient +// failure, so callers can treat it as fatal instead of retrying. +var ErrNotEsploraAPI = errors.New("endpoint is not an Esplora REST API") + // maxErrBodyBytes bounds how much of a response body an error quotes back. // A base URL pointing at a web frontend answers with a full HTML page, and // the unbounded form of these errors carried three kilobytes of markup @@ -717,6 +723,44 @@ func truncateBody(body []byte) string { return collapsed[:cut] + "... (truncated)" } +// checkNotHTML rejects an otherwise-successful response that carries an HTML +// body. +// +// No Esplora endpoint answers with HTML, so such a response did not come +// from the API: something else is answering at this address. The case that +// motivates the check is a base URL naming a mempool.space web frontend +// rather than its REST root. That frontend is a single-page app, so it +// serves index.html with a 200 for every path it does not route, including +// every path this client asks for. The status check above therefore passes +// and the markup reaches a parser, surfacing as +// `strconv.ParseInt: parsing "…` from inside wallet startup +// — several layers away from the setting that caused it. +// +// The check keys on the content type alone. Body sniffing would be wrong +// here: /tx/:txid/raw and /block/:hash/raw return arbitrary binary, which +// can legitimately begin with the same '<' byte an HTML document does. +func (c *EsploraClient) checkNotHTML(resp *http.Response, path string) error { + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + return nil + } + + // A parse failure is deliberately not treated as evidence either way. + // ParseMediaType still yields the media type when only the parameters + // are malformed, and yields the empty string otherwise, which falls + // through to the caller's own parser rather than being reported as a + // misconfiguration. + mediaType, _, _ := mime.ParseMediaType(contentType) + if mediaType != "text/html" { + return nil + } + + return fmt.Errorf("%w: %s%s answered with an HTML page; a "+ + "mempool.space or Esplora instance serves this API under "+ + "/api, so the URL likely needs that suffix (e.g. "+ + "https://mempool.space/api)", ErrNotEsploraAPI, c.baseURL, path) +} + // get performs an HTTP GET request and returns the response body. func (c *EsploraClient) get(ctx context.Context, path string) ([]byte, error) { req, err := http.NewRequestWithContext( @@ -744,6 +788,10 @@ func (c *EsploraClient) get(ctx context.Context, path string) ([]byte, error) { truncateBody(body)) } + if err := c.checkNotHTML(resp, path); err != nil { + return nil, err + } + return body, nil } @@ -804,6 +852,10 @@ func (c *EsploraClient) TestMempoolAccept(ctx context.Context, truncateBody(respBody)) } + if err := c.checkNotHTML(resp, "/txs/test"); err != nil { + return nil, err + } + var results []testMempoolAcceptResult if err := json.Unmarshal(respBody, &results); err != nil { return nil, fmt.Errorf("parse test mempool response: %w", err) @@ -899,6 +951,10 @@ func (c *EsploraClient) SubmitPackage(ctx context.Context, truncateBody(respBody)) } + if err := c.checkNotHTML(resp, "/txs/package"); err != nil { + return err + } + if len(respBody) > 0 { c.log.DebugS(ctx, "Package response", slog.String("body", string(respBody)), @@ -990,5 +1046,9 @@ func (c *EsploraClient) post(ctx context.Context, path string, body string) ( truncateBody(respBody)) } + if err := c.checkNotHTML(resp, path); err != nil { + return nil, err + } + return respBody, nil } diff --git a/lwwallet/esplora_test.go b/lwwallet/esplora_test.go index e01fb8daa..cf0512a85 100644 --- a/lwwallet/esplora_test.go +++ b/lwwallet/esplora_test.go @@ -459,6 +459,120 @@ func TestScriptHashEncoding(t *testing.T) { ) } +// mempoolFrontendHTML is the opening of the single-page app a mempool.space +// web frontend serves for any path it does not route. It is what the client +// receives when its base URL names the frontend rather than the REST root +// under /api. +const mempoolFrontendHTML = ` + + + + + mempool - Bitcoin Explorer + + + + + + + + +` + +// newHTMLFrontendServer returns a server that imitates that frontend: every +// path answers 200 with the HTML document rather than 404, which is what +// makes the misconfiguration survive the status check. +func newHTMLFrontendServer(t *testing.T) *httptest.Server { + t.Helper() + + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; "+ + "charset=utf-8") + w.WriteHeader(http.StatusOK) + + _, err := w.Write([]byte(mempoolFrontendHTML)) + require.NoError(t, err) + }), + ) + t.Cleanup(srv.Close) + + return srv +} + +// TestEsploraRejectsHTMLResponse verifies that a 200 carrying an HTML body +// is reported as a misconfigured endpoint rather than handed to a parser. +// Without the check the markup reaches strconv.ParseInt and surfaces from +// inside wallet startup as a parse error quoting the whole document. +func TestEsploraRejectsHTMLResponse(t *testing.T) { + t.Parallel() + + srv := newHTMLFrontendServer(t) + + client := NewEsploraClient(srv.URL, btclog.Disabled) + _, err := client.GetTipHeight(t.Context()) + require.ErrorIs(t, err, ErrNotEsploraAPI) + + // The message has to name the fix, since the symptom points at a + // parser rather than at the setting that caused it. + require.Contains(t, err.Error(), "/api") + require.Contains(t, err.Error(), "/blocks/tip/height") + + // And it must not carry the page: the unbounded form of this error + // dragged kilobytes of markup up through every layer to the caller. + require.NotContains(t, err.Error(), "") + require.Less(t, len(err.Error()), 512) +} + +// TestEsploraRejectsHTMLResponseOnPost verifies the POST path is guarded +// too. A misconfigured base URL reaches broadcast the same way it reaches +// every read, and the SPA answers a POST with the same 200 and page. +func TestEsploraRejectsHTMLResponseOnPost(t *testing.T) { + t.Parallel() + + srv := newHTMLFrontendServer(t) + + client := NewEsploraClient(srv.URL, btclog.Disabled) + _, err := client.BroadcastTx(t.Context(), wire.NewMsgTx(2)) + require.ErrorIs(t, err, ErrNotEsploraAPI) + require.Contains(t, err.Error(), "/tx") +} + +// TestEsploraAcceptsBinaryBodyStartingWithAngleBracket pins that the guard +// keys on the content type and not on the body. The raw transaction and raw +// block endpoints return arbitrary binary, which can legitimately begin with +// the same '<' byte an HTML document does; sniffing the body would reject +// those responses. A transaction with version 60 serializes to a leading +// 0x3c, which is exactly that byte. +func TestEsploraAcceptsBinaryBodyStartingWithAngleBracket(t *testing.T) { + t.Parallel() + + tx := wire.NewMsgTx(60) + tx.AddTxIn(wire.NewTxIn(&wire.OutPoint{}, nil, nil)) + tx.AddTxOut(wire.NewTxOut(1000, []byte{0x00, 0x14})) + + var buf bytes.Buffer + require.NoError(t, tx.Serialize(&buf)) + require.Equal(t, byte('<'), buf.Bytes()[0]) + + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set( + "Content-Type", "application/octet-stream", + ) + + _, err := w.Write(buf.Bytes()) + require.NoError(t, err) + }), + ) + defer srv.Close() + + client := NewEsploraClient(srv.URL, btclog.Disabled) + got, err := client.GetRawTx(t.Context(), tx.TxHash()) + require.NoError(t, err) + require.Equal(t, tx.TxHash(), got.TxHash()) +} + // TestEsploraHTTPErrorBodyTruncated verifies a non-200 body is collapsed // and bounded in the error. A misconfigured base URL naming a web frontend // answers with a full HTML page rather than a short status line, so the From b226f1732ddc51c6c2620b61945bc45a684b81fe Mon Sep 17 00:00:00 2001 From: litbot-9000 <58193817+litbot-9000@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:21:37 +0000 Subject: [PATCH 3/3] lwwallet: document the Esplora endpoint-shape guard The package doc described the Esplora client's cache-integrity check as the only thing standing between a response and a parser. It now also passes an endpoint-shape guard, and neither that guard nor the new exported ErrNotEsploraAPI sentinel appeared anywhere in the doc. Record both, along with the two decisions a future reader is most likely to want to undo: that the guard keys on Content-Type alone rather than sniffing the body, because the /raw endpoints return binary that can legitimately start with '<', and that a body quoted back in an error is bounded to 256 bytes. --- lwwallet/AGENTS.md | 18 +++++++++++++++++- lwwallet/CLAUDE.md | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lwwallet/AGENTS.md b/lwwallet/AGENTS.md index 3f4f8b4b2..6bd768fde 100644 --- a/lwwallet/AGENTS.md +++ b/lwwallet/AGENTS.md @@ -41,7 +41,13 @@ base logic with the neutrino-backed `btcwbackend` sibling via the extracted (transactions, blocks, headers) are cached in LRU caches bounded by cumulative serialized byte size (see `esplora_cache.go`). Mutable live data (tip height, UTXOs, fee estimates) is never cached. Cache integrity: every - response is verified to hash to the requested key before insertion. + response is verified to hash to the requested key before insertion. Every + response also passes an endpoint-shape guard before reaching a parser; a + 200 carrying an HTML body is rejected with `ErrNotEsploraAPI`. +- `ErrNotEsploraAPI` — Sentinel error returned when an endpoint answers with + an HTML page. It marks a misconfigured base URL rather than a transient + failure, so callers can treat it as fatal instead of retrying. Match with + `errors.Is`; the message names the likely fix (the `/api` suffix). - `EsploraChainService` — `chain.Interface` adapter over `EsploraClient`, driven by a shared `TipPoller`. Feeds btcwallet's internal address-credit pipeline. Constructor: `NewEsploraChainService(esplora, tipPoller, logger)`. @@ -89,6 +95,16 @@ base logic with the neutrino-backed `btcwbackend` sibling via the extracted (known limitation; acceptable for confirmation-target use cases). - LRU caches only hold immutable, hash-addressed data; a verified hash prevents a compromised Esplora endpoint from injecting arbitrary cache entries. +- The HTML guard keys on `Content-Type` alone, never on the body bytes. + `/tx/:txid/raw` and `/block/:hash/raw` return arbitrary binary that can + legitimately begin with the same `<` byte an HTML document does, so + sniffing would reject valid responses. A missing or unparseable content + type falls through to the caller's own parser rather than being reported + as a misconfiguration. +- Response bodies quoted back in an error are collapsed onto one line and + bounded to 256 bytes, cut on a rune boundary. A base URL naming a web + frontend answers with a full HTML page, and the unbounded form carried + kilobytes of markup through every layer up to the RPC client. - `scriptHashHex` hex-encodes the SHA256 digest in its natural byte order. Esplora's REST API differs from the Electrum wire protocol here, which reverses it, and a wrong order fails silently because the API answers an diff --git a/lwwallet/CLAUDE.md b/lwwallet/CLAUDE.md index 3f4f8b4b2..6bd768fde 100644 --- a/lwwallet/CLAUDE.md +++ b/lwwallet/CLAUDE.md @@ -41,7 +41,13 @@ base logic with the neutrino-backed `btcwbackend` sibling via the extracted (transactions, blocks, headers) are cached in LRU caches bounded by cumulative serialized byte size (see `esplora_cache.go`). Mutable live data (tip height, UTXOs, fee estimates) is never cached. Cache integrity: every - response is verified to hash to the requested key before insertion. + response is verified to hash to the requested key before insertion. Every + response also passes an endpoint-shape guard before reaching a parser; a + 200 carrying an HTML body is rejected with `ErrNotEsploraAPI`. +- `ErrNotEsploraAPI` — Sentinel error returned when an endpoint answers with + an HTML page. It marks a misconfigured base URL rather than a transient + failure, so callers can treat it as fatal instead of retrying. Match with + `errors.Is`; the message names the likely fix (the `/api` suffix). - `EsploraChainService` — `chain.Interface` adapter over `EsploraClient`, driven by a shared `TipPoller`. Feeds btcwallet's internal address-credit pipeline. Constructor: `NewEsploraChainService(esplora, tipPoller, logger)`. @@ -89,6 +95,16 @@ base logic with the neutrino-backed `btcwbackend` sibling via the extracted (known limitation; acceptable for confirmation-target use cases). - LRU caches only hold immutable, hash-addressed data; a verified hash prevents a compromised Esplora endpoint from injecting arbitrary cache entries. +- The HTML guard keys on `Content-Type` alone, never on the body bytes. + `/tx/:txid/raw` and `/block/:hash/raw` return arbitrary binary that can + legitimately begin with the same `<` byte an HTML document does, so + sniffing would reject valid responses. A missing or unparseable content + type falls through to the caller's own parser rather than being reported + as a misconfiguration. +- Response bodies quoted back in an error are collapsed onto one line and + bounded to 256 bytes, cut on a rune boundary. A base URL naming a web + frontend answers with a full HTML page, and the unbounded form carried + kilobytes of markup through every layer up to the RPC client. - `scriptHashHex` hex-encodes the SHA256 digest in its natural byte order. Esplora's REST API differs from the Electrum wire protocol here, which reverses it, and a wrong order fails silently because the API answers an