tile: drop nonstandard /log/v1/entry endpoint; read entries from data tiles

The browser UI and cactus-cli both used a cactus-specific
/log/v1/entry/<index> endpoint to fetch a single entry. Remove it so the
read path is standard tlog-tiles only and the UI works against any log.

- tile UI: clicking an entry in a fetched data tile now opens it in the
  inspector straight from the bytes already in hand (entryCache), no extra
  fetch; the by-index box and cactus-cli both resolve an entry by fetching
  its standard tile/entries/<N>[.p/<W>] data tile and splitting it.
- Remove the server handler/route and update MTC.md.
- Add a live integration test (TestCLIEntry) covering the CLI entry read
  and asserting the removed endpoint now 404s.

Author: mcpherrinm

MTC.md

@@ -140,8 +140,9 @@ that means the read-path (HTTP) serves files at:
 
 - `/checkpoint` — a c2sp signed-note with the latest size + root.
 - `/tile/<L>/<NNN>[.p/<W>]` — Merkle hash tiles.
-- `/tile/entries/<NNN>[.p/<W>]` — entry blobs (the "data" tiles).
-- `/log/v1/entry/<index>` — fetch one entry by index.
+- `/tile/entries/<NNN>[.p/<W>]` — entry blobs (the "data" tiles). A single
+  entry is read by fetching its data tile and splitting out the entry at its
+  position within the tile; there is no per-entry endpoint.
 - `/subtree/<name>` — cached signed subtree blob (the name is the
   `start-end` storage key).
 

cmd/cactus-cli/main.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/letsencrypt/cactus/cert"
+	"github.com/letsencrypt/cactus/log/tilewriter"
 	"github.com/letsencrypt/cactus/tlogx"
 )
 
@@ -121,12 +122,42 @@ func treeShow(logURL string) {
 	fmt.Printf("root:   %x\n", root[:])
 }
 
-// entryShow fetches an entry blob and prints a brief decode.
+// entryShow fetches an entry blob and prints a brief decode. It reads the
+// entry the way any tlog-tiles client would — there is no per-entry endpoint.
+// It fetches the standard data tile that contains the index (sized from the
+// checkpoint so the rightmost partial tile resolves to its .p/<width> path)
+// and splits out the entry at its position within that tile.
 func entryShow(logURL string, idx uint64) {
-	body, err := httpGet(fmt.Sprintf("%s/log/v1/entry/%d", logURL, idx))
+	cp, err := httpGet(logURL + "/checkpoint")
 	if err != nil {
-		die("fetch entry: %v", err)
+		die("fetch checkpoint: %v", err)
+	}
+	size, _, _, err := parseSignedNoteFlat(cp)
+	if err != nil {
+		die("parse checkpoint: %v", err)
+	}
+	if idx >= size {
+		die("entry %d out of range (tree size %d)", idx, size)
+	}
+	tileN := int64(idx) / int64(tilewriter.EntriesPerDataTile)
+	posInTile := int(int64(idx) % int64(tilewriter.EntriesPerDataTile))
+	width := tilewriter.EntriesPerDataTile
+	if w := int(size - uint64(tileN)*tilewriter.EntriesPerDataTile); w < width {
+		width = w // rightmost (partial) data tile
+	}
+	tilePath := logURL + "/" + tilewriter.DataTilePath(tileN, width)
+	tile, err := httpGet(tilePath)
+	if err != nil {
+		die("fetch data tile: %v", err)
+	}
+	entries, err := tilewriter.SplitDataTile(tile)
+	if err != nil {
+		die("parse data tile: %v", err)
+	}
+	if posInTile >= len(entries) {
+		die("entry %d not present in %s", idx, tilePath)
 	}
+	body := entries[posInTile]
 	// MerkleTreeCertEntry (§5.2.1): extensions<0..2^16-1> then uint16 type
 	// then the type-specific data. The leading uint16 is the extensions
 	// vector length, NOT the type.

integration/cli_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -147,6 +148,48 @@ func TestCLIProve(t *testing.T) {
 	}
 }
 
+// TestCLIEntry issues a cert, then runs `cactus-cli entry` to confirm the
+// CLI reads the entry through the standard data tile (there is no per-entry
+// endpoint), and checks the read path directly: the data tile serves, while
+// the old /log/v1/entry/<index> endpoint is gone (404).
+func TestCLIEntry(t *testing.T) {
+	s := bringUp(t, t.TempDir())
+	defer s.close()
+	if _, err := acmeIssueOne(s.acmeBase, "entry.test"); err != nil {
+		t.Fatal(err)
+	}
+
+	bin := buildCLI(t)
+	out, err := exec.Command(bin, "entry", s.tileBase, "0").CombinedOutput()
+	if err != nil {
+		t.Fatalf("cactus-cli entry: %v\nout=%s", err, out)
+	}
+	for _, want := range []string{"entry 0: tbs_cert_entry", "subject:", "spki hash:"} {
+		if !strings.Contains(string(out), want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+
+	// The first (and only) entry lives in the partial data tile 0 at width 1.
+	if r, err := http.Get(s.tileBase + "/tile/entries/000.p/1"); err != nil {
+		t.Fatalf("GET data tile: %v", err)
+	} else {
+		r.Body.Close()
+		if r.StatusCode != http.StatusOK {
+			t.Errorf("standard data tile: status = %d, want 200", r.StatusCode)
+		}
+	}
+	// The nonstandard per-entry endpoint must no longer exist.
+	if r, err := http.Get(s.tileBase + "/log/v1/entry/0"); err != nil {
+		t.Fatalf("GET removed endpoint: %v", err)
+	} else {
+		r.Body.Close()
+		if r.StatusCode != http.StatusNotFound {
+			t.Errorf("removed /log/v1/entry/0: status = %d, want 404", r.StatusCode)
+		}
+	}
+}
+
 // mustHex decodes hex string s into dst. dst is assumed to be the
 // right length.
 func mustHex(t *testing.T, s string, dst []byte) {

tile/app.js

@@ -185,10 +185,14 @@ function drillDown(level, index, i) {
   else selectAndFetch("entries", index);
 }
 
-// Open one entry (global index) in the entry viewer below.
+// Open one entry (global index) in the entry viewer below, reusing the bytes
+// already fetched as part of its data tile. Falls back to fetching the data
+// tile (a standard request) if this entry hasn't been loaded yet.
 function openEntry(globalIndex) {
   $("eindex").value = globalIndex;
-  fetchEntry();
+  const e = entryCache[globalIndex];
+  if (!e) { fetchEntry(); return; }
+  renderEntry(e.bytes, e.label);
   $("inspector").scrollIntoView({ behavior: "smooth", block: "nearest" });
 }
 
@@ -231,26 +235,42 @@ function renderHashTile(out, url, buf, level, index) {
   out.innerHTML = html;
 }
 
-// Entry (data) tiles are uint16-big-endian length-prefixed entries.
+// Split an entry (data) tile into its entries. Data tiles are a sequence of
+// uint16-big-endian length-prefixed blobs; subarray() clamps a final
+// length that overruns a truncated/partial tile, so this never throws.
+function splitDataTile(buf) {
+  const entries = [];
+  let pos = 0;
+  while (pos + 2 <= buf.length) {
+    const len = (buf[pos] << 8) | buf[pos + 1];
+    pos += 2;
+    entries.push(buf.subarray(pos, pos + len));
+    pos += len;
+  }
+  return entries;
+}
+
+// Bytes of entries we've already pulled out of fetched data tiles, keyed by
+// global entry index, so clicking an entry link opens it in the viewer with
+// no extra fetch. Each value is { bytes, label } (label is the source line
+// shown in the inspector header).
+const entryCache = {};
+
+// Render a fetched entry (data) tile, one clickable row per entry. Clicking a
+// row opens that entry in the inspector below straight from the bytes we
+// already have — no per-entry request, so this stays standard-API-only.
 function renderEntriesTile(out, url, buf, tileIndex) {
-  let html = '<span class="ok">GET ' + esc(url) + "</span>\n" + buf.length + " bytes\n" +
-    '<span class="hint">click an entry to open it in the entry viewer</span>\n\n';
-  let pos = 0, i = 0;
-  try {
-    while (pos + 2 <= buf.length) {
-      const len = (buf[pos] << 8) | buf[pos + 1];
-      pos += 2;
-      const body = buf.subarray(pos, pos + len);
-      pos += len;
-      const preview = hex(body.subarray(0, 32));
-      const gidx = tileIndex * TILE_W + i;
-      html += '<a href="#" onclick="openEntry(' + gidx + ');return false">entry ' + i + "</a>" +
-        " (#" + gidx + "): " + len + " bytes  " + preview + (len > 32 ? "…" : "") + "\n";
-      i++;
-    }
-    html = html.replace("\n\n", "\n" + i + " entr" + (i === 1 ? "y" : "ies") + "\n\n");
-  } catch (e) {
-    html += '<span class="err">decode error: ' + esc(String(e.message)) + "</span>";
+  const entries = splitDataTile(buf);
+  let html = '<span class="ok">GET ' + esc(url) + "</span>\n" + buf.length + " bytes · " +
+    entries.length + " entr" + (entries.length === 1 ? "y" : "ies") + "\n" +
+    '<span class="hint">click an entry to open it in the entry viewer below</span>\n\n';
+  for (let i = 0; i < entries.length; i++) {
+    const body = entries[i];
+    const gidx = tileIndex * TILE_W + i;
+    entryCache[gidx] = { bytes: body, label: url + " · entry " + i + " (#" + gidx + ")" };
+    const preview = hex(body.subarray(0, 32));
+    html += '<a href="#" onclick="openEntry(' + gidx + ');return false">entry ' + i + "</a>" +
+      " (#" + gidx + "): " + body.length + " bytes  " + preview + (body.length > 32 ? "…" : "") + "\n";
   }
   out.innerHTML = html;
 }
@@ -677,12 +697,14 @@ function renderAnnotations(container, ann) {
   container.innerHTML = ann.length ? html : '<div class="hint">(no annotations)</div>';
 }
 
-// Render the full three-column inspector for one entry blob.
-function renderEntry(buf, url) {
+// Render the full three-column inspector for one entry blob. `label` is the
+// source line shown in the header (e.g. the data-tile path the entry came
+// from).
+function renderEntry(buf, label) {
   const res = parseEntry(buf);
   const meta = $("entrymeta");
   meta.style.display = "block";
-  let m = '<span class="ok">GET ' + esc(url) + "</span> · " + buf.length + " bytes (MerkleTreeCertEntry)";
+  let m = '<span class="ok">' + esc(label) + "</span> · " + buf.length + " bytes (MerkleTreeCertEntry)";
   if (res.typeLabel) m += " · type " + esc(res.typeLabel);
   if (res.error) m += ' · <span class="err">' + esc(res.error) + "</span>";
   meta.innerHTML = m;
@@ -694,9 +716,15 @@ function renderEntry(buf, url) {
   return res;
 }
 
+// Look up an entry by global index using only the standard tile API: fetch
+// the data tile that contains it (auto-sizing the partial width from the
+// checkpoint) and pull out the entry at its position within that tile.
 async function fetchEntry() {
   const idx = parseInt($("eindex").value, 10) || 0;
-  const url = "log/v1/entry/" + idx;
+  const tileIndex = Math.floor(idx / TILE_W);
+  const posInTile = idx % TILE_W;
+  const width = tileWidth("entries", tileIndex);
+  const url = tileURL("entries", tileIndex, width || null);
   const meta = $("entrymeta");
   meta.style.display = "block";
   meta.textContent = "GET " + url + " …";
@@ -705,7 +733,15 @@ async function fetchEntry() {
     const r = await fetch(url, { cache: "no-store" });
     if (!r.ok) { meta.innerHTML = '<span class="err">GET ' + esc(url) + " → HTTP " + r.status + "</span>"; return; }
     const buf = new Uint8Array(await r.arrayBuffer());
-    renderEntry(buf, url);
+    const entries = splitDataTile(buf);
+    if (posInTile >= entries.length) {
+      meta.innerHTML = '<span class="err">entry #' + idx + " not in " + esc(url) +
+        " (tile holds " + entries.length + " entr" + (entries.length === 1 ? "y" : "ies") + ")</span>";
+      return;
+    }
+    const body = entries[posInTile];
+    entryCache[idx] = { bytes: body, label: url + " · entry " + posInTile + " (#" + idx + ")" };
+    renderEntry(body, entryCache[idx].label);
   } catch (e) {
     meta.innerHTML = '<span class="err">' + esc(String(e.message)) + "</span>";
   }
@@ -789,7 +825,7 @@ if (typeof module !== "undefined" && module.exports) {
   module.exports = {
     TILE_W, formatTileIndex, tileURL, treeHeight, tileLevels, tileWidth,
     loadCheckpoint, renderTileLists, populateLevels, selectAndFetch, drillDown,
-    openEntry, fetchTile, renderHashTile, renderEntriesTile, fetchEntry,
+    openEntry, fetchTile, renderHashTile, splitDataTile, renderEntriesTile, fetchEntry,
     DER, tagName, decodeOID, oidName, decodeDN, decodeSAN, hexPreview,
     previewInteger, formatTime, previewPrimitive, derNode, derForest,
     parseEntry, renderHexDump, renderStructure, renderAnnotations, renderEntry,

tile/app.test.js

@@ -125,11 +125,25 @@ test("drillDown from level 0 descends to the aligned entry (data) tile", () => {
   expect(fetches.at(-1)).toBe("tile/entries/001");
 });
 
-test("openEntry fetches the single entry by global index", () => {
+test("openEntry opens a tile-loaded entry without any extra fetch", () => {
   const { els, fetches } = makeUI();
-  app.openEntry(1283);
-  expect(els.eindex.value).toBe("1283");
-  expect(fetches.at(-1)).toBe("log/v1/entry/1283");
+  app.treeSize = 600;
+  // Render a data tile so its entries are cached, then click one.
+  const out = new El("#tileout");
+  const buf = new Uint8Array([0x00, 0x04, 0x00, 0x00, 0x00, 0x00]); // one 4-byte null_entry blob
+  app.renderEntriesTile(out, "tile/entries/000", buf, 0);
+  app.openEntry(0);
+  expect(els.eindex.value).toBe("0");
+  expect(els.inspector.style.display).toBe("grid"); // inspector populated
+  expect(fetches.length).toBe(0);                   // no network call
+});
+
+test("openEntry falls back to the standard data tile for an uncached entry", () => {
+  const { els, fetches } = makeUI();
+  app.treeSize = 600;
+  app.openEntry(257); // not loaded from a tile yet → fetch its data tile
+  expect(els.eindex.value).toBe("257");
+  expect(fetches.at(-1)).toBe("tile/entries/001"); // tile 1 (full), standard path
 });
 
 // =========================================================================
@@ -309,7 +323,7 @@ test("renderAnnotations: each row carries its structure's byte range", () => {
 test("renderEntry: fills all three columns and reports byte count", () => {
   const { els } = makeUI();
   const buf = fixtureEntry();
-  const res = app.renderEntry(buf, "log/v1/entry/0");
+  const res = app.renderEntry(buf, "tile/entries/000 · entry 0 (#0)");
   expect(res.type).toBe(1);
   expect(els.entrymeta.innerHTML).toContain(buf.length + " bytes");
   expect(els.entrymeta.innerHTML).toContain("tbs_cert_entry");

tile/index.html

@@ -157,8 +157,11 @@ <h2>Entry viewer</h2>
     <div class="row">
       <label>Index</label><input type="number" id="eindex" min="0" value="0">
       <button id="efetch">Fetch entry</button>
-      <span class="hint">GET <code>log/v1/entry/&lt;index&gt;</code> — one MerkleTreeCertEntry blob</span>
+      <span class="hint">extracted from its standard <code>tile/entries/&lt;N&gt;</code> data tile</span>
     </div>
+    <p class="hint">Click an entry in a data tile above to open it here with no extra fetch, or
+      look one up by index. Either way the entry is read straight from the standard
+      tlog-tiles data tile — no log-specific API.</p>
     <p class="hint">A §5.2.1 <code>MerkleTreeCertEntry</code>: TLS-presentation framing
       (<code>extensions</code> ‖ <code>entry_type</code>) wrapping the DER contents octets of a
       <code>TBSCertificateLogEntry</code>. Hover any column to light up the matching bytes.</p>

tile/server.go

@@ -12,12 +12,10 @@ import (
 	"errors"
 	"io/fs"
 	"net/http"
-	"strconv"
 	"strings"
 
 	"github.com/letsencrypt/cactus/landmark"
 	"github.com/letsencrypt/cactus/log"
-	"github.com/letsencrypt/cactus/log/tilewriter"
 	"github.com/letsencrypt/cactus/storage"
 )
 
@@ -58,7 +56,6 @@ func (s *Server) WithLandmarks(seq *landmark.Sequence) *Server {
 //	GET /checkpoint            — latest signed note
 //	GET /tile/<L>/<NNN..>      — hash tiles (c2sp tlog-tiles)
 //	GET /tile/entries/<NNN..>  — entry (data) tiles (c2sp tlog-tiles)
-//	GET /log/v1/entry/<index>  — single entry blob (the §5.2.1 MerkleTreeCertEntry)
 //	GET /subtree/<start>-<end> — cached signed subtree signature
 //	GET /landmarks             — §6.3.1 landmark list (only if WithLandmarks)
 func (s *Server) Handler() http.Handler {
@@ -67,7 +64,6 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("GET /app.js", s.handleAppJS)
 	mux.HandleFunc("GET /checkpoint", s.handleCheckpoint)
 	mux.HandleFunc("GET /tile/", s.handleTile)
-	mux.HandleFunc("GET /log/v1/entry/{index}", s.handleEntry)
 	mux.HandleFunc("GET /subtree/{name}", s.handleSubtree)
 	if s.landmarks != nil {
 		mux.Handle("GET /landmarks", s.landmarks.Handler())
@@ -128,45 +124,6 @@ func (s *Server) handleTile(w http.ResponseWriter, r *http.Request) {
 	w.Write(data)
 }
 
-func (s *Server) handleEntry(w http.ResponseWriter, r *http.Request) {
-	idxStr := r.PathValue("index")
-	idx, err := strconv.ParseUint(idxStr, 10, 64)
-	if err != nil {
-		http.Error(w, "bad index", http.StatusBadRequest)
-		return
-	}
-
-	// Locate the data tile and the position of the requested entry within it.
-	tileN := int64(idx) / int64(tilewriter.EntriesPerDataTile)
-	posInTile := int(int64(idx) - tileN*int64(tilewriter.EntriesPerDataTile))
-
-	// Find any persisted data tile at width >= posInTile+1, preferring
-	// the widest (most up-to-date) one.
-	for width := tilewriter.EntriesPerDataTile; width >= posInTile+1; width-- {
-		data, err := s.fs.Get("log/" + tilewriter.DataTilePath(tileN, width))
-		if errors.Is(err, fs.ErrNotExist) {
-			continue
-		}
-		if err != nil {
-			http.Error(w, "entry read failed", http.StatusInternalServerError)
-			return
-		}
-		entries, err := tilewriter.SplitDataTile(data)
-		if err != nil {
-			http.Error(w, "data tile parse failed", http.StatusInternalServerError)
-			return
-		}
-		if posInTile >= len(entries) {
-			continue
-		}
-		w.Header().Set("Content-Type", "application/octet-stream")
-		w.Header().Set("Cache-Control", "public, max-age=86400, immutable")
-		w.Write(entries[posInTile])
-		return
-	}
-	http.NotFound(w, r)
-}
-
 func (s *Server) handleSubtree(w http.ResponseWriter, r *http.Request) {
 	name := r.PathValue("name") // e.g. "8-13"
 	data, err := s.fs.Get("log/subtrees/" + name)