cmd/witnessctl: add pull-logs command

Fixes #31

Author: FiloSottile

cmd/litewitness/README.md

@@ -88,6 +88,15 @@ the origin and key from a 32-byte hex-encoded Ed25519 public key.
 
     witnessctl list-logs -db <path>
 
+The `pull-logs` command fetches a list of known logs from [a witness network log
+list](https://github.com/transparency-dev/witness-network/blob/main/site/content/participate.md#automate-discovery-of-logs-in-the-selected-lists)
+URL or file path. It only adds new logs, it does not remove or change the keys
+of existing ones. It is designed to run in a cronjob, and by default (without
+`-verbose`) it only prints output if there are logs with keys different from
+what is already in the database, suggesting they may need to be rotated.
+
+    witnessctl pull-logs -db <path> -source <log list url> -verbose
+
 The `list-logs` command lists known logs, in JSON lines like the following.
 
     {"origin":"sigsum.org/v1/tree/4d6d8825a6bb689d459628312889dfbb0bcd41b5211d9e1ce768b0ff0309e562","size":5,"root_hash":"QrtXrQZCCvpIgsSmOsah7HdICzMLLyDfxToMql9WTjY=","keys":["sigsum.org/v1/tree/4d6d8825a6bb689d459628312889dfbb0bcd41b5211d9e1ce768b0ff0309e562+5202289b+Af/cLU2Y5BJNP+r3iMDC+av9eWCD0fBJVDfzAux5zxAP"]}

cmd/litewitness/testdata/gentest/sigsum.go

@@ -16,7 +16,6 @@ import (
 	"log"
 	"net/url"
 
-	"github.com/caarlos0/sshmarshal"
 	"golang.org/x/crypto/hkdf"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/mod/sumdb/note"
@@ -55,8 +54,11 @@ func main() {
 	fmt.Printf("- origin URL-encoded: %s\n", url.QueryEscape(origin))
 
 	const algEd25519 = 1
-	skey := fmt.Sprintf("PRIVATE+KEY+%s+%08x+%s", origin, noteKeyHash(origin, append([]byte{algEd25519}, publicKey...)), base64.StdEncoding.EncodeToString(append([]byte{algEd25519}, privateKey.Seed()...)))
+	kh := noteKeyHash(origin, append([]byte{algEd25519}, publicKey...))
+	vkey := fmt.Sprintf("%s+%08x+%s", origin, kh, base64.StdEncoding.EncodeToString(append([]byte{algEd25519}, publicKey...)))
+	skey := fmt.Sprintf("PRIVATE+KEY+%s+%08x+%s", origin, kh, base64.StdEncoding.EncodeToString(append([]byte{algEd25519}, privateKey.Seed()...)))
 	s, _ := note.NewSigner(skey)
+	fmt.Printf("- log note vkey: %s\n", vkey)
 	fmt.Printf("- log note key: %s\n", skey)
 
 	witSeed := make([]byte, ed25519.SeedSize)
@@ -69,7 +71,7 @@ func main() {
 	pkHash := sigsum.HashBytes(ss.PublicKey().(ssh.CryptoPublicKey).CryptoPublicKey().(ed25519.PublicKey))
 	fmt.Printf("- witness key hash: %s\n", hex.EncodeToString(pkHash[:]))
 	fmt.Printf("- witness key: %x\n", witKey)
-	pemKey, err := sshmarshal.MarshalPrivateKey(witKey, "")
+	pemKey, err := ssh.MarshalPrivateKey(witKey, "")
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/litewitness/testdata/loglist.txt

@@ -0,0 +1,74 @@
+# pull logs
+exec witnessctl pull-logs -source=log_list.0
+! stdout .
+! stderr .
+
+# pull logs again, with no changes
+exec witnessctl pull-logs -source=log_list.1
+! stdout .
+! stderr .
+
+# pull logs again, with a key change
+exec witnessctl pull-logs -source=log_list.2
+! stdout .
+stderr 'listed with a different key'
+
+# check state
+exec witnessctl list-logs
+cmp stdout list-logs.jsonl
+! stderr .
+
+
+-- log_list.0 --
+#
+# List:      10qps-100klogs
+# Revision:  123
+# Generated: YYYY-MM-DD HH:MM:SS UTC
+# Other undefined debug information.
+#
+logs/v0
+
+# gentest seed 467e9782a4eaa2bca9c9f5bcb067fb7bce1bfcae7b18f27bac0900ad93825300
+vkey sigsum.org/v1/tree/f48d4a1d0c6370ec189dc537f648ef3bb347b012fbd3c899a630a4cd2e9b8702+9bc54c7f+AVv6q3xDaHxI2aTemqEb7W6ZcbO7QbTqTr20thOqfqsw
+qpd 86400
+contact https://tlog.foo.org/contact
+unknown
+foo bar
+
+# gentest seed 07210b16c5409251a63305371bb0e86bbe9e005c41ca1963f9ed3d89931a8a16
+vkey sigsum.org/v1/tree/2ef59132082631d13e353b5ae49b22bc51a9bd59f41a2d570960a9658c1ed151+e2137795+ATp+37IPHc3SbPGzFMyZmPTOUlClk6PYPH+Ce5JiCb/h
+origin example.com/foo
+qpd 24
+contact sysadmin (at) bar.org
+
+-- log_list.1 --
+logs/v0
+
+# new log
+# gentest seed 92540ba271142b0011c96531067b8f1f7695fd16ff7d92833767f821ab983c4d
+vkey sigsum.org/v1/tree/fae7fd8f084f9e7a1482162da8a3e52b08e6c1bac74ab831d00eb5c983b84120+7f693d84+AUlxeri80AO7/4j/+OGo+5M2Sud0ktFg34uZl2fZnjJT
+
+# existing log, same key
+vkey sigsum.org/v1/tree/2ef59132082631d13e353b5ae49b22bc51a9bd59f41a2d570960a9658c1ed151+e2137795+ATp+37IPHc3SbPGzFMyZmPTOUlClk6PYPH+Ce5JiCb/h
+origin example.com/foo
+
+-- log_list.2 --
+logs/v0
+
+# new log
+# gentest seed 92540ba271142b0011c96531067b8f1f7695fd16ff7d92833767f821ab983c4d
+vkey sigsum.org/v1/tree/fae7fd8f084f9e7a1482162da8a3e52b08e6c1bac74ab831d00eb5c983b84120+7f693d84+AUlxeri80AO7/4j/+OGo+5M2Sud0ktFg34uZl2fZnjJT
+
+# existing log, same key
+vkey sigsum.org/v1/tree/2ef59132082631d13e353b5ae49b22bc51a9bd59f41a2d570960a9658c1ed151+e2137795+ATp+37IPHc3SbPGzFMyZmPTOUlClk6PYPH+Ce5JiCb/h
+origin example.com/foo
+
+# known log, different key
+# gentest seed cf8477f7158ac47b018a55fda4bd1af29b0e864cdd3fd495efa477639d0b690c
+vkey sigsum.org/v1/tree/80f121ea8f6364ca8c19212d7caf4e235d00aa34a98d86b60204c24f981a4933+a05ec139+ASZ0biMUCMjOMmt0NyD8g6Mt6RGZGXJU/ZpM5Oy0otex
+origin sigsum.org/v1/tree/f48d4a1d0c6370ec189dc537f648ef3bb347b012fbd3c899a630a4cd2e9b8702
+
+-- list-logs.jsonl --
+{"origin":"example.com/foo","size":0,"root_hash":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","keys":["sigsum.org/v1/tree/2ef59132082631d13e353b5ae49b22bc51a9bd59f41a2d570960a9658c1ed151+e2137795+ATp+37IPHc3SbPGzFMyZmPTOUlClk6PYPH+Ce5JiCb/h"]}
+{"origin":"sigsum.org/v1/tree/f48d4a1d0c6370ec189dc537f648ef3bb347b012fbd3c899a630a4cd2e9b8702","size":0,"root_hash":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","keys":["sigsum.org/v1/tree/f48d4a1d0c6370ec189dc537f648ef3bb347b012fbd3c899a630a4cd2e9b8702+9bc54c7f+AVv6q3xDaHxI2aTemqEb7W6ZcbO7QbTqTr20thOqfqsw"]}
+{"origin":"sigsum.org/v1/tree/fae7fd8f084f9e7a1482162da8a3e52b08e6c1bac74ab831d00eb5c983b84120","size":0,"root_hash":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","keys":["sigsum.org/v1/tree/fae7fd8f084f9e7a1482162da8a3e52b08e6c1bac74ab831d00eb5c983b84120+7f693d84+AUlxeri80AO7/4j/+OGo+5M2Sud0ktFg34uZl2fZnjJT"]}

cmd/witnessctl/loglist.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"golang.org/x/mod/sumdb/note"
+)
+
+func parseLogList(logList []byte, verbose bool) (map[string]string, error) {
+	logs := make(map[string]string)
+	var sawHeader bool
+	var vkey, origin string
+	finalizeLogEntry := func() {
+		if vkey == "" {
+			// The list may be empty.
+			return
+		}
+		defer func() {
+			vkey = ""
+			origin = ""
+		}()
+		v, err := note.NewVerifier(vkey)
+		if err != nil {
+			if verbose {
+				log.Printf("Skipping invalid vkey %q: %v", vkey, err)
+			}
+			return
+		}
+		if origin == "" {
+			origin = v.Name()
+		}
+		if logs[origin] != "" {
+			if verbose {
+				log.Printf("Skipping duplicate log entry for %q", origin)
+				log.Printf("    - %q", logs[origin])
+				log.Printf("    - %q (skipped)", vkey)
+			}
+			return
+		}
+		logs[origin] = vkey
+	}
+	for line := range strings.Lines(string(logList)) {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "#") {
+			// Comment line, skip.
+			continue
+		}
+		if line == "" {
+			// Empty line, skip.
+			continue
+		}
+		if !sawHeader {
+			// First non-comment, non-empty line is the header.
+			if line != "logs/v0" {
+				return nil, fmt.Errorf("invalid log list header: %q", line)
+			}
+			sawHeader = true
+			continue
+		}
+		key, value, _ := strings.Cut(line, " ")
+		if vkey == "" && key != "vkey" {
+			return nil, fmt.Errorf("expected vkey entry, got %q", line)
+		}
+		switch key {
+		case "vkey":
+			finalizeLogEntry()
+			if value == "" {
+				return nil, fmt.Errorf("empty vkey entry")
+			}
+			vkey = value
+		case "origin":
+			origin = value
+		default:
+			// Unknown key, ignore.
+		}
+	}
+	finalizeLogEntry()
+	return logs, nil
+}

cmd/witnessctl/witnessctl.go

@@ -5,8 +5,12 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"os"
+	"strings"
+	"time"
 
 	"filippo.io/torchwood/internal/witness"
 	"golang.org/x/mod/sumdb/note"
@@ -23,6 +27,7 @@ func usage() {
 	fmt.Println("    add-key -db <path> -origin <origin> -key <verifier key>")
 	fmt.Println("    del-key -db <path> -origin <origin> -key <verifier key>")
 	fmt.Println("    add-sigsum-log -db <path> -key <hex-encoded key>")
+	fmt.Println("    pull-logs -db <path> -source <witness url> [-verbose]")
 	fmt.Println("    list-logs -db <path>")
 	os.Exit(1)
 }
@@ -39,27 +44,38 @@ func main() {
 		fs.Parse(os.Args[2:])
 		db := openDB(*dbFlag)
 		addLog(db, *originFlag)
+		log.Printf("Added log %q.", *originFlag)
 
 	case "add-key":
 		originFlag := fs.String("origin", "", "log name")
 		keyFlag := fs.String("key", "", "verifier key")
 		fs.Parse(os.Args[2:])
 		db := openDB(*dbFlag)
+		checkKeyMatches(*originFlag, *keyFlag)
 		addKey(db, *originFlag, *keyFlag)
+		log.Printf("Added key %q for log %q.", *keyFlag, *originFlag)
 
 	case "del-key":
 		originFlag := fs.String("origin", "", "log name")
 		keyFlag := fs.String("key", "", "verifier key")
 		fs.Parse(os.Args[2:])
 		db := openDB(*dbFlag)
 		delKey(db, *originFlag, *keyFlag)
+		log.Printf("Deleted key %q for log %q.", *keyFlag, *originFlag)
 
 	case "add-sigsum-log":
 		keyFlag := fs.String("key", "", "hex-encoded key")
 		fs.Parse(os.Args[2:])
 		db := openDB(*dbFlag)
 		addSigsumLog(db, *keyFlag)
 
+	case "pull-logs":
+		sourceFlag := fs.String("source", "", "witness network log list URL or file path")
+		verboseFlag := fs.Bool("verbose", false, "verbose output")
+		fs.Parse(os.Args[2:])
+		db := openDB(*dbFlag)
+		pullLogs(db, *sourceFlag, *verboseFlag)
+
 	case "list-logs":
 		fs.Parse(os.Args[2:])
 		db := openDB(*dbFlag)
@@ -84,22 +100,23 @@ func addLog(db *sqlite.Conn, origin string) {
 		nil, origin, base64.StdEncoding.EncodeToString(treeHash[:])); err != nil {
 		log.Fatalf("Error adding log: %v", err)
 	}
-	log.Printf("Added log %q.", origin)
 }
 
-func addKey(db *sqlite.Conn, origin string, vk string) {
+func checkKeyMatches(origin string, vk string) {
 	v, err := note.NewVerifier(vk)
 	if err != nil {
 		log.Fatalf("Error parsing verifier key: %v", err)
 	}
 	if v.Name() != origin {
 		log.Printf("Warning: verifier key name %q does not match origin %q.", v.Name(), origin)
 	}
-	err = sqlitex.Exec(db, "INSERT INTO key (origin, key) VALUES (?, ?)", nil, origin, vk)
+}
+
+func addKey(db *sqlite.Conn, origin string, vk string) {
+	err := sqlitex.Exec(db, "INSERT INTO key (origin, key) VALUES (?, ?)", nil, origin, vk)
 	if err != nil {
 		log.Fatalf("Error adding key: %v", err)
 	}
-	log.Printf("Added key %q.", vk)
 }
 
 func delKey(db *sqlite.Conn, origin string, vk string) {
@@ -110,7 +127,6 @@ func delKey(db *sqlite.Conn, origin string, vk string) {
 	if db.Changes() == 0 {
 		log.Fatalf("Key %q not found.", vk)
 	}
-	log.Printf("Deleted key %q.", vk)
 }
 
 func addSigsumLog(db *sqlite.Conn, keyFlag string) {
@@ -129,6 +145,83 @@ func addSigsumLog(db *sqlite.Conn, keyFlag string) {
 	}
 	addLog(db, origin)
 	addKey(db, origin, vk)
+	log.Printf("Added Sigsum log %q with key %q.", origin, vk)
+}
+
+func pullLogs(db *sqlite.Conn, source string, verbose bool) {
+	var logList []byte
+	if strings.HasPrefix(source, "https://") {
+		client := http.Client{Timeout: 30 * time.Second}
+		resp, err := client.Get(source)
+		if err != nil {
+			log.Fatalf("Error fetching log list: %v", err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			log.Fatalf("Error fetching log list: HTTP %d", resp.StatusCode)
+		}
+		logList, err = io.ReadAll(resp.Body)
+		if err != nil {
+			log.Fatalf("Error reading log list: %v", err)
+		}
+	} else {
+		var err error
+		logList, err = os.ReadFile(source)
+		if err != nil {
+			log.Fatalf("Error reading log list: %v", err)
+		}
+	}
+	logs, err := parseLogList(logList, verbose)
+	if err != nil {
+		log.Fatalf("Error parsing log list: %v", err)
+	}
+	for origin, vkey := range logs {
+		keys, exists := logKeys(db, origin)
+		if exists {
+			if keys[vkey] {
+				// Key already exists, nothing to do.
+				if verbose {
+					log.Printf("Log %q with key %q already exists, skipping.", origin, vkey)
+				}
+				continue
+			}
+			// Key is different, warn.
+			log.Printf("Warning: log %q is listed with a different key than the one in the database.\n", origin)
+			log.Printf("  - existing keys:\n")
+			for k := range keys {
+				log.Printf("    - %q\n", k)
+			}
+			log.Printf("  - new key:\n")
+			log.Printf("    - %q\n", vkey)
+			continue
+		}
+		// New log, add it.
+		addLog(db, origin)
+		addKey(db, origin, vkey)
+		if verbose {
+			log.Printf("Added log %q with key %q.", origin, vkey)
+		}
+	}
+}
+
+func logKeys(db *sqlite.Conn, origin string) (keys map[string]bool, exists bool) {
+	keys = make(map[string]bool)
+	if err := sqlitex.Exec(db, "SELECT 1 FROM log WHERE origin = ?", func(stmt *sqlite.Stmt) error {
+		exists = true
+		return nil
+	}, origin); err != nil {
+		log.Fatalf("Error looking for log %q: %v", origin, err)
+	}
+	if !exists {
+		return keys, false
+	}
+	if err := sqlitex.Exec(db, "SELECT key FROM key WHERE origin = ?", func(stmt *sqlite.Stmt) error {
+		keys[stmt.ColumnText(0)] = true
+		return nil
+	}, origin); err != nil {
+		log.Fatalf("Error querying keys: %v", err)
+	}
+	return keys, true
 }
 
 func listLogs(db *sqlite.Conn) {