Enhance wallet backup and recovery with a mnemonic hsm_secret and standard taproot wallet derivations

.msggen.json

@@ -1546,7 +1546,8 @@
         },
         "ExposesecretResponse": {
             "ExposeSecret.codex32": 2,
-            "ExposeSecret.identifier": 1
+            "ExposeSecret.identifier": 1,
+            "ExposeSecret.mnemonic": 3
         },
         "FeeratesOnchainFeeEstimates": {
             "Feerates.onchain_fee_estimates.htlc_success_satoshis": 5,
@@ -6760,6 +6761,10 @@
             "added": "v24.11",
             "deprecated": null
         },
+        "ExposeSecret.mnemonic": {
+            "added": "v25.12",
+            "deprecated": null
+        },
         "ExposeSecret.passphrase": {
             "added": "v24.11",
             "deprecated": null

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2602-gfd3fd70c
+CCAN version: init-2605-gc47bf0d9

ccan/ccan/cdump/_info

@@ -33,7 +33,7 @@
  *	size_t j;
  *
  * 	// Read code from stdin.
- * 	code = grab_file(NULL, NULL);
+ * 	code = grab_file_str(NULL, NULL);
  *
  * 	defs = cdump_extract(NULL, code, &problems);
  * 	if (!defs)

ccan/ccan/cdump/tools/cdump-enumstr.c

@@ -27,7 +27,7 @@ int main(int argc, char *argv[])
 	if (argc < 2)
 		errx(1, "Usage: cdump-enumstr <filename> [<enums>...]");
 
-	code = grab_file(NULL, streq(argv[1], "-") ? NULL : argv[1]);
+	code = grab_file_str(NULL, streq(argv[1], "-") ? NULL : argv[1]);
 	if (!code)
 		err(1, "Reading %s", argv[1]);
 

ccan/ccan/crc32c/benchmark/bench.c

@@ -31,7 +31,7 @@ int main(int argc, char *argv[])
 	if (argc < 2 || (runs = atol(argv[1])) == 0)
 		errx(1, "Usage: bench <num-runs> [<file>]");
 
-	p = grab_file(NULL, argv[2]);
+	p = grab_file_str(NULL, argv[2]);
 	if (!p)
 		err(1, "Reading %s", argv[2] ? argv[2] : "<stdin>");
 	len = tal_count(p) - 1;

ccan/ccan/htable/tools/hsearchspeed.c

@@ -29,7 +29,7 @@ int main(int argc, char *argv[])
 	char **w;
 	ENTRY *words, *misswords;
 
-	w = tal_strsplit(NULL, grab_file(NULL,
+	w = tal_strsplit(NULL, grab_file_str(NULL,
 					 argv[1] ? argv[1] : "/usr/share/dict/words"), "\n", STR_NO_EMPTY);
 	num = tal_count(w) - 1;
 	printf("%zu words\n", num);

ccan/ccan/htable/tools/stringspeed.c

@@ -48,7 +48,7 @@ int main(int argc, char *argv[])
 	struct htable_str ht;
 	char **words, **misswords;
 
-	words = tal_strsplit(NULL, grab_file(NULL,
+	words = tal_strsplit(NULL, grab_file_str(NULL,
 					     argv[1] ? argv[1] : "/usr/share/dict/words"), "\n",
 			     STR_NO_EMPTY);
 	htable_str_init(&ht);

ccan/ccan/rune/test/run.c

@@ -39,7 +39,7 @@ int main(void)
 	mr = rune_new(NULL, secret_zero, sizeof(secret_zero), NULL); 
 
 	/* Python runes library generates test vectors */
-	vecs = grab_file(mr, "test/test_vectors.csv");
+	vecs = grab_file_str(mr, "test/test_vectors.csv");
 	assert(vecs);
 	lines = tal_strsplit(mr, take(vecs), "\n", STR_NO_EMPTY);
 

ccan/ccan/strset/_info

@@ -32,7 +32,7 @@
  *		char *file, *word;
  *
  *		strset_init(&words);
- *		file = grab_fd(NULL, 0);
+ *		file = grab_fd_str(NULL, 0);
  *		if (!file)
  *			err(1, "Reading stdin");
  *

ccan/ccan/strset/tools/cbspeed.c

@@ -395,7 +395,7 @@ int main(int argc, char *argv[])
 	critbit0_tree ct;
 	char **words, **misswords;
 
-	words = tal_strsplit(NULL, grab_file(NULL,
+	words = tal_strsplit(NULL, grab_file_str(NULL,
 					 argv[1] ? argv[1] : "/usr/share/dict/words"), "\n", STR_NO_EMPTY);
 	ct.root = NULL;
 	num = tal_count(words) - 1;

ccan/ccan/strset/tools/speed.c

@@ -46,7 +46,7 @@ int main(int argc, char *argv[])
 	struct strset set;
 	char **words, **misswords;
 
-	words = tal_strsplit(NULL, grab_file(NULL,
+	words = tal_strsplit(NULL, grab_file_str(NULL,
 					     argv[1] ? argv[1] : "/usr/share/dict/words"),
 			     "\n", STR_NO_EMPTY);
 	strset_init(&set);

ccan/ccan/tal/grab_file/_info

@@ -20,7 +20,7 @@
  *
  *		if (argc > 2)
  *			err(1, "Takes 0 or 1 arguments");
- *		file = grab_file(NULL, argv[1]);
+ *		file = grab_file_str(NULL, argv[1]);
  *		if (!file)
  *			err(1, "Could not read file %s", argv[1]);
  *		if (strlen(file)+1 != tal_count(file))
@@ -45,6 +45,7 @@ int main(int argc, char *argv[])
 		return 1;
 
 	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
 		printf("ccan/tal\n");
 		printf("ccan/noerr\n");
 		return 0;

ccan/ccan/tal/grab_file/grab_file.c

@@ -8,7 +8,7 @@
 #include <errno.h>
 #include <fcntl.h>
 
-void *grab_fd(const void *ctx, int fd)
+static void *grab_fd_internal(const void *ctx, int fd, bool add_nul_term)
 {
 	int ret;
 	size_t max, size;
@@ -22,7 +22,7 @@ void *grab_fd(const void *ctx, int fd)
 	else
 		max = 16384;
 
-	buffer = tal_arr(ctx, char, max+1);
+	buffer = tal_arr(ctx, char, max+add_nul_term);
 	while ((ret = read(fd, buffer + size, max - size)) != 0) {
 		if (ret < 0) {
 			if (errno == EINTR)
@@ -35,19 +35,20 @@ void *grab_fd(const void *ctx, int fd)
 			if (extra > 1024 * 1024)
 				extra = 1024 * 1024;
 
-			if (!tal_resize(&buffer, max+extra+1))
+			if (!tal_resize(&buffer, max+extra+add_nul_term))
 				return NULL;
 
 			max += extra;
 		}
 	}
-	buffer[size] = '\0';
-	tal_resize(&buffer, size+1);
+	if (add_nul_term)
+		buffer[size] = '\0';
+	tal_resize(&buffer, size+add_nul_term);
 
 	return buffer;
 }
 
-void *grab_file(const void *ctx, const char *filename)
+static void *grab_file_internal(const void *ctx, const char *filename, bool add_nul_term)
 {
 	int fd;
 	char *buffer;
@@ -60,7 +61,27 @@ void *grab_file(const void *ctx, const char *filename)
 	if (fd < 0)
 		return NULL;
 
-	buffer = grab_fd(ctx, fd);
+	buffer = grab_fd_internal(ctx, fd, add_nul_term);
 	close_noerr(fd);
 	return buffer;
 }
+
+void *grab_fd_raw(const void *ctx, int fd)
+{
+	return grab_fd_internal(ctx, fd, false);
+}
+
+void *grab_fd_str(const void *ctx, int fd)
+{
+	return grab_fd_internal(ctx, fd, true);
+}
+
+void *grab_file_str(const void *ctx, const char *filename)
+{
+	return grab_file_internal(ctx, filename, true);
+}
+
+void *grab_file_raw(const void *ctx, const char *filename)
+{
+	return grab_file_internal(ctx, filename, false);
+}

ccan/ccan/tal/grab_file/grab_file.h

@@ -2,9 +2,31 @@
 #ifndef CCAN_TAL_GRAB_FILE_H
 #define CCAN_TAL_GRAB_FILE_H
 #include <stdio.h> // For size_t
+#include <ccan/compiler/compiler.h>
 
 /**
- * grab_fd - read all of a file descriptor into memory
+ * grab_fd_raw - read all of a file descriptor into memory WITHOUT adding a nul.
+ * @ctx: the context to tallocate from (often NULL)
+ * @fd: the file descriptor to read from
+ * @size: the (optional) size of the file
+ *
+ * This function reads from the given file descriptor until no more
+ * input is available.  The content is talloced off @ctx, and the
+ * tal_count() is the size in bytes.
+ *
+ * Note that this does *not* currently exit on EINTR, but continues
+ * reading. *
+ * Example:
+ *	// Return the first line.
+ *	static char *read_stdin_all(void)
+ *	{
+ *		return grab_fd_raw(NULL, 0);
+ *	}
+ */
+void *grab_fd_raw(const void *ctx, int fd);
+
+/**
+ * grab_fd_str - read all of a file descriptor into memory with a NUL terminator.
  * @ctx: the context to tallocate from (often NULL)
  * @fd: the file descriptor to read from
  *
@@ -25,18 +47,26 @@
  *	{
  *		char **lines, *all;
  *
- *		all = grab_fd(NULL, 0);
+ *		all = grab_fd_str(NULL, 0);
  *		if (!all)
  *			return NULL;
  *		lines = tal_strsplit(NULL, all, "\n", STR_EMPTY_OK);
  *		tal_free(all);
  *		return lines;
  *	}
  */
-void *grab_fd(const void *ctx, int fd);
+void *grab_fd_str(const void *ctx, int fd);
+
+/* Deprecated synonym for grab_fd_str */
+static inline void *grab_fd(const void *ctx, int fd)
+	WARN_DEPRECATED;
+static inline void *grab_fd(const void *ctx, int fd)
+{
+	return grab_fd_str(ctx, fd);
+}
 
 /**
- * grab_file - read all of a file (or stdin) into memory
+ * grab_file_str - read all of a file (or stdin) into memory with a NUL terminator
  * @ctx: the context to tallocate from (often NULL)
  * @filename: the file to read (NULL for stdin)
  *
@@ -51,13 +81,40 @@ void *grab_fd(const void *ctx, int fd);
  *	{
  *		char **lines, *all;
  *
- *		all = grab_file(NULL, filename);
+ *		all = grab_file_str(NULL, filename);
  *		if (!all)
  *			return NULL;
  *		lines = tal_strsplit(NULL, all, "\n", STR_EMPTY_OK);
  *		tal_free(all);
  *		return lines;
  *	}
  */
-void *grab_file(const void *ctx, const char *filename);
+void *grab_file_str(const void *ctx, const char *filename);
+
+/**
+ * grab_file_raw - read all of a file (or stdin) into memory WITHOUT a NUL terminator
+ * @ctx: the context to tallocate from (often NULL)
+ * @filename: the file to read (NULL for stdin)
+ * @size: the (optional) size of the file
+ *
+ * This function reads from the given file until no more input is
+ * available.  The content is talloced off @ctx, and the tal_count()
+ * is the size in bytes.
+ *
+ * Example:
+ *	static char *read_file_all(const char *filename)
+ *	{
+ *		return grab_file_raw(NULL, filename);
+ *	}
+ */
+void *grab_file_raw(const void *ctx, const char *filename);
+
+/* Deprecated synonym for grab_file_str */
+static inline void *grab_file(const void *ctx, const char *filename)
+	WARN_DEPRECATED;
+static inline void *grab_file(const void *ctx, const char *filename)
+{
+	return grab_file_str(ctx, filename);
+}
+
 #endif /* CCAN_TAL_GRAB_FILE_H */

ccan/ccan/tal/grab_file/test/run-grab.c

@@ -1,4 +1,4 @@
-/* This is test for grab_file() function
+/* This is test for grab_file_str() and grab_file_raw() functions
  */
 #include <ccan/tal/grab_file/grab_file.h>
 #include <stdlib.h>
@@ -13,25 +13,31 @@ int
 main(void)
 {
 	unsigned int	i;
-	char 		**split, *str;
+	char 		**split, *str, *raw;
 	int 		length;
 	struct 		stat st;
 
-	str = grab_file(NULL, "test/run-grab.c");
+	plan_tests(5);
+	str = grab_file_str(NULL, "test/run-grab.c");
 	split = tal_strsplit(str, str, "\n", STR_EMPTY_OK);
 	length = strlen(split[0]);
-	ok1(!strcmp(split[0], "/* This is test for grab_file() function"));
+	ok1(!strcmp(split[0], "This is test for grab_file_str() and grab_file_raw() functions"));
 	for (i = 1; split[i]; i++)
 		length += strlen(split[i]);
-	ok1(!strcmp(split[i-1], "/* End of grab_file() test */"));
+	ok1(!strcmp(split[i-1], "/* End of grab_file.c test */"));
 	if (stat("test/run-grab.c", &st) != 0)
 		/* FIXME: ditto */
 		if (stat("ccan/tal/grab_file/test/run-grab.c", &st) != 0)
 			err(1, "Could not stat self");
 	ok1(st.st_size == length + i);
+
+	/* Raw does not nul term */
+	raw = grab_file_raw(str, "test/run-grab.c");
+	ok1(tal_count(raw) + 1 == tal_count(str));
+	ok1(memcmp(raw, str, tal_bytelen(raw)) == 0);
 	tal_free(str);
 
 	return 0;
 }
 
-/* End of grab_file() test */
+/* End of grab_file.c test */

ccan/ccan/tal/str/_info

@@ -25,7 +25,7 @@
  *		if (argc > 2)
  *			errx(1, "Takes 0 or 1 arguments");
  *		// Grab lines in file.
- *		textfile = grab_file(NULL, argv[1]);
+ *		textfile = grab_file_str(NULL, argv[1]);
  *		if (!textfile)
  *			err(1, "Failed reading %s", argv[1]);
  *		lines = tal_strsplit(textfile, textfile, "\n", STR_EMPTY_OK);

ccan/tools/configurator/configurator.c

@@ -137,8 +137,8 @@ static const struct test base_tests[] = {
 	  "DEFINES_FUNC", NULL, NULL,
 	  "static int __attribute__((const)) func(int x) { return x; }" },
 	{ "HAVE_ATTRIBUTE_DEPRECATED", "__attribute__((deprecated)) support",
-	  "DEFINES_FUNC", NULL, NULL,
-	  "static int __attribute__((deprecated)) func(int x) { return x; }" },
+	  "OUTSIDE_MAIN", NULL, NULL,
+	  "int __attribute__((deprecated)) depr(int x);" },
 	{ "HAVE_ATTRIBUTE_NONNULL", "__attribute__((nonnull)) support",
 	  "DEFINES_FUNC", NULL, NULL,
 	  "static char *__attribute__((nonnull)) func(char *p) { return p; }" },

common/Makefile

@@ -45,7 +45,7 @@ COMMON_SRC_NOGEN :=				\
 	common/hash_u5.c			\
 	common/hmac.c				\
 	common/hsm_capable.c			\
-	common/hsm_encryption.c			\
+	common/hsm_secret.c			\
 	common/htlc_state.c			\
 	common/htlc_trim.c			\
 	common/htlc_tx.c			\