Update documentation using deterministic generation

.github/workflows/ci.yaml

@@ -481,8 +481,7 @@ jobs:
           uv run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
+    name: Update examples in doc schemas
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     strategy:
@@ -608,6 +607,7 @@ jobs:
       - check-units
       - integration-valgrind
       - integration-sanitizers
+      - update-docs-examples
       - min-btc-support
     if: ${{ always() }}
     steps:

Makefile

@@ -569,7 +569,7 @@ check-tmpctx:
 	@if git grep -n 'tal_free[(]tmpctx)' | grep -Ev '^ccan/|/test/|^common/setup.c:|^common/utils.c:'; then echo "Don't free tmpctx!">&2; exit 1; fi
 
 check-discouraged-functions:
-	@if git grep -E "[^a-z_/](fgets|fputs|gets|scanf|sprintf)\(" -- "*.c" "*.h" ":(exclude)ccan/" ":(exclude)contrib/"; then exit 1; fi
+	@if git grep -nE "[^a-z_/](fgets|fputs|gets|scanf|sprintf|randombytes_buf|time_now)\(" -- "*.c" "*.h" ":(exclude)ccan/" ":(exclude)contrib/" | grep -Fv '/* discouraged:'; then exit 1; fi
 
 # Don't access amount_msat and amount_sat members directly without a good reason
 # since it risks overflow.
@@ -582,14 +582,18 @@ repeat-doc-examples:
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
 		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
+		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) pytest -vvv tests/autogenerate-rpc-examples.py; \
 		git diff >> tests/autogenerate-examples-repeat.log; \
 		git reset --hard; \
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
 	done
 
 update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
+	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
+
+# If you changed tests/autogenerate-rpc-examples.py to require new blocks, you have to run this:
+update-doc-examples-newchain:
+	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) REGENERATE_BLOCKCHAIN=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
 
 check-doc-examples: update-doc-examples
 	git diff --exit-code HEAD

bitcoin/script.c

@@ -187,17 +187,6 @@ u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr)
 	return script;
 }
 
-u8 *scriptpubkey_opreturn_padded(const tal_t *ctx)
-{
-	u8 *script = tal_arr(ctx, u8, 0);
-	u8 random[20];
-	randombytes_buf(random, sizeof(random));
-
-	add_op(&script, OP_RETURN);
-	script_push_bytes(&script, random, sizeof(random));
-	return script;
-}
-
 /* Create an input script which spends p2pkh */
 u8 *bitcoin_redeem_p2pkh(const tal_t *ctx, const struct pubkey *pubkey,
 			 const struct bitcoin_signature *sig)

bitcoin/script.h

@@ -27,12 +27,6 @@ u8 *scriptpubkey_p2sh_hash(const tal_t *ctx, const struct ripemd160 *redeemhash)
 /* Create an output script using p2pkh */
 u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr);
 
-/* Create a prunable output script with 20 random bytes.
- * This is needed since a spend from a p2wpkh to an `OP_RETURN` without
- * any other outputs would result in a transaction smaller than the
- * minimum size.  */
-u8 *scriptpubkey_opreturn_padded(const tal_t *ctx);
-
 /* Create an input script which spends p2pkh */
 u8 *bitcoin_redeem_p2pkh(const tal_t *ctx, const struct pubkey *pubkey,
 			 const struct bitcoin_signature *sig);

bitcoin/short_channel_id.c

@@ -1,7 +1,7 @@
 #include "config.h"
 #include <bitcoin/short_channel_id.h>
 #include <ccan/tal/str/str.h>
-#include <sodium/randombytes.h>
+#include <common/randbytes.h>
 #include <stdio.h>
 #include <wire/wire.h>
 
@@ -104,6 +104,6 @@ struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max
 struct short_channel_id random_scid(void)
 {
 	struct short_channel_id scid;
-	randombytes_buf(&scid, sizeof(scid));
+	randbytes(&scid, sizeof(scid));
 	return scid;
 }

bitcoin/test/Makefile

@@ -2,7 +2,7 @@ BITCOIN_TEST_SRC := $(wildcard bitcoin/test/run-*.c)
 BITCOIN_TEST_OBJS := $(BITCOIN_TEST_SRC:.c=.o)
 BITCOIN_TEST_PROGRAMS := $(BITCOIN_TEST_OBJS:.o=)
 
-BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o common/autodata.o
+BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o common/autodata.o common/clock_time.o
 
 $(BITCOIN_TEST_PROGRAMS): $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o
 $(BITCOIN_TEST_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(BITCOIN_SRC)

bitcoin/test/run-secret_eq_consttime.c

@@ -22,7 +22,7 @@ static struct timerel const_time_test(struct secret *s1,
 				      struct secret *s2,
 				      size_t off)
 {
-	struct timeabs start, end;
+	struct timemono start, end;
 	int result = 0;
 
 	memset(s1, 0, RUNS * sizeof(*s1));
@@ -31,16 +31,16 @@ static struct timerel const_time_test(struct secret *s1,
 	for (size_t i = 0; i < RUNS; i++)
 		s2[i].data[off] = i;
 
-	start = time_now();
+	start = time_mono();
 	for (size_t i = 0; i < RUNS; i++)
 		result += secret_eq_consttime(&s1[i], &s2[i]);
-	end = time_now();
+	end = time_mono();
 
 	if (result != RUNS / 256)
 		errx(1, "Expected %u successes at offset %zu, not %u!",
 		     RUNS / 256, off, result);
 
-	return time_between(end, start);
+	return timemono_between(end, start);
 }
 
 static inline bool secret_eq_nonconst(const struct secret *a,
@@ -53,7 +53,7 @@ static struct timerel nonconst_time_test(struct secret *s1,
 					 struct secret *s2,
 					 size_t off)
 {
-	struct timeabs start, end;
+	struct timemono start, end;
 	int result = 0;
 
 	memset(s1, 0, RUNS * sizeof(*s1));
@@ -62,16 +62,16 @@ static struct timerel nonconst_time_test(struct secret *s1,
 	for (size_t i = 0; i < RUNS; i++)
 		s2[i].data[off] = i;
 
-	start = time_now();
+	start = time_mono();
 	for (size_t i = 0; i < RUNS; i++)
 		result += secret_eq_nonconst(&s1[i], &s2[i]);
-	end = time_now();
+	end = time_mono();
 
 	if (result != RUNS / 256)
 		errx(1, "Expected %u successes at offset %zu, not %u!",
 		     RUNS / 256, off, result);
 
-	return time_between(end, start);
+	return timemono_between(end, start);
 }
 
 static struct secret *s1, *s2;

channeld/Makefile

@@ -48,6 +48,7 @@ CHANNELD_COMMON_OBJS :=				\
 	common/channel_config.o			\
 	common/channel_id.o			\
 	common/channel_type.o			\
+	common/clock_time.o			\
 	common/cryptomsg.o			\
 	common/daemon.o				\
 	common/daemon_conn.o			\
@@ -84,6 +85,7 @@ CHANNELD_COMMON_OBJS :=				\
 	common/psbt_open.o			\
 	common/psbt_internal.o			\
 	common/pseudorand.o			\
+	common/randbytes.o			\
 	common/read_peer_msg.o			\
 	common/setup.o				\
 	common/status.o				\

channeld/full_channel.c

@@ -17,12 +17,6 @@
   /* Needs to be at end, since it doesn't include its own hdrs */
   #include "full_channel_error_names_gen.h"
 
-static void memleak_help_htlcmap(struct htable *memtable,
-				 struct htlc_map *htlcs)
-{
-	memleak_scan_htable(memtable, &htlcs->raw);
-}
-
 /* This is a dangerous thing!  Because we apply HTLCs in many places
  * in bulk, we can temporarily go negative.  You must check balance_ok()
  * at the end! */
@@ -113,11 +107,9 @@ struct channel *new_full_channel(const tal_t *ctx,
 						      option_wumbo,
 						      opener);
 
-	if (channel) {
-		channel->htlcs = tal(channel, struct htlc_map);
-		htlc_map_init(channel->htlcs);
-		memleak_add_helper(channel->htlcs, memleak_help_htlcmap);
-	}
+	if (channel)
+		channel->htlcs = new_htable(channel, htlc_map);
+
 	return channel;
 }
 

channeld/test/Makefile

@@ -21,6 +21,7 @@ CHANNELD_TEST_COMMON_OBJS :=			\
 	common/msg_queue.o			\
 	common/permute_tx.o			\
 	common/pseudorand.o			\
+	common/randbytes.o			\
 	common/setup.o				\
 	common/utils.o
 

channeld/test/run-commit_tx.c

@@ -15,6 +15,7 @@ static bool print_superverbose;
 #include <common/channel_id.h>
 #include <common/daemon.h>
 #include <common/key_derive.h>
+#include <common/memleak.h>
 #include <common/setup.h>
 #include <common/status.h>
 

cli/Makefile

@@ -10,6 +10,7 @@ LIGHTNING_CLI_COMMON_OBJS :=			\
 	common/configdir.o			\
 	common/configvar.o			\
 	common/json_parse_simple.o		\
+	common/memleak.o			\
 	common/status_levels.o			\
 	common/utils.o				\
 	common/version.o

cli/test/Makefile

@@ -15,6 +15,7 @@ CLI_TEST_COMMON_OBJS :=				\
 	common/htlc_state.o			\
 	common/json_parse_simple.o		\
 	common/pseudorand.o			\
+	common/randbytes.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\
 	common/setup.o				\

closingd/Makefile

@@ -25,6 +25,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/bigsize.o			\
 	common/bip32.o				\
 	common/channel_id.o			\
+	common/clock_time.o			\
 	common/close_tx.o			\
 	common/cryptomsg.o			\
 	common/daemon.o				\
@@ -47,6 +48,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/psbt_keypath.o			\
 	common/psbt_open.o			\
 	common/pseudorand.o			\
+	common/randbytes.o			\
 	common/status_wiregen.o			\
 	common/read_peer_msg.o			\
 	common/setup.o				\

common/Makefile

@@ -21,6 +21,7 @@ COMMON_SRC_NOGEN :=				\
 	common/channel_config.c			\
 	common/channel_id.c			\
 	common/channel_type.c			\
+	common/clock_time.c			\
 	common/close_tx.c			\
 	common/codex32.c			\
 	common/coin_mvt.c			\
@@ -85,6 +86,7 @@ COMMON_SRC_NOGEN :=				\
 	common/psbt_keypath.c			\
 	common/psbt_open.c			\
 	common/pseudorand.c			\
+	common/randbytes.c			\
 	common/random_select.c			\
 	common/read_peer_msg.c			\
 	common/route.c				\

common/blindedpath.h

@@ -23,7 +23,7 @@ struct tlv_encrypted_data_tlv_payment_relay;
  * @next_path_privkey: (out) e(i+1), the next blinding secret (optional)
  * @node_alias: (out) the blinded pubkey of the node to tell the recipient.
  *
- * You create a blinding secret using randombytes_buf(), then call this
+ * You create a blinding secret using randbytes(), then call this
  * iteratively for each node in the path.
  */
 u8 *encrypt_tlv_encrypted_data(const tal_t *ctx,

common/clock_time.c

@@ -0,0 +1,36 @@
+#include "config.h"
+#include <assert.h>
+#include <common/clock_time.h>
+
+static bool used = false;
+static struct timeabs dev_override;
+
+bool clock_time_overridden(void)
+{
+	return dev_override.ts.tv_sec != 0;
+}
+
+struct timeabs clock_time(void)
+{
+	used = true;
+	if (!clock_time_overridden())
+		return time_now(); /* discouraged: use clock_time so we can override */
+
+	return dev_override;
+}
+
+struct timeabs clock_time_progresses_(u64 *progress)
+{
+	if (!clock_time_overridden())
+		return clock_time();
+
+	return timeabs_add(dev_override, time_from_sec((*progress)++));
+}
+
+void dev_override_clock_time(struct timeabs now)
+{
+	assert(!used);
+
+	dev_override = now;
+	assert(clock_time_overridden());
+}

common/clock_time.h

@@ -0,0 +1,19 @@
+#ifndef LIGHTNING_COMMON_CLOCK_TIME_H
+#define LIGHTNING_COMMON_CLOCK_TIME_H
+#include "config.h"
+#include <ccan/short_types/short_types.h>
+#include <ccan/time/time.h>
+
+/* We use this instead of time_now, for overriding when we want reproducibility */
+struct timeabs clock_time(void);
+
+/* If you need a clock that progresses even when reproducible, use this. */
+#define clock_time_progresses() ({static u64 progress; clock_time_progresses_(&progress);})
+struct timeabs clock_time_progresses_(u64 *progress);
+
+/* dev setting to override time */
+void dev_override_clock_time(struct timeabs now);
+
+/* Did someone override time? */
+bool clock_time_overridden(void);
+#endif /* LIGHTNING_COMMON_CLOCK_TIME_H */