VLS Modifications to CLN v24.02

.github/scripts/setup.sh

@@ -15,6 +15,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      build-essential \
      clang \
      cppcheck \
+     curl \
      docbook-xml \
      eatmydata \
      gcc-aarch64-linux-gnu \

CHANGELOG.md

@@ -3,6 +3,108 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [24.02] - 2024-02-27: "uint needs signature"
+
+This release named by Erik de Smedt (@ErikDeSmedt).
+
+### Added
+
+ - JSON-RPC: `listpeerchannels` field `last_stable_connection` and `listclosedchannels` field `last_stable_connection`  showing when we last held an established channel for a minute or more. ([#6904])
+ - JSON-RPC: `listpeerchannels` new field `reestablished` set once we've exchanged `channel_reestablish` messages. ([#6904])
+ - JSON-RPC: `fundchannel`, `multifundchannel`, `fundchannel_start` and `openchannel_init`: new field `channel_type`. ([#6864])
+ - JSON-RPC: `fundchannel` and `multifundchannel` now take an optional `channel_type` parameter. ([#6864])
+ - JSON-RPC: `fundchannel_start` and `openchannel_init` now take an optional `channel_type` parameter. ([#6864])
+ - Plugin: options and commands can specify deprecation start (and optional end) versions. ([#6936])
+ - Plugins: rpcmethods and options can set `deprecated` to a pair of version strings, not just a boolean. ([#6936])
+ - JSON-RPC: `deprecations` to enable/disable deprecated APIs from this caller. ([#6936])
+ - config: `i-promise-to-fix-broken-api-user` allows for a one-release re-enablement of long-deprecated features. ([#6936])
+ - Protocol: `option_anchors_zero_fee_htlc_tx` enabled, no longer experimental. ([#6785])
+ - Plugins: `deprecated_oneshot` notifiction subscription to change deprecated status for a single command. ([#6936])
+ - hsmd: Added hsmd_forget_channel to enable explicit channel deletion. ([#6987]) ([#6988])
+ - tracing: It is now possible to inject a parent for the startup trace by setting the `CLN_TRACEPARENT` envvar ([#6912])
+ - Plugins: notification custommsg for receiving an unknown protocol message ([#6899])
+ - JSON-RPC: `listpeerchannels` now shows gossip update contents (even if channel unannounced). ([#6869])
+
+
+### Changed
+
+ - lightningd: Speed up blocksync by not parsing unused parts of the transactions ([#6984])
+ - Config: `experimental-anchors` now does nothing (it's enabled by default). ([#6785])
+ - reckless installs python plugins into virtual environments ([#7018])
+ - Protocol: `option_gossip_queries` is now required (advertized by all but 16 nodes) ([#6864])
+ - Protocol: `option_gossip_queries` is now required (advertized by all but 11 nodes) ([#6864])
+ - Protocol: `option_data_loss_protect` is now required (advertized by all but 11 nodes) ([#6864])
+ - core: Processing blocks should now be faster ([#6983])
+ - Enable optimizations for libwally/libsecp256k1-zkp ([#6983])
+ - Update libwally to 1.2.0 ([#6983])
+ - pyln-client: no longer autoconverts _msat field to Millisatoshi class (leaves as ints). ([#6865])
+ - JSON-RPC: `listnodes` no longer shows private (peer) nodes: use listpeers ([#6869])
+ - startup_regtest.sh: `fund_nodes` will now make balanced channels ([#6898])
+ - startup_regtest.sh PATH_TO_LIGHTNING + PATH_TO_BITCOIN are no more. Use LIGHTNING_BIN and BITCOIN_DIR ([#6898])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - `listchannels` no longer uses local knowledge to set `active` to false if disconnected. ([#6869])
+ - JSON-RPC: `listchannels` listing private channels: use listpeerchannels ([#6869])
+
+
+### Removed
+
+ - Protocol: we no longer ratelimit gossip messages by channel, making our code far simpler. ([#6941])
+ - Config: `disable-ip-discovery` (deprecated in v23.02): use `announce-addr-discovered` ([#6936])
+ - wallet: removal of p2sh-segwit addresses; newaddr won't issue them, we won't watch them for new funds (deprecated in *23.02*) ([#6936])
+ - JSON-RPC: `invoice`, `sendonion`, `sendpay`, `pay`, `keysend`, `fetchinvoice`, `sendinvoice`: `msatoshi` argument (deprecated 0.12.0). Use `amount_msat`. ([#6936])
+
+
+### Fixed
+
+ - JSON-RPC: `close` with `destination` works even if prior `destination` was rejected. ([#7072])
+ - JSON-RPC: `channel_type` reflects option_zeroconf if explicitly negotiated. ([#6864])
+ - configure: We now respect the `PKG_CONFIG_PATH` environment variable ([#6967])
+ - Default bolt11 invoices are payable by LND nodes. ([#6957])
+ - channeld: We could crash `closingd` by sending it a `channeld` message ([#6937])
+ - `bkpr-listbalances` would crash for nodes on signet with payments in channels, because onchain events were using a different currency than inchannel events. ([#6888])
+ - Hsmtool: Fix segmentation fault when calling `getcodexsecret` without id. ([#6895])
+
+
+### EXPERIMENTAL
+
+ - JSON-RPC: Deprecated `offer` parameter `recurrence_base` with `@` prefix: use `recurrence_start_any_period`. ([#7034])
+ - JSON-RPC: Added `offer` parameter `recurrence_start_any_period`. ([#7034])
+ - Plugins: `funder` option "lease-fee-base-msat" removed (deprecated in v0.11, use "lease-fee-base-sat") ([#6936])
+
+
+
+[#6983]: https://github.com/ElementsProject/lightning/pull/6983
+[#7034]: https://github.com/ElementsProject/lightning/pull/7034
+[#6864]: https://github.com/ElementsProject/lightning/pull/6864
+[#7072]: https://github.com/ElementsProject/lightning/pull/7072
+[#6899]: https://github.com/ElementsProject/lightning/pull/6899
+[#6936]: https://github.com/ElementsProject/lightning/pull/6936
+[#6895]: https://github.com/ElementsProject/lightning/pull/6895
+[#6888]: https://github.com/ElementsProject/lightning/pull/6888
+[#6898]: https://github.com/ElementsProject/lightning/pull/6898
+[#6957]: https://github.com/ElementsProject/lightning/pull/6957
+[#6912]: https://github.com/ElementsProject/lightning/pull/6912
+[#6967]: https://github.com/ElementsProject/lightning/pull/6967
+[#6937]: https://github.com/ElementsProject/lightning/pull/6937
+[#6988]: https://github.com/ElementsProject/lightning/pull/6988
+[#6785]: https://github.com/ElementsProject/lightning/pull/6785
+[#7010]: https://github.com/ElementsProject/lightning/pull/7010
+[#6992]: https://github.com/ElementsProject/lightning/pull/6992
+[#6904]: https://github.com/ElementsProject/lightning/pull/6904
+[#7018]: https://github.com/ElementsProject/lightning/pull/7018
+[#6869]: https://github.com/ElementsProject/lightning/pull/6869
+[#6984]: https://github.com/ElementsProject/lightning/pull/6984
+[#6865]: https://github.com/ElementsProject/lightning/pull/6865
+[#6941]: https://github.com/ElementsProject/lightning/pull/6941
+[24.02]: https://github.com/ElementsProject/lightning/releases/tag/v24.02
+[24.02rc2]: https://github.com/ElementsProject/lightning/releases/tag/v24.02rc2
+
+
 ## [23.11] - 2023-11-28: "Bitcoin Orangepaper"
 
 This release named by Shahana Farooqui
@@ -57,7 +159,7 @@ This release named by Shahana Farooqui
 
 Note: You should always set `allow-deprecated-apis=false` to test for changes.
 
-- Plugins: `clnrest` parameters `rest-port`, `rest-protocol`, `rest-host` and `rest-certs`: prefix `cln` to them ([#6876])
+ - Plugins: `clnrest` parameters `rest-port`, `rest-protocol`, `rest-host` and `rest-certs`: prefix `cln` to them ([#6876])
 
 
 ### Removed
@@ -149,7 +251,7 @@ Bugfix release for bad issues found since 23.08 which can't wait for 23.11, and
 
  - Protocol: Fixed a wrong number type being used in routes ([#6642])
  - JSON-RPC: `showrunes` on a specific rune would always say `stored`: false. ([#6640])
- - MacOS: `clnrest` now works ([#6605]) 
+ - MacOS: `clnrest` now works ([#6605])
  - Build: test for `python3` or `python`, rather than assuming `python3` ([#6630])
 
 
@@ -417,7 +519,7 @@ Note: You should always set `allow-deprecated-apis=false` to test for changes.
  - JSON-RPC: `close`, `fundchannel`, `fundpsbt`, `multifundchannel`, `multiwithdraw`, `txprepare`, `upgradewallet`, `withdraw` `feerate` (`feerange` for `close`) expressed as, "delayed_to_us", "htlc_resolution", "max_acceptable" or "min_acceptable".  Use explicit block counts or *slow*/*normal*/*urgent*/*minimum*. ([#6120])
  - Plugins: `estimatefees` returning feerates by name (e.g. "opening"); use `fee_floor` and `feerates`. ([#6120])
  - Protocol: Not setting `option_scid_alias` in `option_channel` `channel_type` for unannounced channels. ([#6136])
- 
+
 
 ### Removed
 
@@ -2759,6 +2861,7 @@ There predate the BOLT specifications, and are only of vague historic interest:
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
+[24.02]: https://github.com/ElementsProject/lightning/releases/tag/v24.02
 [23.11]: https://github.com/ElementsProject/lightning/releases/tag/v23.11
 [23.05]: https://github.com/ElementsProject/lightning/releases/tag/v23.05
 [23.02.1]: https://github.com/ElementsProject/lightning/releases/tag/v23.02.1

Makefile

@@ -71,7 +71,7 @@ COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -D
 endif
 
 # (method=thread to support xdist)
-PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
+PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS) $(PYTEST_MOREOPTS)
 MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
 # Collect generated python files to be excluded from lint checks
 PYTHON_GENERATED= \

common/gossmap.c

@@ -690,9 +690,13 @@ static bool load_gossip_store(struct gossmap *map)
 {
 	map->map_size = lseek(map->fd, 0, SEEK_END);
 	map->local = NULL;
+
+	/* gossipd uses pwritev(), which is not consistent with mmap on OpenBSD! */
+#ifndef __OpenBSD__
 	/* If this fails, we fall back to read */
 	map->mmap = mmap(NULL, map->map_size, PROT_READ, MAP_SHARED, map->fd, 0);
 	if (map->mmap == MAP_FAILED)
+#endif /* __OpenBSD__ */
 		map->mmap = NULL;
 
 	/* We only support major version 0 */
@@ -994,8 +998,11 @@ bool gossmap_refresh_mayfail(struct gossmap *map, bool *updated)
 	if (map->mmap)
 		munmap(map->mmap, map->map_size);
 	map->map_size = len;
+	/* gossipd uses pwritev(), which is not consistent with mmap on OpenBSD! */
+#ifndef __OpenBSD__
 	map->mmap = mmap(NULL, map->map_size, PROT_READ, MAP_SHARED, map->fd, 0);
 	if (map->mmap == MAP_FAILED)
+#endif /* __OpenBSD__ */
 		map->mmap = NULL;
 
 	return map_catchup(map, updated);

common/hsm_version.h

@@ -22,7 +22,8 @@
  * v4 with forget_channel: d87c6934ea188f92785d38d7cd0b13ed7f76aa7417f3200baf0c7b5aa832fe29
  * v5 with hsmd_revoke_commitment_tx: 5742538f87ef5d5bf55b66dc19e52c8683cfeb1b887d3e64ba530ba9a4d8e638
  * v5 with sign_any_cannouncement: 5fdb9068c43a21887dc03f7dce410d2e3eeff6277f0d49b4fc56595a798fd4a4
+ * v5 drop init v2: 5024454532fe5a78bb7558000cb344190888b9915360d3d56ddca22eaba9b872
 */
-#define HSM_MIN_VERSION 3
+#define HSM_MIN_VERSION 5
 #define HSM_MAX_VERSION 5
 #endif /* LIGHTNING_COMMON_HSM_VERSION_H */

contrib/docker/Dockerfile.builder.fedora

@@ -12,7 +12,8 @@ RUN dnf update -y && \
 		libsq3-devel \
 		python3-devel \
 		python3-mako \
-		python3-pip \
+                python3-pip \
+                python3-virtualenv \
 		python3-setuptools \
 		redhat-lsb \
 		net-tools \
@@ -30,5 +31,7 @@ RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_
     mv bitcoin-$BITCOIN_VERSION/share/man/man1/* /usr/share/man/man1 && \
     rm -rf bitcoin.tar.gz bitcoin-$BITCOIN_VERSION
 
-RUN python3 -m pip install --force-reinstall -U pip setuptools && \
-    python3 -m pip install python-bitcoinlib pytest pytest-test-groups flake8 pytest-rerunfailures ephemeral-port-reserve
+ENV PATH=/opt/venv/bin:${PATH}
+RUN python3 -m pip install pip wheel && \
+    python3 -m virtualenv /opt/venv && \
+    /opt/venv/bin/python3 -m pip install --force-reinstall -U pip poetry wheel

contrib/pyln-client/pyln/client/__init__.py

@@ -3,7 +3,7 @@
 from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapHalfchannel, GossmapNodeId, LnFeatureBits
 from .gossmapstats import GossmapStats
 
-__version__ = "23.11"
+__version__ = "24.02"
 
 __all__ = [
     "LightningRpc",

contrib/pyln-client/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyln-client"
-version = "23.11"
+version = "24.02"
 description = "Client library and plugin library for Core Lightning"
 authors = ["Christian Decker <[email protected]>"]
 license = "BSD-MIT"

contrib/pyln-proto/pyln/proto/__init__.py

@@ -4,7 +4,7 @@
 from .onion import OnionPayload, TlvPayload, LegacyOnionPayload
 from .wire import LightningConnection, LightningServerSocket
 
-__version__ = "23.11"
+__version__ = "24.02"
 
 __all__ = [
     "Invoice",

contrib/pyln-proto/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyln-proto"
-version = "23.11"
+version = "24.02"
 description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)."
 authors = ["Christian Decker <[email protected]>"]
 license = "BSD-MIT"

contrib/pyln-testing/pyln/testing/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "23.11"
+__version__ = "24.02"
 
 __all__ = [
     "__version__",

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -1,6 +1,6 @@
 from concurrent import futures
 from pyln.testing.db import SqliteDbProvider, PostgresDbProvider
-from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, LightningNode, TEST_DEBUG, TEST_NETWORK
+from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, LightningNode, TEST_DEBUG, TEST_NETWORK, LssD
 from pyln.client import Millisatoshi
 from typing import Dict
 
@@ -32,6 +32,9 @@ def test_base_dir():
 
     yield directory
 
+    if bool(int(os.getenv('TEST_KEEPDIR', '0'))):
+        return
+
     # Now check if any test directory is left because the corresponding test
     # failed. If there are no such tests we can clean up the root test
     # directory.
@@ -93,7 +96,7 @@ def directory(request, test_base_dir, test_name):
     outcome = 'passed' if rep_call is None else rep_call.outcome
     failed = not outcome or request.node.has_errors or outcome != 'passed'
 
-    if not failed:
+    if not failed and not bool(int(os.getenv('TEST_KEEPDIR', '0'))):
         try:
             shutil.rmtree(directory)
         except OSError:
@@ -165,6 +168,25 @@ def bitcoind(directory, teardown_checks):
     bitcoind.proc.wait()
 
 
+@pytest.fixture
+def lssd(directory, teardown_checks):
+    lssd = LssD(directory)
+
+    try:
+        lssd.start()
+    except Exception:
+        lssd.stop()
+        raise
+
+    yield lssd
+
+    try:
+        lssd.stop()
+    except Exception:
+        lssd.proc.kill()
+    lssd.proc.wait()
+
+
 class TeardownErrors(object):
     def __init__(self):
         self.errors = []
@@ -447,11 +469,12 @@ def jsonschemas():
 
 
 @pytest.fixture
-def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, jsonschemas):
+def node_factory(request, directory, test_name, bitcoind, lssd, executor, db_provider, teardown_checks, node_cls, jsonschemas):
     nf = NodeFactory(
         request,
         test_name,
         bitcoind,
+        lssd,
         executor,
         directory=directory,
         db_provider=db_provider,