Merge main into verkle

.github/ISSUE_TEMPLATE/release-checklist.md

@@ -7,41 +7,42 @@ assignees: ''
 
 ---
 
-- [ ] Confirm anything outstanding for release with other maintainers on #besu-release in Discord
+- [ ] Confirm at least 24 hours prior anything outstanding for release with other maintainers on #besu-release in Discord
 - [ ] Update changelog if necessary, and merge a PR for it to main
   - [ ] Notify maintainers about updating changelog for in-flight PRs 
 - [ ] Optional: for hotfixes, create a release branch and cherry-pick, e.g. `release-<version>-hotfix`
   - [ ] Optional: for hotfixes, create a PR into main from the hotfix branch to see the CI checks pass
 - [ ] On the appropriate branch/commit, create a calver tag for the release candidate, format example: `24.4.0-RC1`
-  - [ ] git tag 24.4.0-RC1
-  - [ ] git push upstream 24.4.0-RC1
+  - [ ] `git tag 24.4.0-RC1`
+  - [ ] `git push upstream 24.4.0-RC1`
 - [ ] Sign-off with team; announce the tag in #besu-release in Discord
   - [ ] Targeting this tag for the burn-in: https://github.com/hyperledger/besu/releases/tag/24.4.0-RC1
 - [ ] Consensys staff start burn-in using this tag
 - [ ] Seek sign off for burn-in
   - [ ] Pass? Go ahead and complete the release process
   - [ ] Fail? Put a message in #besu-release in Discord indicating the release will be aborted because it failed burn-in 
 - [ ] Optional: Perform a dry run with https://github.com/consensys/protocols-release-sandbox to test the workflows
-  - [ ] Sync fork
-  - [ ] git checkout <sha of 24.4.0-RC1>
-  - [ ] git tag 24.4.0
-  - [ ] git push origin 24.4.0
+  - [ ] Sync fork in github
+  - [ ] `git checkout <sha of 24.4.0-RC1>`
+  - [ ] `git tag 24.4.0`
+  - [ ] `git push <your remote name> 24.4.0`
   - [ ] Manually run https://github.com/Consensys/protocols-release-sandbox/actions/workflows/draft-release.yml using `main` branch and `24.4.0` tag
 - [ ] Back on besu, using the same git sha as 24.4.0-RC1, create a calver tag for the FULL RELEASE, example format `24.4.0`
-  - [ ] git checkout 24.4.0-RC1
-  - [ ] git tag 24.4.0
-  - [ ] git push upstream 24.4.0
+  - [ ] `git checkout 24.4.0-RC1`
+  - [ ] `git tag 24.4.0`
+  - [ ] `git push upstream 24.4.0`
 - [ ] Manually run https://github.com/hyperledger/besu/actions/workflows/draft-release.yml using `main` branch` and the FULL RELEASE tag name, i.e. `24.4.0`. Note, this workflow should always be run from `main` branch (hotfix tags will still be released even if they were created based on another branch)
     - publishes artefacts and version-specific docker tags but does not fully publish the GitHub release so subscribers are not yet notified
 - [ ] Check all draft-release workflow jobs went green
 - [ ] Check binary SHAs are correct on the release page
+- [ ] Check artifacts exist in https://hyperledger.jfrog.io/ui/repos/tree/General/besu-maven
 - [ ] Update release notes in the GitHub draft release, save draft and sign-off with team
 - [ ] Publish draft release ensuring it is marked as latest release (if appropriate)
     - this is now public and notifies subscribed users
     - makes the release "latest" in github
     - publishes the docker `latest` tag variants
-- [ ] Create homebrew release using [update-version workflow](https://github.com/hyperledger/homebrew-besu/actions/workflows/update-version.yml)
+- [ ] Create homebrew release PR using [update-version workflow](https://github.com/hyperledger/homebrew-besu/actions/workflows/update-version.yml)
   - If the PR has not been automatically created, create the PR manually using the created branch `update-<version>`
-  - Run commands `brew tap hyperledger/besu && brew install besu` on MacOSX and verify latest version has been installed
+- [ ] Verify homebrew release once the PR has merged using `brew tap hyperledger/besu && brew install besu` on MacOSX to verify latest version has been installed
 - [ ] Delete the burn-in nodes (unless required for further analysis eg performance)
 - [ ] Social announcements

.github/workflows/acceptance-tests.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release-*
+      - verkle
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/integration-tests.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release-*
+      - verkle
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/pre-review.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release-*
+      - verkle
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/reference-tests.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release-*
+      - verkle
 
 env:
   GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.caching=true"

CHANGELOG.md

@@ -1,9 +1,68 @@
 # Changelog
 
-## [Unreleased]
+## Unreleased
+
+### Breaking Changes
+- `--host-whitelist` has been deprecated since 2020 and this option is removed. Use the equivalent `--host-allowlist` instead. 
+
+### Upcoming Breaking Changes
+- `MetricSystem::createLabelledGauge` is deprecated and will be removed in a future release, replace it with `MetricSystem::createLabelledSuppliedGauge`
+- k8s (KUBERNETES) Nat method is now deprecated and will be removed in a future release. Use docker or none instead.
+- Sunsetting features - for more context on the reasoning behind the deprecation of these features, including alternative options, read [this blog post](https://www.lfdecentralizedtrust.org/blog/sunsetting-tessera-and-simplifying-hyperledger-besu)
+  - Tessera privacy
+  - Smart-contract-based (onchain) permissioning
+  - Proof of Work consensus
+  - Fast Sync
+
+### Additions and Improvements
+- Add RPC HTTP options to specify custom truststore and its password [#7978](https://github.com/hyperledger/besu/pull/7978)
+- Retrieve all transaction receipts for a block in one request [#6646](https://github.com/hyperledger/besu/pull/6646)
+- Implement EIP-7840: Add blob schedule to config files [#8042](https://github.com/hyperledger/besu/pull/8042)
+- Allow gasPrice (legacy) and 1559 gasPrice params to be specified simultaneously for `eth_call`, `eth_createAccessList`, and `eth_estimateGas` [#8059](https://github.com/hyperledger/besu/pull/8059)
+
+
+### Bug fixes
+- Fix serialization of state overrides when `movePrecompileToAddress` is present [#8204](https://github.com/hyperledger/besu/pull/8024)
+- Revise the approach for setting level_compaction_dynamic_level_bytes RocksDB configuration option [#8037](https://github.com/hyperledger/besu/pull/8037)
+- Fix possible incomplete txpool restore from dump file [#7991](https://github.com/hyperledger/besu/pull/7991)
+
+## 24.12.2 Hotfix
+
+This is an optional hotfix to address serialization of state overrides parameter when `movePrecompileToAddress` is present. 
+
+There is no need to upgrade from 24.12.0 (or 24.12.1) to this release if you are not yet using this functionality.
+
+### Bug fixes
+- Fix serialization of state overrides when `movePrecompileToAddress` is present [#8204](https://github.com/hyperledger/besu/pull/8024)
+
+## 24.12.1 Hotfix
+
+This is a hotfix to address publishing besu maven artifacts.  There are no issues with 24.12.0 other than incomplete artifact publishing, and there is no functional difference between 24.12.0 and 24.12.1 release binaries.
+
+### Bug fixes
+- Fix BOM pom publication to Artifactory [#8201](https://github.com/hyperledger/besu/pull/8021)
+
+## 24.12.0
 
 ### Breaking Changes
 - Removed Retesteth rpc service and commands [#7833](https://github.com/hyperledger/besu/pull/7783)
+- TLS for P2P (early access feature) has been removed [#7942](https://github.com/hyperledger/besu/pull/7942)
+- In the plugin API, `BesuContext` has been renamed to `ServiceManager` to better reflect its function, plugins must be updated to work with this version
+- With the upgrade of the Prometheus Java Metrics library, there are the following changes:
+  - Gauge names are not allowed to end with `total`, therefore the metric `besu_blockchain_difficulty_total` is losing the `_total` suffix
+  - The `_created` timestamps are not returned by default, you can set the env var `BESU_OPTS="-Dio.prometheus.exporter.includeCreatedTimestamps=true"` to enable them
+  - Some JVM metrics have changed name to adhere to the OTEL standard (see the table below), [Besu Full Grafana dashboard](https://grafana.com/grafana/dashboards/16455-besu-full/) is updated to support both names
+
+    | Old Name                        | New Name                        |
+    |---------------------------------|---------------------------------|
+    | jvm_memory_bytes_committed      | jvm_memory_committed_bytes      |
+    | jvm_memory_bytes_init           | jvm_memory_init_bytes           |
+    | jvm_memory_bytes_max            | jvm_memory_max_bytes            |
+    | jvm_memory_bytes_used           | jvm_memory_used_bytes           |
+    | jvm_memory_pool_bytes_committed | jvm_memory_pool_committed_bytes |
+    | jvm_memory_pool_bytes_init      | jvm_memory_pool_init_bytes      |
+    | jvm_memory_pool_bytes_max       | jvm_memory_pool_max_bytes       |
+    | jvm_memory_pool_bytes_used      | jvm_memory_pool_used_bytes      |
 
 ### Upcoming Breaking Changes
 - Plugin API will be deprecating the BesuContext interface to be replaced with the ServiceManager interface.
@@ -12,7 +71,7 @@
 - `--host-whitelist` has been deprecated in favor of `--host-allowlist` since 2020 and will be removed in a future release
 - Sunsetting features - for more context on the reasoning behind the deprecation of these features, including alternative options, read [this blog post](https://www.lfdecentralizedtrust.org/blog/sunsetting-tessera-and-simplifying-hyperledger-besu)
   - Tessera privacy
-  - Smart-contract-based permissioning
+  - Smart-contract-based (onchain) permissioning
   - Proof of Work consensus
   - Fast Sync
 
@@ -26,10 +85,18 @@
 - Add a method to check if a metric category is enabled to the plugin API [#7832](https://github.com/hyperledger/besu/pull/7832)
 - Add a new metric collector for counters which get their value from suppliers [#7894](https://github.com/hyperledger/besu/pull/7894)
 - Add account and state overrides to `eth_call` [#7801](https://github.com/hyperledger/besu/pull/7801) and `eth_estimateGas` [#7890](https://github.com/hyperledger/besu/pull/7890)
+- Add RPC WS options to specify password file for keystore and truststore [#7970](https://github.com/hyperledger/besu/pull/7970)
+- Prometheus Java Metrics library upgraded to version 1.3.3 [#7880](https://github.com/hyperledger/besu/pull/7880)
+- Add histogram to Prometheus metrics system [#7944](https://github.com/hyperledger/besu/pull/7944)
+- Improve newPayload and FCU logs [#7961](https://github.com/hyperledger/besu/pull/7961)
+- Proper support for `pending` block tag when calling `eth_estimateGas` and `eth_createAccessList` [#7951](https://github.com/hyperledger/besu/pull/7951)
 
 ### Bug fixes
 - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825)
 - Fix CVE-2024-47535 [7878](https://github.com/hyperledger/besu/pull/7878)
+- Fix QBFT prepared block based proposal validation [#7875](https://github.com/hyperledger/besu/pull/7875)
+- Correct default parameters for frontier transactions in `eth_call` and `eth_estimateGas` [#7965](https://github.com/hyperledger/besu/pull/7965)
+- Correctly parse nonce as hex in `eth_call` account overrides [#7999](https://github.com/hyperledger/besu/pull/7999)
 
 ## 24.10.0
 

CONTRIBUTING.md

@@ -11,6 +11,9 @@ Having the following accounts is necessary for contributing code/issues to Besu.
 * To ask questions or chat with us, join our [Discord](https://discord.com/invite/hyperledger)
 * To edit pages in our wiki, you'll need a [Linux Foundation (LF) account].
 
+### Guidelines for Non-Code and other Trivial Contributions
+Please keep in mind that we do not accept non-code contributions like fixing comments, typos or some other trivial fixes. Although we appreciate the extra help, managing lots of these small contributions is unfeasible, and puts extra pressure in our continuous delivery systems (running all tests, etc). Feel free to open an issue pointing any of those errors, and we will batch them into a single change.
+
 ### Useful contributing links
 
 * [I just have a quick question](https://wiki.hyperledger.org/display/BESU/I+just+have+a+quick+question)

acceptance-tests/dsl/build.gradle

@@ -18,8 +18,6 @@ dependencies {
   implementation project(':ethereum:permissioning')
   implementation project(':ethereum:rlp')
   implementation project(':metrics:core')
-  implementation project(':pki')
-  implementation project(path: ':pki', configuration: 'testArtifacts')
   implementation project(':plugin-api')
   implementation project(':plugins:rocksdb')
   implementation project(':services:kvstore')

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/clique/CliqueConditions.java

@@ -19,7 +19,7 @@
 import static org.hyperledger.besu.tests.acceptance.dsl.transaction.clique.CliqueTransactions.LATEST;
 
 import org.hyperledger.besu.config.CliqueConfigOptions;
-import org.hyperledger.besu.config.GenesisConfigFile;
+import org.hyperledger.besu.config.GenesisConfig;
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain.ExpectBlockNotCreated;
@@ -89,9 +89,9 @@ public Condition awaitSignerSetChange(final Node node) {
 
   private int cliqueBlockPeriod(final BesuNode node) {
     final String config = node.getGenesisConfigProvider().create(emptyList()).get();
-    final GenesisConfigFile genesisConfigFile = GenesisConfigFile.fromConfig(config);
+    final GenesisConfig genesisConfig = GenesisConfig.fromConfig(config);
     final CliqueConfigOptions cliqueConfigOptions =
-        genesisConfigFile.getConfigOptions().getCliqueConfigOptions();
+        genesisConfig.getConfigOptions().getCliqueConfigOptions();
     return cliqueConfigOptions.getBlockPeriodSeconds();
   }
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -32,7 +32,6 @@
 import org.hyperledger.besu.ethereum.core.Util;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration;
-import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
@@ -96,7 +95,6 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
   private final Properties portsProperties = new Properties();
   private final Boolean p2pEnabled;
   private final int p2pPort;
-  private final Optional<TLSConfiguration> tlsConfiguration;
   private final NetworkingConfiguration networkingConfiguration;
   private final boolean revertReasonEnabled;
 
@@ -156,7 +154,6 @@ public BesuNode(
       final GenesisConfigurationProvider genesisConfigProvider,
       final boolean p2pEnabled,
       final int p2pPort,
-      final Optional<TLSConfiguration> tlsConfiguration,
       final NetworkingConfiguration networkingConfiguration,
       final boolean discoveryEnabled,
       final boolean bootnodeEligible,
@@ -207,7 +204,6 @@ public BesuNode(
     this.network = network;
     this.p2pEnabled = p2pEnabled;
     this.p2pPort = p2pPort;
-    this.tlsConfiguration = tlsConfiguration;
     this.networkingConfiguration = networkingConfiguration;
     this.discoveryEnabled = discoveryEnabled;
     this.bootnodeEligible = bootnodeEligible;
@@ -659,10 +655,6 @@ public boolean isP2pEnabled() {
     return p2pEnabled;
   }
 
-  public Optional<TLSConfiguration> getTLSConfiguration() {
-    return tlsConfiguration;
-  }
-
   public NetworkingConfiguration getNetworkingConfiguration() {
     return networkingConfiguration;
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -22,7 +22,6 @@
 import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
 import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
-import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
@@ -365,26 +364,6 @@ private List<String> commandlineArgs(final BesuNode node, final Path dataDir) {
       final List<String> networkConfigParams =
           NetworkingOptions.fromConfig(node.getNetworkingConfiguration()).getCLIOptions();
       params.addAll(networkConfigParams);
-      if (node.getTLSConfiguration().isPresent()) {
-        final TLSConfiguration config = node.getTLSConfiguration().get();
-        params.add("--Xp2p-tls-enabled");
-        params.add("--Xp2p-tls-keystore-type");
-        params.add(config.getKeyStoreType());
-        params.add("--Xp2p-tls-keystore-file");
-        params.add(config.getKeyStorePath().toAbsolutePath().toString());
-        params.add("--Xp2p-tls-keystore-password-file");
-        params.add(config.getKeyStorePasswordPath().toAbsolutePath().toString());
-        params.add("--Xp2p-tls-crl-file");
-        params.add(config.getCrlPath().toAbsolutePath().toString());
-        if (null != config.getTrustStoreType()) {
-          params.add("--Xp2p-tls-truststore-type");
-          params.add(config.getTrustStoreType());
-          params.add("--Xp2p-tls-truststore-file");
-          params.add(config.getTrustStorePath().toAbsolutePath().toString());
-          params.add("--Xp2p-tls-truststore-password-file");
-          params.add(config.getTrustStorePasswordPath().toAbsolutePath().toString());
-        }
-      }
     }
 
     if (node.isRevertReasonEnabled()) {