Generalize number of supported soroban hosts, add host for p22.

Author: graydon

Cargo.lock

@@ -46,7 +46,3 @@ endif # USE_SPDLOG
 if ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
 AM_CPPFLAGS += -DENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
 endif # ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
-
-if ENABLE_PROTOCOL_UPGRADE_VIA_SOROBAN_ENV_HOST_PREV
-AM_CPPFLAGS += -DENABLE_PROTOCOL_UPGRADE_VIA_SOROBAN_ENV_HOST_PREV
-endif # ENABLE_PROTOCOL_UPGRADE_VIA_SOROBAN_ENV_HOST_PREV

common.mk

@@ -397,12 +397,6 @@ AC_ARG_ENABLE(next-protocol-version-unsafe-for-production,
 AM_CONDITIONAL(ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION,
     [test x$enable_next_protocol_version_unsafe_for_production = xyes])
 
-AC_ARG_ENABLE(protocol-upgrade-via-soroban-env-host-prev,
-    AS_HELP_STRING([--enable-protocol-upgrade-via-soroban-env-host-prev],
-        [Pass "--feature=soroban-env-host-prev" to "cargo build" to enable protocol upgrades]))
-AM_CONDITIONAL(ENABLE_PROTOCOL_UPGRADE_VIA_SOROBAN_ENV_HOST_PREV,
-    [test x$enable_protocol_upgrade_via_soroban_env_host_prev = xyes])
-
 AC_ARG_ENABLE(libunwind,
     AS_HELP_STRING([--disable-libunwind],
         [Disable backtraces using libunwind]))

configure.ac

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.74.1"
+channel = "1.79.0"

rust-toolchain.toml

@@ -14,11 +14,7 @@ else # !BUILD_TESTS
 stellar_core_SOURCES = main/StellarCoreVersion.cpp main/XDRFilesSha256.cpp $(SRC_CXX_FILES)
 endif # !BUILD_TESTS
 
-if ENABLE_PROTOCOL_UPGRADE_VIA_SOROBAN_ENV_HOST_PREV
-CARGO_FEATURES = --features soroban-env-host-prev
-else
 CARGO_FEATURES =
-endif
 if USE_TRACY
 CARGO_FEATURES += --features tracy
 endif
@@ -80,7 +76,7 @@ RUST_BIN_DIR=$(RUST_BUILD_DIR)/bin
 RUST_TARGET_DIR=$(top_builddir)/target
 RUST_CXXBRIDGE=$(RUST_BIN_DIR)/cxxbridge
 RUST_PROFILE=release
-RUST_HOST_DEPFILES=rust/Cargo.toml $(top_srcdir)/Cargo.toml $(top_srcdir)/Cargo.lock rust/src/host-dep-tree-curr.txt rust/src/host-dep-tree-prev.txt
+RUST_HOST_DEPFILES=rust/Cargo.toml $(top_srcdir)/Cargo.toml $(top_srcdir)/Cargo.lock $(wildcard rust/src/dep-trees/*.txt)
 LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_PROFILE)/librust_stellar_core.a
 stellar_core_LDADD += $(LIBRUST_STELLAR_CORE) -ldl
 
@@ -89,11 +85,11 @@ $(RUST_CXXBRIDGE):
 	CARGO_HTTP_MULTIPLEXING=false $(CARGO) install --locked --root $(RUST_BUILD_DIR) cxxbridge-cmd --version 1.0.68
 
 rust/RustBridge.h: rust/src/lib.rs $(SRC_RUST_FILES) Makefile $(RUST_CXXBRIDGE)
-	$(RUST_CXXBRIDGE) $< --header --output $@.tmp
+	$(RUST_CXXBRIDGE) $< --cfg test=false --header --output $@.tmp
 	if cmp -s $@.tmp $@; then rm -v $@.tmp; else mv -v $@.tmp $@; fi
 
 rust/RustBridge.cpp: rust/src/lib.rs $(SRC_RUST_FILES) Makefile $(RUST_CXXBRIDGE)
-	$(RUST_CXXBRIDGE) $< --output $@.tmp
+	$(RUST_CXXBRIDGE) $< --cfg test=false --output $@.tmp
 	if cmp -s $@.tmp $@; then rm -v $@.tmp; else mv -v $@.tmp $@; fi
 
 $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile

src/Makefile.am

@@ -1650,81 +1650,43 @@ runSignTransaction(CommandLineArgs const& args)
 int
 runVersion(CommandLineArgs const&)
 {
+    rust::Vec<SorobanVersionInfo> rustVersions =
+        rust_bridge::get_soroban_version_info(
+            Config::CURRENT_LEDGER_PROTOCOL_VERSION);
+
     std::cout << STELLAR_CORE_VERSION << std::endl;
     std::cout << "ledger protocol version: "
               << Config::CURRENT_LEDGER_PROTOCOL_VERSION << std::endl;
     std::cout << "rust version: " << rust_bridge::get_rustc_version().c_str()
               << std::endl;
 
-    std::cout << "soroban-env-host: " << std::endl;
-
-    std::cout << "    curr:" << std::endl;
-    std::cout << "        package version: "
-              << rust_bridge::get_soroban_env_pkg_versions().curr.c_str()
-              << std::endl;
-
-    std::cout << "        git version: "
-              << rust_bridge::get_soroban_env_git_versions().curr.c_str()
-              << std::endl;
-
-    std::cout << "        ledger protocol version: "
-              << rust_bridge::get_soroban_env_ledger_protocol_versions().curr
-              << std::endl;
-
-    std::cout << "        pre-release version: "
-              << rust_bridge::get_soroban_env_pre_release_versions().curr
-              << std::endl;
-
-    std::cout << "        rs-stellar-xdr:" << std::endl;
-
-    std::cout
-        << "            package version: "
-        << rust_bridge::get_soroban_xdr_bindings_pkg_versions().curr.c_str()
-        << std::endl;
-    std::cout
-        << "            git version: "
-        << rust_bridge::get_soroban_xdr_bindings_git_versions().curr.c_str()
-        << std::endl;
-    std::cout << "            base XDR git version: "
-              << rust_bridge::get_soroban_xdr_bindings_base_xdr_git_versions()
-                     .curr.c_str()
-              << std::endl;
+    std::cout << "soroban-env-host versions: " << std::endl;
 
-    if (rust_bridge::compiled_with_soroban_prev())
+    size_t i = 0;
+    for (auto& host : rustVersions)
     {
-        std::cout << "    prev:" << std::endl;
-        std::cout << "        package version: "
-                  << rust_bridge::get_soroban_env_pkg_versions().prev.c_str()
+        std::cout << "    host[" << i << "]:" << std::endl;
+        std::cout << "        package version: " << host.env_pkg_ver.c_str()
                   << std::endl;
 
-        std::cout << "        git version: "
-                  << rust_bridge::get_soroban_env_git_versions().prev.c_str()
+        std::cout << "        git version: " << host.env_git_rev.c_str()
                   << std::endl;
 
-        std::cout
-            << "        ledger protocol version: "
-            << rust_bridge::get_soroban_env_ledger_protocol_versions().prev
-            << std::endl;
+        std::cout << "        ledger protocol version: " << host.env_max_proto
+                  << std::endl;
 
-        std::cout << "        pre-release version: "
-                  << rust_bridge::get_soroban_env_pre_release_versions().prev
+        std::cout << "        pre-release version: " << host.env_pre_release_ver
                   << std::endl;
 
         std::cout << "        rs-stellar-xdr:" << std::endl;
 
-        std::cout
-            << "            package version: "
-            << rust_bridge::get_soroban_xdr_bindings_pkg_versions().prev.c_str()
-            << std::endl;
-        std::cout
-            << "            git version: "
-            << rust_bridge::get_soroban_xdr_bindings_git_versions().prev.c_str()
-            << std::endl;
-        std::cout
-            << "            base XDR git version: "
-            << rust_bridge::get_soroban_xdr_bindings_base_xdr_git_versions()
-                   .prev.c_str()
-            << std::endl;
+        std::cout << "            package version: " << host.xdr_pkg_ver.c_str()
+                  << std::endl;
+        std::cout << "            git version: " << host.xdr_git_rev.c_str()
+                  << std::endl;
+        std::cout << "            base XDR git version: "
+                  << host.xdr_base_git_rev.c_str() << std::endl;
+        ++i;
     }
     return 0;
 }

src/main/CommandLine.cpp

@@ -170,7 +170,12 @@ void
 checkXDRFileIdentity()
 {
     using namespace stellar::rust_bridge;
-    rust::Vec<XDRFileHash> rustHashes = get_xdr_hashes().curr;
+
+    rust::Vec<SorobanVersionInfo> rustVersions = get_soroban_version_info(
+        stellar::Config::CURRENT_LEDGER_PROTOCOL_VERSION);
+    rust::Vec<XDRFileHash> const& rustHashes =
+        rustVersions.back().xdr_file_hashes;
+
     for (auto const& cpp : stellar::XDR_FILES_SHA256)
     {
         if (cpp.first.empty())

src/main/main.cpp

@@ -2,7 +2,7 @@
 name = "stellar-core"
 version = "0.1.0"
 edition = "2021"
-rust-version = "1.74"
+rust-version = "1.79.0"
 publish = false
 
 [lib]
@@ -23,64 +23,75 @@ petgraph = "=0.6.4"
 toml = "=0.7.8"
 itertools = "=0.11.0"
 
-# This copy of the soroban host is always enabled, and should always point to a
-# version that supports stellar-core's Config::CURRENT_LEDGER_PROTOCOL_VERSION.
-# When upgrading from protocol N to N+1, this copy will therefore have its
-# version advance first (to enable support for N+1) even before the network has
-# voted to upgrade to it. In order to avoid divergence during such an upgrade,
-# we will simultaneously enable the `prev` copy of soroban (below), continuing
-# to point to the previous already-released soroban that only supported protocol
-# N.
+# Stellar-core supports multiple versions of the Soroban host linked in
+# simultaneously. Each host is labeled with the _maximum_ protocol that it
+# supports. The minimum protocol supported by each host is implicitly one more
+# than the less-higher-numbered host's maximum protocol, though in some cases a
+# host may also be configured with an explicit minimum supported protocol, in
+# which case we have to keep extra hosts around to fill in any gaps in support.
+#
+# As an example: if we have hosts labeled -p21 and -p23, then the -p21 host will
+# be used to apply ledgers with protocol 20 and 21, and the -p23 host will be
+# used to apply ledgers with protocol 22 and 23.
+#
+# We add new hosts when performing a protocol upgrade. This allows us to hold
+# back any changes to the host's dependencies to the next protocol transition
+# and then simultaneously transition the entire network from one set of
+# dependencies to another at the (consensus) protocol upgrade boundary.
+#
+# We remove hosts -- when possible -- _after_ a protocol upgrade, once no more
+# ledgers with the previous protocol N-1 are being closed, and only when we have
+# convinced ourselves that the host everyone is now running, that supports the
+# new protocol N, can also faithfully replay all ledgers recorded under the
+# previous protocol N-1. If there is any divergence during replay, then either
+# the protocol N host has to be patched with some backward-compatibility code,
+# or else we have to keep the protocol N-1 host around indefinitely. If we can
+# get a clean replay with the new protocol N host, then we can remove the
+# protocol N-1 host from the codebase.
+#
+# Each additional copy of the soroban host adds a small amount of complexity to
+# the build and about 1.5MB of object code to the resulting binary. So we want
+# to keep the number of hosts down when and if it's easy to, but having the
+# number grow gradually is also not the end of the world.
 
-[dependencies.soroban-env-host-curr]
-version = "=21.2.0"
+[dependencies.soroban-env-host-p22]
+version = "=22.0.0"
 git = "https://github.com/stellar/rs-soroban-env"
 package = "soroban-env-host"
-rev = "8809852dcf8489f99407a5ceac12625ee3d14693"
-
-# This copy of the soroban host is _optional_ and only enabled during protocol
-# transitions. When transitioning from protocol N to N+1, the `curr` copy
-# (above) will bump its version to one that supports protocol N+1, and we will
-# simultaneously enable this `prev` copy by building with
-# `--feature=soroban-env-host-prev` and pointing its version and git rev to the
-# last soroban released to the network that speaks protocol N. When we deploy
-# this to a network running protocol N it will run on `prev` until the upgrade,
-# then switch from `prev` to `curr` at the upgrade, and then in the next release
-# we can turn `prev` off again.
-#
-# A note on `rev` hashes: When `prev` is disabled, the rust-analyzer IDE will
-# still want it to point to some version of soroban just for auto-completion
-# (because it builds with `--all-features`). To make this work, the `rev` spec
-# below has to be different from the `rev` spec in the `curr` copy of soroban
-# above. The easiest way to make this work is to just point to one git rev 
-# before the rev specified for `curr`.
+rev = "81a4239f3e1ab5e4030140310faa64aba28dfdcb"
 
-[dependencies.soroban-env-host-prev]
-optional = true
-version = "=20.3.0"
+[dependencies.soroban-env-host-p21]
+version = "=21.2.0"
 git = "https://github.com/stellar/rs-soroban-env"
 package = "soroban-env-host"
-rev = "93120b6b32cd910fcc224cbf6aec1333f771a8bc"
+rev = "8809852dcf8489f99407a5ceac12625ee3d14693"
+
+# The test wasms and synth-wasm crate should usually be taken from the highest
+# supported host, since test material usually just grows over time.
 
 [dependencies.soroban-test-wasms]
-version = "=21.2.0"
+version = "=22.0.0"
 git = "https://github.com/stellar/rs-soroban-env"
-rev = "8809852dcf8489f99407a5ceac12625ee3d14693"
+rev = "81a4239f3e1ab5e4030140310faa64aba28dfdcb"
 
 [dependencies.soroban-synth-wasm]
-version = "=21.2.0"
+version = "=22.0.0"
 git = "https://github.com/stellar/rs-soroban-env"
-rev = "8809852dcf8489f99407a5ceac12625ee3d14693"
+rev = "81a4239f3e1ab5e4030140310faa64aba28dfdcb"
 
 [dependencies.cargo-lock]
 version = "=9.0.0"
 git = "https://github.com/rustsec/rustsec"
 rev = "a5c69fc6e4b6068b43d7143f3a2f68c3f3de37d8"
 features = ["dependency-tree"]
 
-# This feature definition is implied by the optional=true line in the dep, but
-# writing it out in full here cross-checks that nobody _removed_ that line.
 [features]
-soroban-env-host-prev = ["dep:soroban-env-host-prev"]
-tracy = ["dep:tracy-client", "soroban-env-host-curr/tracy"]
-core-vnext = ["soroban-env-host-curr/next"]
+
+# The tracy feature should enable the tracy features of all supported hosts.
+tracy = ["dep:tracy-client", "soroban-env-host-p21/tracy", "soroban-env-host-p22/tracy"]
+
+# The "vnext" feature enables not-yet-released features that (a) break protocol
+# but (b) are still under development, in between protocol releases. This
+# feature is not enabled by default, and is only intended for temporary use when
+# staging work across intra-protocol releases.
+core-vnext = ["soroban-env-host-p22/next"]

src/rust/Cargo.toml

@@ -7,7 +7,7 @@ use crate::{
     rust_bridge::{
         CxxBuf, CxxFeeConfiguration, CxxLedgerEntryRentChange, CxxLedgerInfo,
         CxxRentFeeConfiguration, CxxTransactionResources, CxxWriteFeeConfiguration, FeePair,
-        InvokeHostFunctionOutput, RustBuf, XDRFileHash,
+        InvokeHostFunctionOutput, RustBuf, SorobanVersionInfo, XDRFileHash,
     },
 };
 use log::{debug, trace};
@@ -26,13 +26,14 @@ use super::soroban_env_host::{
         compute_write_fee_per_1kb as host_compute_write_fee_per_1kb, FeeConfiguration,
         LedgerEntryRentChange, RentFeeConfiguration, TransactionResources, WriteFeeConfiguration,
     },
+    meta,
     xdr::{
         self, ContractCostParams, ContractEvent, ContractEventBody, ContractEventType,
         ContractEventV0, DiagnosticEvent, ExtensionPoint, LedgerEntry, LedgerEntryData,
         LedgerEntryExt, Limits, ReadXdr, ScError, ScErrorCode, ScErrorType, ScSymbol, ScVal,
         TtlEntry, WriteXdr, XDR_FILES_SHA256,
     },
-    HostError, LedgerInfo,
+    HostError, LedgerInfo, VERSION,
 };
 use std::error::Error;
 
@@ -184,6 +185,28 @@ pub fn get_xdr_hashes() -> Vec<XDRFileHash> {
         .collect()
 }
 
+pub const fn get_max_proto() -> u32 {
+    meta::get_ledger_protocol_version(VERSION.interface)
+}
+
+pub fn get_soroban_version_info(core_max_proto: u32) -> SorobanVersionInfo {
+    use meta::{get_ledger_protocol_version, get_pre_release_version};
+    SorobanVersionInfo {
+        env_max_proto: get_ledger_protocol_version(VERSION.interface),
+        env_pkg_ver: VERSION.pkg.to_string(),
+        env_git_rev: VERSION.rev.to_string(),
+        env_pre_release_ver: get_pre_release_version(VERSION.interface),
+        xdr_pkg_ver: VERSION.xdr.pkg.to_string(),
+        xdr_git_rev: VERSION.xdr.rev.to_string(),
+        xdr_base_git_rev: match VERSION.xdr.xdr {
+            "next" => VERSION.xdr.xdr_next.to_string(),
+            "curr" => VERSION.xdr.xdr_curr.to_string(),
+            _ => "unknown configuration".to_string(),
+        },
+        xdr_file_hashes: get_xdr_hashes(),
+    }
+}
+
 fn log_diagnostic_events(events: &Vec<DiagnosticEvent>) {
     for e in events {
         debug!("Diagnostic event: {:?}", e);

src/rust/src/contract.rs

@@ -0,0 +1,8 @@
+This directory contains explicit dependency trees for the different soroban
+hosts linked into stellar-core. To add a new one, install cargo-lock and run
+something like:
+
+  $ cargo lock tree --exact [email protected] > soroban_p22.txt
+  $ git add soroban_p22.txt
+
+With whatever version you want to support in place of "22".