diff --git a/.gitignore b/.gitignore index 504133a..446acff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ +.vscode +docker-images electrs.s9pk image.tar scripts/embassy.js -.vscode -docker-images diff --git a/Makefile b/Makefile index 24f8ce8..d778655 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ install: $(PKG_ID).s9pk start-cli package install $(PKG_ID).s9pk clean: + docker run --rm -it -v "$(shell pwd)"/configurator:/home/rust/src messense/rust-musl-cross:x86_64-musl cargo clean -q rm -rf docker-images rm -rf image.tar rm -f $(PKG_ID).s9pk @@ -28,7 +29,7 @@ arm: docker-images/aarch64.tar scripts/embassy.js x86: docker-images/x86_64.tar scripts/embassy.js start-sdk pack -$(PKG_ID).s9pk: manifest.yaml instructions.md scripts/embassy.js electrs/LICENSE docker-images/aarch64.tar docker-images/x86_64.tar +$(PKG_ID).s9pk: manifest.yaml instructions.md scripts/embassy.js electrs/LICENSE docker-images/x86_64.tar docker-images/aarch64.tar start-sdk pack docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh check-synced.sh check-electrum.sh configurator/target/aarch64-unknown-linux-musl/release/configurator $(ELECTRS_SRC) diff --git a/check-synced.sh b/check-synced.sh index 490b7c6..673d183 100644 --- a/check-synced.sh +++ b/check-synced.sh @@ -1,17 +1,31 @@ #!/bin/bash +# vim: ts=2 sw=2 sts=2 ai et DURATION=$(</dev/stdin) if (($DURATION <= 9000 )); then exit 60 else set -e - - b_host="bitcoind.embassy" - b_username=$(yq '.user' /data/start9/config.yaml) - b_password=$(yq '.password' /data/start9/config.yaml) - + +b_type=$(yq '.bitcoind.type' /data/start9/config.yaml) + +if [ "$b_type" == "bitcoind-testnet" ]; then + b_host="bitcoind-testnet.embassy" + b_rpc_port=48332 + e_monitoring_port=44224 + e_db_path=/data/db/testnet4 +else + b_host="bitcoind.embassy" + b_rpc_port=8332 + e_monitoring_port=4224 + e_db_path=/data/db/bitcoin +fi + + b_username=$(yq '.bitcoind.username' /data/start9/config.yaml) + b_password=$(yq '.bitcoind.password' /data/start9/config.yaml) + #Get blockchain info from the bitcoin rpc - b_gbc_result=$(curl -sS --user $b_username:$b_password --data-binary '{"jsonrpc": "1.0", "id": "sync-hck", "method": "getblockchaininfo", "params": []}' -H 'content-type: text/plain;' http://$b_host:8332/ 2>&1) + b_gbc_result=$(curl -sS --user $b_username:$b_password --data-binary '{"jsonrpc": "1.0", "id": "sync-hck", "method": "getblockchaininfo", "params": []}' -H 'content-type: text/plain;' http://$b_host:$b_rpc_port/ 2>&1) error_code=$? b_gbc_error=$(echo $b_gbc_result | yq '.error' -) if [[ $error_code -ne 0 ]]; then @@ -37,20 +51,20 @@ else exit 61 else #Gather keys/values from prometheus rpc: - curl_res=$(curl -sS localhost:4224) + curl_res=$(curl -sS localhost:$e_monitoring_port 2>/dev/null) error_code=$? - + if [[ $error_code -ne 0 ]]; then echo "Error contacting the electrs Prometheus RPC" >&2 exit 61 fi - + #Determine whether we are actively doing a database compaction: #compaction_res=$(echo -e "$features_res" | grep num-running-compactions | sed "s/\s$//g" | grep " [^0]$"|awk '{print $NF}'|head -1) #^The prometheus RPC's num-running-compactions key doesn't seem to correspond to actual # compaction events, so we'll determine compaction by another, dumber but accurate method: chk_numlines=100000 #Look through the last 100,000 lines of the db LOG - log_file="/data/db/bitcoin/LOG" + log_file="$e_db_path/LOG" tail_log="ionice -c3 tail -$chk_numlines $log_file" compaction_job=$($tail_log|nice -n19 grep EVENT_LOG|nice -n19 grep "ManualCompaction"|nice -n19 tail -1|nice -n19 cut -d" " -f7) if [ -n "$compaction_job" ] ; then @@ -82,4 +96,4 @@ else exit 61 fi fi -fi \ No newline at end of file +fi diff --git a/configurator/src/electrs.toml.template b/configurator/src/electrs.toml.template index 58754ed..2664cdd 100644 --- a/configurator/src/electrs.toml.template +++ b/configurator/src/electrs.toml.template @@ -1,7 +1,7 @@ auth = "{bitcoin_rpc_user:}:{bitcoin_rpc_pass}" daemon_rpc_addr = "{bitcoin_rpc_host}:{bitcoin_rpc_port}" daemon_p2p_addr = "{bitcoin_p2p_host}:{bitcoin_p2p_port}" -network = "bitcoin" +network = "{network}" electrum_rpc_addr = "0.0.0.0:50001" log_filters = "{log_filters}" {index_batch_size} diff --git a/configurator/src/main.rs b/configurator/src/main.rs index 3661dd6..799997c 100644 --- a/configurator/src/main.rs +++ b/configurator/src/main.rs @@ -1,28 +1,54 @@ use std::fs::File; use std::io::Write; -use http::Uri; use serde::{ - de::{Deserializer, Error as DeserializeError, Unexpected}, Deserialize, }; #[derive(Deserialize)] #[serde(rename_all = "kebab-case")] struct Config { - user: String, - password: String, + bitcoind: BitcoinCoreConfig, log_filters: String, index_batch_size: Option<u16>, index_lookup_limit: Option<u16>, } +#[derive(Deserialize)] +#[serde(tag = "type")] +enum BitcoinCoreConfig { + #[serde(rename = "bitcoind")] + Bitcoind { + username: String, + password: String, + }, + #[serde(rename = "bitcoind-testnet")] + BitcoindTestnet { + username: String, + password: String, + }, +} + fn main() -> Result<(), anyhow::Error> { let config: Config = serde_yaml::from_reader(File::open("/data/start9/config.yaml")?)?; { let mut outfile = File::create("/data/electrs.toml")?; + let (bitcoin_rpc_user, bitcoin_rpc_pass, bitcoin_rpc_host, bitcoin_rpc_port, bitcoin_p2p_host, bitcoin_p2p_port, network) = + match config.bitcoind { + BitcoinCoreConfig::Bitcoind { username, password } => { + let hostname = format!("{}", "bitcoind.embassy"); + let network = format!("{}", "bitcoin"); + (username, password, hostname.clone(), 8332, hostname.clone(), 8333, network.clone()) + } + BitcoinCoreConfig::BitcoindTestnet { username, password } => { + let hostname = format!("{}", "bitcoind-testnet.embassy"); + let network = format!("{}", "testnet4"); + (username, password, hostname.clone(), 48332, hostname.clone(), 8333, network.clone()) + } + }; + let mut index_batch_size: String = "".to_string(); if config.index_batch_size.is_some() { index_batch_size = format!( @@ -42,12 +68,13 @@ fn main() -> Result<(), anyhow::Error> { write!( outfile, include_str!("electrs.toml.template"), - bitcoin_rpc_user = config.user, - bitcoin_rpc_pass = config.password, - bitcoin_rpc_host = "bitcoind.embassy", - bitcoin_rpc_port = 8332, - bitcoin_p2p_host = "bitcoind.embassy", - bitcoin_p2p_port = 8333, + bitcoin_rpc_user = bitcoin_rpc_user, + bitcoin_rpc_pass = bitcoin_rpc_pass, + bitcoin_rpc_host = bitcoin_rpc_host, + bitcoin_rpc_port = bitcoin_rpc_port, + bitcoin_p2p_host = bitcoin_p2p_host, + bitcoin_p2p_port = bitcoin_p2p_port, + network = network, log_filters = config.log_filters, index_batch_size = index_batch_size, index_lookup_limit = index_lookup_limit, diff --git a/manifest.yaml b/manifest.yaml index d279125..330ac4f 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -1,7 +1,8 @@ id: electrs title: "electrs" -version: 0.10.9 +version: 0.10.9.1 release-notes: | + * Update electrs-wrapper tag to v0.10.9.1, added support for testnet4 * Update upstream electrs to [v0.10.9](https://github.com/romanz/electrs/blob/master/RELEASE-NOTES.md#0109-feb-01-2025) * Update Rust base Docker image * Make some parts of the check-synced healthcheck script (io)nice @@ -79,13 +80,27 @@ dependencies: bitcoind: version: ">=0.21.1.2 <30.0.0" requirement: - type: "required" + type: "opt-out" + how: Set "Bitcoin Core" to "Bitcoin Core" description: Needed for peer interface and rpc interface. config: check: type: script auto-configure: type: script + requires-runtime-config: true + bitcoind-testnet: + version: '>=0.21.1.2 <29.0.0' + requirement: + type: 'opt-in' + how: Set "Bitcoin Core" type to "Bitcoin Core (testnet)" + description: Testnet Bitcoin Core node for testing purposes + config: + check: + type: script + auto-configure: + type: script + requires-runtime-config: true backup: create: type: script diff --git a/scripts/services/dependencies.ts b/scripts/services/dependencies.ts index 4a13361..7208e75 100644 --- a/scripts/services/dependencies.ts +++ b/scripts/services/dependencies.ts @@ -2,6 +2,11 @@ import { types as T, matches } from "../deps.ts"; const { shape, number, boolean, string } = matches; +type Check = { + currentError(config: T.Config): string | void + fix(config: T.Config): void +} + const matchBitcoindConfig = shape({ rpc: shape({ enable: boolean, @@ -19,41 +24,109 @@ const matchBitcoindConfig = shape({ }), }); -export const dependencies: T.ExpectedExports.dependencies = { - bitcoind: { - // deno-lint-ignore require-await - async check(effects, configInput) { - effects.info("check bitcoind"); - const config = matchBitcoindConfig.unsafeCast(configInput); +const bitcoindChecks: Array<Check> = [ + { + currentError(config) { if (!matchBitcoindConfig.test(config)) { - return { error: "Bitcoind config is not the correct shape" }; + return 'Config is not the correct shape' } if (!config.rpc.enable) { - return { error: "Must have RPC enabled" }; + return 'Must have RPC enabled' + } + return + }, + fix(config) { + if (!matchBitcoindConfig.test(config)) { + return + } + config.rpc.enable = true + }, + }, + { + currentError(config) { + if (!matchBitcoindConfig.test(config)) { + return 'Config is not the correct shape' } if (!config.advanced.peers.listen) { - return { error: "Must have peer interface enabled" }; + return 'Must have peer interface enabled' + } + return + }, + fix(config) { + if (!matchBitcoindConfig.test(config)) { + return + } + config.advanced.peers.listen = true + }, + }, + { + currentError(config) { + if (!matchBitcoindConfig.test(config)) { + return 'Config is not the correct shape' } - if (config.advanced.pruning.mode !== "disabled") { - return { error: "Pruning must be disabled (must be an archival node)" }; + if (config.advanced.pruning.mode !== 'disabled') { + return 'Pruning must be disabled (must be an archival node)' } - if (config.rpc.advanced.threads < 4) { - return { error: "Must be greater than or equal to 4" }; + return + }, + fix(config) { + if (!matchBitcoindConfig.test(config)) { + return } - return { result: null }; + config.advanced.pruning.mode = 'disabled' }, + }, +] +export const dependencies: T.ExpectedExports.dependencies = { + bitcoind: { + // deno-lint-ignore require-await + async check(effects, configInput) { + effects.info('check bitcoind') + for (const checker of bitcoindChecks) { + const error = checker.currentError(configInput) + if (error) { + effects.error(`throwing error: ${error}`) + return { error } + } + } + return { result: null } + }, // deno-lint-ignore require-await async autoConfigure(effects, configInput) { - effects.info("autoconfigure bitcoind"); - const config = matchBitcoindConfig.unsafeCast(configInput); - config.rpc.enable = true; - config.advanced.peers.listen = true; - config.advanced.pruning.mode = "disabled"; - if (config.rpc.advanced.threads < 4) { - config.rpc.advanced.threads = 4; - } - return { result: config }; + effects.info('autoconfigure bitcoind') + for (const checker of bitcoindChecks) { + const error = checker.currentError(configInput) + if (error) { + checker.fix(configInput) + } + } + return { result: configInput } + }, + }, + 'bitcoind-testnet': { + // deno-lint-ignore require-await + async check(effects, configInput) { + effects.info('check bitcoind-testnet') + for (const checker of bitcoindChecks) { + const error = checker.currentError(configInput) + if (error) { + effects.error(`throwing error: ${error}`) + return { error } + } + } + return { result: null } + }, + // deno-lint-ignore require-await + async autoConfigure(effects, configInput) { + effects.info('autoconfigure bitcoind-testnet') + for (const checker of bitcoindChecks) { + const error = checker.currentError(configInput) + if (error) { + checker.fix(configInput) + } + } + return { result: configInput } }, }, }; diff --git a/scripts/services/getConfig.ts b/scripts/services/getConfig.ts index 0cd8825..7a301b1 100644 --- a/scripts/services/getConfig.ts +++ b/scripts/services/getConfig.ts @@ -10,25 +10,66 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({ target: "tor-address", interface: "electrum", }, - user: { - type: "pointer", - name: "RPC Username", - description: "The username for Bitcoin Core's RPC interface", - subtype: "package", - "package-id": "bitcoind", - target: "config", - multi: false, - selector: "$.rpc.username", - }, - password: { - type: "pointer", - name: "RPC Password", - description: "The password for Bitcoin Core's RPC interface", - subtype: "package", - "package-id": "bitcoind", - target: "config", - multi: false, - selector: "$.rpc.password", + "bitcoind": { + "type": "union", + "name": "Bitcoin Node", + "description": "The Bitcoin node type you would like to use for electrs", + "tag": { + "id": "type", + "name": "Select Bitcoin Node", + "variant-names": { + "bitcoind": "Bitcoin Core", + "bitcoind-testnet": "Bitcoin Core (testnet4)", + }, + "description": "The Bitcoin node type you would like to use for electrs", + }, + "default": "bitcoind", + "variants": { + "bitcoind": { + "username": { + "type": "pointer", + "name": "RPC Username", + "description": "The username for Bitcoin Core's RPC interface", + "subtype": "package", + "package-id": "bitcoind", + "target": "config", + "multi": false, + "selector": "$.rpc.username", + }, + "password": { + "type": "pointer", + "name": "RPC Password", + "description": "The password for Bitcoin Core's RPC interface", + "subtype": "package", + "package-id": "bitcoind", + "target": "config", + "multi": false, + "selector": "$.rpc.password", + }, + }, + "bitcoind-testnet": { + "username": { + "type": "pointer", + "name": "RPC Username", + "description": "The username for Bitcoin Core Testnet RPC interface", + "subtype": "package", + "package-id": "bitcoind-testnet", + "target": "config", + "multi": false, + "selector": "$.rpc.username", + }, + "password": { + "type": "pointer", + "name": "RPC Password", + "description": "The password for Bitcoin Core Testnet RPC interface", + "subtype": "package", + "package-id": "bitcoind-testnet", + "target": "config", + "multi": false, + "selector": "$.rpc.password", + }, + }, + }, }, "log-filters": { type: "enum", diff --git a/scripts/services/migrations.ts b/scripts/services/migrations.ts index 7093252..4e5139f 100644 --- a/scripts/services/migrations.ts +++ b/scripts/services/migrations.ts @@ -34,6 +34,38 @@ export const migration: T.ExpectedExports.migration = { version: "0.9.14.2", type: "down" } ), }, + "0.10.8.1": { + up: compat.migrations.updateConfig( + (config: any) => { + config.bitcoind = { + type: "bitcoind", + username: config.user, + password: config.password, + }; + + delete config.user; + delete config.password; + delete config.type; + + return config; + }, + true, + { version: "0.10.8.1", type: "up" } + ), + down: compat.migrations.updateConfig( + (config: any) => { + config.type = config.bitcoind.type; + config.user = config.bitcoind.username; + config.password = config.bitcoind.password; + + delete config.bitcoind; + + return config; + }, + true, + { version: "0.10.8.1", type: "down" } + ), + }, }, - "0.10.9" + "0.10.9.1" ); diff --git a/scripts/services/setConfig.ts b/scripts/services/setConfig.ts index 6a0fd1d..df9aa68 100644 --- a/scripts/services/setConfig.ts +++ b/scripts/services/setConfig.ts @@ -1,3 +1,24 @@ import { compat, types as T } from "../deps.ts"; -export const setConfig: T.ExpectedExports.setConfig = compat.setConfig; +// deno-lint-ignore require-await +export const setConfig: T.ExpectedExports.setConfig = async ( + effects: T.Effects, + newConfig: T.Config, +) => { + // deno-lint-ignore no-explicit-any + const dependsOnBitcoind: { [key: string]: string[] } = + (newConfig as any)?.bitcoind?.type === 'bitcoind' + ? { bitcoind: ['synced'] } + : {} + + // deno-lint-ignore no-explicit-any + const dependsOnBitcoindTestnet: { [key: string]: string[] } = + (newConfig as any)?.bitcoind?.type === 'bitcoind-testnet' + ? { 'bitcoind-testnet': ['synced'] } + : {} + + return compat.setConfig(effects, newConfig, { + ...dependsOnBitcoind, + ...dependsOnBitcoindTestnet, + }) +}