diff --git a/Cargo.lock b/Cargo.lock index 07ecbc70a7bb..5a2fe8538b18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,33 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-lc-rs" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.6.20" @@ -233,9 +260,9 @@ dependencies = [ [[package]] name = "axum-server" -version = "0.6.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" +checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" dependencies = [ "arc-swap", "bytes", @@ -246,10 +273,11 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "pin-project-lite", - "rustls 0.21.12", + "rustls 0.23.12", "rustls-pemfile 2.1.2", + "rustls-pki-types", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls 0.26.0", "tower", "tower-service", ] @@ -293,6 +321,29 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease 0.2.20", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.72", + "which", +] + [[package]] name = "bitcoin" version = "0.30.2" @@ -370,6 +421,19 @@ name = "cc" version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +dependencies = [ + "jobserver", + "libc", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] [[package]] name = "cfg-if" @@ -377,6 +441,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cln-grpc" version = "0.1.9" @@ -475,6 +550,15 @@ dependencies = [ "utoipa-swagger-ui", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -570,6 +654,12 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.13.0" @@ -668,6 +758,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.30" @@ -784,6 +880,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.26" @@ -1099,6 +1201,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -1114,12 +1225,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1211,6 +1338,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "multimap" version = "0.8.3" @@ -1322,6 +1455,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem" version = "3.0.4" @@ -1406,6 +1545,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.72", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1471,7 +1620,7 @@ dependencies = [ "log", "multimap", "petgraph", - "prettyplease", + "prettyplease 0.1.25", "prost", "prost-types", "regex", @@ -1798,28 +1947,17 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring 0.17.8", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ + "aws-lc-rs", "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.6", + "rustls-webpki", "subtle", "zeroize", ] @@ -1849,22 +1987,13 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - [[package]] name = "rustls-webpki" version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ + "aws-lc-rs", "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", @@ -2006,6 +2135,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "slab" version = "0.4.9" @@ -2262,16 +2397,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" @@ -2390,7 +2515,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ - "prettyplease", + "prettyplease 0.1.25", "proc-macro2", "prost-build", "quote", @@ -3035,6 +3160,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] [[package]] name = "zip" diff --git a/plugins/clnrest-plugin/Cargo.toml b/plugins/clnrest-plugin/Cargo.toml index 6adf07cd3d94..fed20e9d4166 100644 --- a/plugins/clnrest-plugin/Cargo.toml +++ b/plugins/clnrest-plugin/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1" tokio-util = { version = "0.7", features = ["codec"] } tokio = { version="1", features = ['io-std', 'rt-multi-thread', 'sync', 'macros', 'io-util'] } axum = "0.7" -axum-server = { version = "0.6", features = ["tls-rustls"] } +axum-server = { version = "0.7", features = ["tls-rustls"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } rcgen = "0.13" hyper = "1" diff --git a/plugins/clnrest-plugin/src/handlers.rs b/plugins/clnrest-plugin/src/handlers.rs index 947f40edbb6f..7c2deb868072 100644 --- a/plugins/clnrest-plugin/src/handlers.rs +++ b/plugins/clnrest-plugin/src/handlers.rs @@ -1,8 +1,8 @@ -use std::{collections::HashMap, process}; +use std::{collections::hash_map::Entry, process}; use anyhow::anyhow; use axum::{ - body::{to_bytes, Body}, + body::Body, extract::{Extension, Json, Path, State}, http::{Request, StatusCode}, middleware::Next, @@ -17,8 +17,11 @@ use serde_json::json; use socketioxide::extract::{Data, SocketRef}; use crate::{ - shared::{call_rpc, filter_json, verify_rune}, - PluginState, SWAGGER_FALLBACK, + shared::{ + call_rpc, filter_json, generate_response, get_clnrest_manifests, get_content_type, + get_plugin_methods, handle_custom_paths, merge_params, parse_request_body, verify_rune, + }, + ClnrestMap, PluginState, SWAGGER_FALLBACK, }; #[derive(Debug)] @@ -55,7 +58,13 @@ impl IntoResponse for AppError { pub async fn list_methods( Extension(plugin): Extension>, ) -> Result, AppError> { - match call_rpc(plugin, "help", json!(HelpRequest { command: None })).await { + match call_rpc( + &plugin.configuration().rpc_file, + "help", + Some(json!(HelpRequest { command: None })), + ) + .await + { Ok(help_response) => { let html_content = process_help_response(help_response); Ok(Html(html_content)) @@ -76,7 +85,14 @@ fn process_help_response(help_response: serde_json::Value) -> String { let mut processed_html_res = String::new(); for row in processed_res.help { - processed_html_res.push_str(&format!("Command: {}\n", row.command)); + processed_html_res.push_str(&format!("Command: {}
", row.command)); + if let Some(clnrest) = row.clnrest { + processed_html_res.push_str(&format!("Clnrest path:: {}\n", clnrest.path)); + processed_html_res.push_str(&format!("Clnrest method: {}\n", clnrest.method)); + processed_html_res + .push_str(&format!("Clnrest content-type: {}\n", clnrest.content_type)); + processed_html_res.push_str(&format!("Clnrest rune: {}\n", clnrest.rune)); + } processed_html_res.push_str(line); } @@ -86,9 +102,9 @@ fn process_help_response(help_response: serde_json::Value) -> String { // Handler for calling RPC methods #[utoipa::path( post, - path = "/v1/{rpc_method}", + path = "/v1/{rpc_method_or_path}", responses( - (status = 201, description = "Call rpc method", body = serde_json::Value), + (status = 201, description = "Call rpc method by name or custom path", body = serde_json::Value), (status = 401, description = "Unauthorized", body = serde_json::Value), (status = 403, description = "Forbidden", body = serde_json::Value), (status = 404, description = "Not Found", body = serde_json::Value), @@ -98,67 +114,93 @@ fn process_help_response(help_response: serde_json::Value) -> String { example = json!({}) ), security(("api_key" = [])) )] -pub async fn call_rpc_method( - Path(rpc_method): Path, +pub async fn post_rpc_method( + Path(path): Path, headers: axum::http::HeaderMap, Extension(plugin): Extension>, body: Request, ) -> Result { - let rune = headers - .get("rune") - .and_then(|v| v.to_str().ok()) - .map(String::from); + let (rpc_method, path_params, rest_map) = handle_custom_paths(&plugin, &path, "POST").await?; - let bytes = match to_bytes(body.into_body(), usize::MAX).await { - Ok(o) => o, - Err(e) => { - return Err(AppError::InternalServerError(RpcError { - code: None, - data: None, - message: format!("Could not read request body: {}", e), - })) - } - }; - - let mut rpc_params = match serde_json::from_slice(&bytes) { - Ok(o) => o, - Err(e1) => { - // it's not json but a form instead - let form_str = String::from_utf8(bytes.to_vec()).unwrap(); - let mut form_data = HashMap::new(); - for pair in form_str.split('&') { - let mut kv = pair.split('='); - if let (Some(key), Some(value)) = (kv.next(), kv.next()) { - form_data.insert(key.to_string(), value.to_string()); - } - } - match serde_json::to_value(form_data) { - Ok(o) => o, - Err(e2) => { - return Err(AppError::InternalServerError(RpcError { - code: None, - data: None, - message: format!( - "Could not parse json from form data: {}\ - Original serde_json error: {}", - e2, e1 - ), - })) + let mut rpc_params = parse_request_body(body).await?; + + filter_json(&mut rpc_params); + + merge_params(&mut rpc_params, path_params)?; + + if rest_map.as_ref().map_or_else(|| true, |map| map.rune) { + let rune = headers + .get("rune") + .and_then(|v| v.to_str().ok()) + .map(String::from); + verify_rune( + &plugin.configuration().rpc_file, + rune, + &rpc_method, + Some(rpc_params.clone()), + ) + .await?; + } + + let content_type = get_content_type(rest_map)?; + + match call_rpc( + &plugin.configuration().rpc_file, + &rpc_method, + Some(rpc_params), + ) + .await + { + Ok(result) => Ok(generate_response(result, content_type)), + Err(err) => { + if let Some(code) = err.code { + if code == -32601 { + return Err(AppError::NotFound(err)); } } + Err(AppError::InternalServerError(err)) } - }; + } +} - filter_json(&mut rpc_params); +// Handler for calling RPC methods +#[utoipa::path( + get, + path = "/v1/{rpc_method_or_path}", + responses( + (status = 201, description = "Call rpc method by name or custom path", body = serde_json::Value), + (status = 401, description = "Unauthorized", body = serde_json::Value), + (status = 403, description = "Forbidden", body = serde_json::Value), + (status = 404, description = "Not Found", body = serde_json::Value), + (status = 500, description = "Server Error", body = serde_json::Value) + ), + security(("api_key" = [])) +)] +pub async fn get_rpc_method( + Path(path): Path, + headers: axum::http::HeaderMap, + Extension(plugin): Extension>, +) -> Result { + let (rpc_method, path_params, rest_map) = handle_custom_paths(&plugin, &path, "GET").await?; - verify_rune(plugin.clone(), rune, &rpc_method, &rpc_params).await?; + if rest_map.as_ref().map_or_else(|| true, |map| map.rune) { + let rune = headers + .get("rune") + .and_then(|v| v.to_str().ok()) + .map(String::from); + verify_rune( + &plugin.configuration().rpc_file, + rune, + &rpc_method, + path_params.clone(), + ) + .await?; + } - match call_rpc(plugin, &rpc_method, rpc_params).await { - Ok(result) => { - let response_body = Json(result); - let response = (StatusCode::CREATED, response_body).into_response(); - Ok(response) - } + let content_type = get_content_type(rest_map)?; + + match call_rpc(&plugin.configuration().rpc_file, &rpc_method, path_params).await { + Ok(result) => Ok(generate_response(result, content_type)), Err(err) => { if let Some(code) = err.code { if code == -32601 { @@ -178,11 +220,42 @@ pub async fn handle_notification( plugin: Plugin, value: serde_json::Value, ) -> Result<(), anyhow::Error> { + log::debug!("notification: {}", value.to_string()); if let Some(sht) = value.get("shutdown") { log::info!("Got shutdown notification: {}", sht); // This seems to error when subscribing to "*" notifications _ = plugin.shutdown(); process::exit(0); + } else if let Some(p_started) = value.get("plugin_started") { + let rpc_methods = get_plugin_methods(p_started); + + let manifests = get_clnrest_manifests(&plugin.configuration().rpc_file).await?; + let mut rest_paths = plugin.state().rest_paths.lock().unwrap(); + for rpc_method in rpc_methods.into_iter() { + let clnrest_data = match manifests.get(&rpc_method) { + Some(c) => c.clone(), + None => continue, + }; + if let Entry::Vacant(entry) = rest_paths.entry(clnrest_data.path.clone()) { + log::info!( + "Registered custom path `{}` for `{}` via `{}`", + clnrest_data.path, + rpc_method, + clnrest_data.method + ); + entry.insert(ClnrestMap { + content_type: clnrest_data.content_type, + http_method: clnrest_data.method, + rpc_method, + rune: clnrest_data.rune, + }); + } + } + } else if let Some(p_stopped) = value.get("plugin_stopped") { + let rpc_methods = get_plugin_methods(p_stopped); + + let mut rest_paths = plugin.state().rest_paths.lock().unwrap(); + rest_paths.retain(|_, v| !rpc_methods.contains(&v.rpc_method)) } match plugin.state().notification_sender.send(value).await { Ok(()) => Ok(()), @@ -213,7 +286,14 @@ pub async fn header_inspection_middleware( .map(String::from); if upgrade.is_some() { - match verify_rune(plugin, rune, "listclnrest-notifications", &json!({})).await { + match verify_rune( + &plugin.configuration().rpc_file, + rune, + "listclnrest-notifications", + None, + ) + .await + { Ok(()) => Ok(next.run(req).await), Err(e) => Err(e), } diff --git a/plugins/clnrest-plugin/src/main.rs b/plugins/clnrest-plugin/src/main.rs index 6392b1d223e3..8e7a93c4fc40 100644 --- a/plugins/clnrest-plugin/src/main.rs +++ b/plugins/clnrest-plugin/src/main.rs @@ -1,19 +1,25 @@ -use std::{net::SocketAddr, str::FromStr}; +use std::{ + collections::{hash_map::Entry, HashMap}, + net::SocketAddr, + str::FromStr, + sync::{Arc, Mutex}, +}; use axum::{ http::{HeaderName, HeaderValue}, middleware, - routing::{get, post}, + routing::get, Extension, Router, }; use axum_server::tls_rustls::RustlsConfig; use certs::generate_certificates; use cln_plugin::Builder; use handlers::{ - call_rpc_method, handle_notification, header_inspection_middleware, list_methods, - socketio_on_connect, + get_rpc_method, handle_notification, header_inspection_middleware, list_methods, + post_rpc_method, socketio_on_connect, }; use options::*; +use shared::get_clnrest_manifests; use socketioxide::SocketIo; use tokio::sync::mpsc::{self, Receiver, Sender}; use tower::ServiceBuilder; @@ -35,13 +41,23 @@ mod shared; #[derive(Clone, Debug)] struct PluginState { notification_sender: Sender, + rest_paths: Arc>>, +} + +#[derive(Debug, Clone)] +pub struct ClnrestMap { + pub content_type: String, + pub http_method: String, + pub rpc_method: String, + pub rune: bool, } #[derive(OpenApi)] #[openapi( paths( handlers::list_methods, - handlers::call_rpc_method, + handlers::post_rpc_method, + handlers::get_rpc_method ), modifiers(&SecurityAddon), )] @@ -96,8 +112,14 @@ async fn main() -> Result<(), anyhow::Error> { let (notify_tx, notify_rx) = mpsc::channel(100); + let rest_paths = match rest_manifests_init(&plugin.configuration().rpc_file).await { + Ok(rest) => rest, + Err(e) => return plugin.disable(&e.to_string()).await, + }; + let state = PluginState { notification_sender: notify_tx, + rest_paths: Arc::new(Mutex::new(rest_paths)), }; let plugin = plugin.start(state.clone()).await?; @@ -131,7 +153,7 @@ async fn main() -> Result<(), anyhow::Error> { "/v1", Router::new() .route("/list-methods", get(list_methods)) - .route("/:rpc_method", post(call_rpc_method)) + .route("/*route", get(get_rpc_method).post(post_rpc_method)) .layer(clnrest_options.cors) .layer(Extension(plugin.clone())) .layer( @@ -194,3 +216,28 @@ async fn notification_background_task(io: SocketIo, mut receiver: Receiver Result, anyhow::Error> { + let manifests = get_clnrest_manifests(rpc_file).await?; + let mut rest_paths: HashMap = HashMap::new(); + for (rpc_method, clnrest_data) in manifests.into_iter() { + if let Entry::Vacant(entry) = rest_paths.entry(clnrest_data.path.clone()) { + log::info!( + "Registered custom path `{}` for `{}` via `{}`", + clnrest_data.path, + rpc_method, + clnrest_data.method + ); + entry.insert(ClnrestMap { + content_type: clnrest_data.content_type, + http_method: clnrest_data.method, + rpc_method, + rune: clnrest_data.rune, + }); + } + } + + Ok(rest_paths) +} diff --git a/plugins/clnrest-plugin/src/shared.rs b/plugins/clnrest-plugin/src/shared.rs index 129d23484351..839f6ef4bd8c 100644 --- a/plugins/clnrest-plugin/src/shared.rs +++ b/plugins/clnrest-plugin/src/shared.rs @@ -1,22 +1,36 @@ +use std::collections::HashMap; + +use anyhow::anyhow; +use axum::{ + body::{to_bytes, Body}, + extract::Request, + http::HeaderValue, + response::{IntoResponse, Response}, +}; use cln_plugin::Plugin; -use cln_rpc::{model::responses::CheckruneResponse, ClnRpc, RpcError}; -use serde_json::json; +use cln_rpc::{ + model::responses::{CheckruneResponse, HelpHelpClnrest, HelpResponse}, + ClnRpc, RpcError, +}; +use hyper::{header, StatusCode}; +use serde_json::{json, Map}; -use crate::{handlers::AppError, PluginState}; +use crate::{handlers::AppError, ClnrestMap, PluginState}; pub async fn verify_rune( - plugin: Plugin, + rpc_file: &String, rune_header: Option, rpc_method: &str, - rpc_params: &serde_json::Value, + rpc_params: Option, ) -> Result<(), AppError> { + let rpc_params = rpc_params.unwrap_or_else(|| json!({})); if let Some(rune) = rune_header { match call_rpc( - plugin, + rpc_file, "checkrune", - json!( {"rune": rune, + Some(json!( {"rune": rune, "method": rpc_method, - "params": rpc_params}), + "params": rpc_params})), ) .await { @@ -51,12 +65,12 @@ pub async fn verify_rune( } pub async fn call_rpc( - plugin: Plugin, + rpc_file: &String, method: &str, - params: serde_json::Value, + params: Option, ) -> Result { - let rpc_path = plugin.configuration().rpc_file; - let mut rpc = ClnRpc::new(rpc_path).await.map_err(|e| RpcError { + let params = params.unwrap_or_else(|| json!({})); + let mut rpc = ClnRpc::new(rpc_file).await.map_err(|e| RpcError { code: None, data: None, message: e.to_string(), @@ -64,6 +78,127 @@ pub async fn call_rpc( rpc.call_raw(method, ¶ms).await } +pub async fn parse_request_body(body: Request) -> Result { + let bytes = match to_bytes(body.into_body(), usize::MAX).await { + Ok(o) => o, + Err(e) => { + return Err(AppError::InternalServerError(RpcError { + code: None, + data: None, + message: format!("Could not read request body: {}", e), + })) + } + }; + serde_json::from_slice(&bytes).or_else(|_| { + let form_str = String::from_utf8(bytes.to_vec()).unwrap(); + let mut form_data = serde_json::Map::new(); + for pair in form_str.split('&') { + let mut kv = pair.split('='); + if let (Some(key), Some(value)) = (kv.next(), kv.next()) { + form_data.insert( + key.to_string(), + serde_json::Value::String(value.to_string()), + ); + } + } + Ok(serde_json::Value::Object(form_data)) + }) +} + +pub fn merge_params( + rpc_params: &mut serde_json::Value, + path_params: Option, +) -> Result<(), AppError> { + if let Some(serde_json::Value::Object(extr_p)) = path_params { + match rpc_params { + serde_json::Value::Object(a_map) => { + for (k, v) in extr_p { + a_map.insert(k, v); + } + } + a => { + return Err(AppError::InternalServerError(RpcError { + code: None, + data: None, + message: format!("Could not parse params as object: {}", a), + })) + } + } + } + Ok(()) +} + +pub async fn handle_custom_paths( + plugin: &Plugin, + path: &str, + method: &str, +) -> Result<(String, Option, Option), AppError> { + let custom_paths = plugin.state().rest_paths.lock().unwrap().clone(); + let mut rest_map = None; + let mut extra_params = None; + let most_specific_match = match match_path(&("/".to_string() + path), method, &custom_paths) { + Ok(m) => m.into_iter().min_by_key(|(_, params)| params.len()), + Err(e) => { + return Err(AppError::InternalServerError(RpcError { + code: Some(-1), + message: e.to_string(), + data: None, + })) + } + }; + if let Some((custom_path, custom_params)) = most_specific_match { + rest_map = Some(custom_paths.get(&custom_path).unwrap().clone()); + extra_params = Some(serde_json::Value::Object(custom_params)); + } + + if let Some(rm) = &rest_map { + if !rm.http_method.eq_ignore_ascii_case(method) { + return Err(AppError::NotFound(RpcError { + code: Some(-32601), + data: None, + message: "Wrong http method".to_string(), + })); + } + } + + let rpc_method = if let Some(rp) = &rest_map { + rp.rpc_method.clone() + } else if path.contains('/') { + return Err(AppError::NotFound(RpcError { + code: Some(-32601), + message: "Path not registered".to_string(), + data: None, + })); + } else { + path.to_string() + }; + + Ok((rpc_method, extra_params, rest_map)) +} + +pub fn generate_response(result: serde_json::Value, content_type: HeaderValue) -> Response { + let body = Body::new(result.to_string()); + let mut response = (StatusCode::CREATED, body).into_response(); + response + .headers_mut() + .insert(header::CONTENT_TYPE, content_type); + response +} + +pub fn get_content_type(rest_map: Option) -> Result { + if let Some(rm) = rest_map { + Ok(HeaderValue::from_str(&rm.content_type).map_err(|_| { + AppError::InternalServerError(RpcError { + code: Some(-1), + message: format!("Invalid content-type: `{}`", rm.content_type), + data: None, + }) + })?) + } else { + Ok(HeaderValue::from_static("application/json")) + } +} + pub fn filter_json(value: &mut serde_json::Value) { match value { serde_json::Value::Array(arr) => { @@ -90,3 +225,170 @@ fn is_unwanted(key: &String, value: &serde_json::Value) -> bool { _ => false, } } + +pub async fn get_clnrest_manifests( + rpc_file: &String, +) -> Result, anyhow::Error> { + let help: HelpResponse = serde_json::from_value(call_rpc(rpc_file, "help", None).await?)?; + let mut help_map = HashMap::new(); + for help in help.help { + if let Some(clnrest_help) = help.clnrest { + let command_name = if let Some((name, _args)) = help.command.split_once(' ') { + name.to_string() + } else { + help.command + }; + help_map.insert(command_name, clnrest_help); + } + } + Ok(help_map) +} + +pub fn get_plugin_methods(input: &serde_json::Value) -> Vec { + input + .get("methods") + .and_then(|m| m.as_array()) + .map_or_else(Vec::new, |array| { + array + .iter() + .filter_map(|s| s.as_str().map(String::from)) + .collect() + }) +} + +pub fn match_path( + path: &str, + http_method: &str, + rest_map: &HashMap, +) -> Result>, anyhow::Error> { + let path_parts: Vec<&str> = path.split('/').collect(); + + let mut matches = HashMap::new(); + + 'outer: for (pattern, map) in rest_map.iter() { + let pattern_parts: Vec<&str> = pattern.split('/').collect(); + if path_parts.len() != pattern_parts.len() { + continue; + } + if !map.http_method.eq_ignore_ascii_case(http_method) { + continue; + } + let mut params = Map::new(); + if path.eq(pattern) && !pattern.contains('<') { + matches.insert(pattern.clone(), params); + continue; + } + let mut unambiguous_match = false; + for (path_part, pattern_part) in path_parts.iter().zip(pattern_parts.iter()) { + if pattern_part.starts_with('<') && pattern_part.ends_with('>') { + unambiguous_match = unambiguous_match || !path_part.starts_with('<'); + + let key = &pattern_part[1..pattern_part.len() - 1]; + params.insert( + key.to_string(), + serde_json::Value::String(path_part.to_string()), + ); + } else if path_part.starts_with('<') && path_part.ends_with('>') { + unambiguous_match = true; + } else if path_part != pattern_part { + continue 'outer; + } + } + if !unambiguous_match { + return Err(anyhow!("Ambiguous path: {}", path)); + } + matches.insert(pattern.clone(), params); + } + Ok(matches) +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_match_path() { + let http_get1 = "GET"; + let http_get2 = "get"; + let http_post1 = "POST"; + + let mut rest_map = HashMap::new(); + + let path1 = "/test/me/now"; + let path2 = "//me/now"; + let path3 = "///now"; + let path4 = "//me/"; + let path_map_get = ClnrestMap { + content_type: "N/A".to_string(), + http_method: "GET".to_string(), + rpc_method: "N/A".to_string(), + rune: false, + }; + let path_map_post = ClnrestMap { + content_type: "N/A".to_string(), + http_method: "POST".to_string(), + rpc_method: "N/A".to_string(), + rune: false, + }; + + assert!(match_path(path1, http_get1, &rest_map).unwrap().is_empty()); + assert!(match_path(path1, http_get2, &rest_map).unwrap().is_empty()); + rest_map.insert(path1.to_string(), path_map_get.clone()); + assert!(match_path(path1, http_get1, &rest_map).unwrap().len() == 1); + assert!(match_path(path1, http_get2, &rest_map).unwrap().len() == 1); + assert!(match_path(path1, http_post1, &rest_map).unwrap().is_empty()); + assert!(match_path(path2, http_get1, &rest_map).unwrap().len() == 1); + rest_map.insert(path2.to_string(), path_map_get.clone()); + assert!(match_path(path2, http_get1, &rest_map).is_err()); + assert!(match_path(path3, http_get1, &rest_map).unwrap().len() == 2); + rest_map.insert(path3.to_string(), path_map_get.clone()); + assert!(match_path(path3, http_get1, &rest_map).is_err()); + assert!(match_path(path4, http_get1, &rest_map).unwrap().len() == 3); + rest_map.insert(path4.to_string(), path_map_get.clone()); + assert!(match_path(path4, http_get1, &rest_map).is_err()); + assert!(match_path(path1, http_get1, &rest_map) + .unwrap() + .into_iter() + .min_by_key(|(_, y)| y.len()) + .unwrap() + .1 + .is_empty()); + assert!( + match_path("/path/me/now", http_get1, &rest_map) + .unwrap() + .into_iter() + .min_by_key(|(_, y)| y.len()) + .unwrap() + .1 + .len() + == 1 + ); + assert!( + match_path("/path/to/now", http_get1, &rest_map) + .unwrap() + .into_iter() + .min_by_key(|(_, y)| y.len()) + .unwrap() + .1 + .len() + == 2 + ); + assert!( + match_path("/path/me/to", http_get1, &rest_map) + .unwrap() + .into_iter() + .min_by_key(|(_, y)| y.len()) + .unwrap() + .1 + .len() + == 2 + ); + assert!(match_path("/path/to/me", http_get1, &rest_map) + .unwrap() + .is_empty()); + assert!(match_path(path1, http_post1, &rest_map).unwrap().is_empty()); + rest_map.insert(path1.to_string(), path_map_post.clone()); + assert!(match_path("/test", http_post1, &rest_map) + .unwrap() + .is_empty()); + } +} diff --git a/tests/plugins/get_manifest.py b/tests/plugins/get_manifest.py index 918ff1c95411..394401cba790 100755 --- a/tests/plugins/get_manifest.py +++ b/tests/plugins/get_manifest.py @@ -17,4 +17,58 @@ def return_this_manifest(plugin, cmd_name: str =None): return cmd_manifest[0] + +@plugin.method("dyncheckmymanifestpost", clnrest_data={"path": "/user//me", "method": "POST"}) +def return_this_dyn_manifest_post(plugin, cmd_name: str =None, id: int = 0): + """Returns the manifest of this plugin.""" + + name_to_check = cmd_name if cmd_name else "dyncheckmymanifestpost" + + cmd_manifest = plugin.rpc.help(name_to_check).get("help") + + if cmd_manifest is []: + raise ValueError(f"Command {name_to_check} not found.") + + cmd_manifest[0]["dyn_id_post"] = id + + return cmd_manifest[0] + + +@plugin.method("dyncheckmymanifestget", clnrest_data={"path": "/stats//me", "method": "GET", "rune": False}) +def return_this_dyn_manifest_get(plugin, cmd_name: str =None, id: int = 0): + """Returns the manifest of this plugin.""" + + name_to_check = cmd_name if cmd_name else "dyncheckmymanifestget" + + cmd_manifest = plugin.rpc.help(name_to_check).get("help") + + if cmd_manifest is []: + raise ValueError(f"Command {name_to_check} not found.") + + cmd_manifest[0]["dyn_id_get"] = id + + return cmd_manifest[0] + + +@plugin.method("dyncheckmymanifestget2", clnrest_data={"path": "///me", "method": "GET", "rune": False, "content_type": "text/plain"}) +def return_this_dyn_manifest_get2(plugin, cmd_name: str =None, id: int = 0): + """Returns the manifest of this plugin.""" + + result = f"{cmd_name} {id}" + return result + + +@plugin.method("dyncheckmymanifestget3", clnrest_data={"path": "/stats/to/me", "method": "GET", "rune": False}) +def return_this_dyn_manifest_get3(plugin, cmd_name: str =None): + """Returns the manifest of this plugin.""" + + name_to_check = cmd_name if cmd_name else "dyncheckmymanifestget3" + + cmd_manifest = plugin.rpc.help(name_to_check).get("help") + + if cmd_manifest is []: + raise ValueError(f"Command {name_to_check} not found.") + + return cmd_manifest[0] + plugin.run() diff --git a/tests/test_clnrest.py b/tests/test_clnrest.py index c90d5f70c963..9daa462ecd9f 100644 --- a/tests/test_clnrest.py +++ b/tests/test_clnrest.py @@ -1,15 +1,16 @@ -from fixtures import * # noqa: F401,F403 -from pyln.testing.utils import env, TEST_NETWORK -from pyln.client import Millisatoshi -import unittest import os -import requests +import time +import unittest from pathlib import Path + +import pytest +import requests +import socketio +from fixtures import * # noqa: F401,F403 +from pyln.client import Millisatoshi +from pyln.testing.utils import TEST_NETWORK, env from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -import socketio -import time -import pytest def http_session_with_retry(): @@ -20,7 +21,7 @@ def http_session_with_retry(): http_session = requests.Session() retry = Retry(connect=10, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) - http_session.mount('https://', adapter) + http_session.mount("https://", adapter) return http_session @@ -29,46 +30,62 @@ def test_clnrest_no_auto_start(node_factory): l1 = node_factory.get_node() # This might happen really early! l1.daemon.logsearch_start = 0 - assert [p for p in l1.rpc.plugin('list')['plugins'] if 'clnrest' in p['name']] == [] - assert l1.daemon.is_in_log(r'plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` option is not configured') + assert [p for p in l1.rpc.plugin("list")["plugins"] if "clnrest" in p["name"]] == [] + assert l1.daemon.is_in_log( + r"plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` option is not configured" + ) def test_clnrest_self_signed_certificates(node_factory): """Test that self-signed certificates have `clnrest-host` IP in Subject Alternative Name.""" rest_port = str(node_factory.get_unused_port()) - rest_host = '127.0.0.1' - base_url = f'https://{rest_host}:{rest_port}' - l1 = node_factory.get_node(options={'disable-plugin': 'cln-grpc', - 'clnrest-port': rest_port, - 'clnrest-host': rest_host}) + rest_host = "127.0.0.1" + base_url = f"https://{rest_host}:{rest_port}" + l1 = node_factory.get_node( + options={ + "disable-plugin": "cln-grpc", + "clnrest-port": rest_port, + "clnrest-host": rest_host, + } + ) # This might happen really early! l1.daemon.logsearch_start = 0 - l1.daemon.wait_for_log(r'plugin-clnrest: REST server running at ' + base_url) - ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem' + l1.daemon.wait_for_log(r"plugin-clnrest: REST server running at " + base_url) + ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / "ca.pem" http_session = http_session_with_retry() - response = http_session.get(base_url + '/v1/list-methods', verify=ca_cert) + response = http_session.get(base_url + "/v1/list-methods", verify=ca_cert) assert response.status_code == 200 -@unittest.skipIf(env('RUST') != '1', 'RUST is not enabled skipping rust-dependent tests') +@unittest.skipIf( + env("RUST") != "1", "RUST is not enabled skipping rust-dependent tests" +) def test_clnrest_uses_grpc_plugin_certificates(node_factory): """Test that clnrest reuses `cln-grpc` plugin certificates if available. Defaults: - clnrest-protocol: https """ - rest_host = 'localhost' + rest_host = "localhost" grpc_port = str(node_factory.get_unused_port()) rest_port = str(node_factory.get_unused_port()) - l1 = node_factory.get_node(options={'grpc-port': grpc_port, 'clnrest-host': rest_host, 'clnrest-port': rest_port}) - base_url = f'https://{rest_host}:{rest_port}' + l1 = node_factory.get_node( + options={ + "grpc-port": grpc_port, + "clnrest-host": rest_host, + "clnrest-port": rest_port, + } + ) + base_url = f"https://{rest_host}:{rest_port}" # This might happen really early! l1.daemon.logsearch_start = 0 - l1.daemon.wait_for_logs([r'serving grpc on 0.0.0.0:', - r'plugin-clnrest: REST server running at ' + base_url]) - ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem' + l1.daemon.wait_for_logs([ + r"serving grpc on 0.0.0.0:", + r"plugin-clnrest: REST server running at " + base_url, + ]) + ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / "ca.pem" http_session = http_session_with_retry() - response = http_session.get(base_url + '/v1/list-methods', verify=ca_cert) + response = http_session.get(base_url + "/v1/list-methods", verify=ca_cert) assert response.status_code == 200 @@ -76,28 +93,36 @@ def test_clnrest_generate_certificate(node_factory): """Test whether we correctly generate the certificates.""" # when `clnrest-protocol` is `http`, certs are not generated at `clnrest-certs` path rest_port = str(node_factory.get_unused_port()) - rest_protocol = 'http' - rest_certs = node_factory.directory + '/clnrest-certs' - l1 = node_factory.get_node(options={'clnrest-port': rest_port, - 'clnrest-protocol': rest_protocol, - 'clnrest-certs': rest_certs}) + rest_protocol = "http" + rest_certs = node_factory.directory + "/clnrest-certs" + l1 = node_factory.get_node( + options={ + "clnrest-port": rest_port, + "clnrest-protocol": rest_protocol, + "clnrest-certs": rest_certs, + } + ) assert not Path(rest_certs).exists() # node l1 not started rest_port = str(node_factory.get_unused_port()) - rest_certs = node_factory.directory + '/clnrest-certs' - l1 = node_factory.get_node(options={'clnrest-port': rest_port, - 'clnrest-certs': rest_certs}, start=False) + rest_certs = node_factory.directory + "/clnrest-certs" + l1 = node_factory.get_node( + options={"clnrest-port": rest_port, "clnrest-certs": rest_certs}, start=False + ) rest_certs_path = Path(rest_certs) - files = [rest_certs_path / f for f in [ - 'ca.pem', - 'ca-key.pem', - 'client.pem', - 'client-key.pem', - 'server-key.pem', - 'server.pem', - ]] + files = [ + rest_certs_path / f + for f in [ + "ca.pem", + "ca-key.pem", + "client.pem", + "client-key.pem", + "server-key.pem", + "server.pem", + ] + ] # before starting no files exist. assert [f.exists() for f in files] == [False] * len(files) @@ -131,13 +156,15 @@ def start_node_with_clnrest(node_factory): - the base url and - the certificate authority path used for the self-signed certificates.""" rest_port = str(node_factory.get_unused_port()) - rest_certs = node_factory.directory + '/clnrest-certs' - l1 = node_factory.get_node(options={'clnrest-port': rest_port, 'clnrest-certs': rest_certs}) - base_url = 'https://127.0.0.1:' + rest_port + rest_certs = node_factory.directory + "/clnrest-certs" + l1 = node_factory.get_node( + options={"clnrest-port": rest_port, "clnrest-certs": rest_certs} + ) + base_url = "https://127.0.0.1:" + rest_port # This might happen really early! l1.daemon.logsearch_start = 0 - l1.daemon.wait_for_log(r'plugin-clnrest: REST server running at ' + base_url) - ca_cert = Path(rest_certs) / 'ca.pem' + l1.daemon.wait_for_log(r"plugin-clnrest: REST server running at " + base_url) + ca_cert = Path(rest_certs) / "ca.pem" return l1, base_url, ca_cert @@ -148,9 +175,9 @@ def test_clnrest_list_methods(node_factory): # /v1/list-methods http_session = http_session_with_retry() - response = http_session.get(base_url + '/v1/list-methods', verify=ca_cert) + response = http_session.get(base_url + "/v1/list-methods", verify=ca_cert) assert response.status_code == 200 - assert response.text.find('getinfo') > 0 + assert response.text.find("getinfo") > 0 def test_clnrest_unknown_method(node_factory): @@ -159,15 +186,21 @@ def test_clnrest_unknown_method(node_factory): l1, base_url, ca_cert = start_node_with_clnrest(node_factory) http_session = http_session_with_retry() - response = http_session.get(base_url + '/v1/unknown-get', verify=ca_cert) - assert response.status_code == 405 + rune = l1.rpc.createrune()["rune"] + response = http_session.get( + base_url + "/v1/unknown-get", headers={"Rune": rune}, verify=ca_cert + ) + assert response.status_code == 404 + assert response.json()["code"] == -32601 + assert response.json()["message"] == "Unknown command 'unknown-get'" """Test POST request error on `/v1/unknown-post` end point.""" - rune = l1.rpc.createrune()['rune'] - response = http_session.post(base_url + '/v1/unknown-post', headers={'Rune': rune}, verify=ca_cert) + response = http_session.post( + base_url + "/v1/unknown-post", headers={"Rune": rune}, verify=ca_cert + ) assert response.status_code == 404 - assert response.json()['code'] == -32601 - assert response.json()['message'] == "Unknown command 'unknown-post'" + assert response.json()["code"] == -32601 + assert response.json()["message"] == "Unknown command 'unknown-post'" def test_clnrest_rpc_method(node_factory): @@ -177,52 +210,61 @@ def test_clnrest_rpc_method(node_factory): http_session = http_session_with_retry() # /v1/getinfo no rune provided in header of the request - response = http_session.post(base_url + '/v1/getinfo', verify=ca_cert) + response = http_session.post(base_url + "/v1/getinfo", verify=ca_cert) assert response.status_code == 403 - assert response.json()['code'] == 1501 - assert response.json()['message'] == 'Not authorized: Missing rune' + assert response.json()["code"] == 1501 + assert response.json()["message"] == "Not authorized: Missing rune" # /v1/getinfo with a rune which doesn't authorized getinfo method - rune_no_getinfo = l1.rpc.createrune(restrictions=[["method/getinfo"]])['rune'] - response = http_session.post(base_url + '/v1/getinfo', headers={'Rune': rune_no_getinfo}, - verify=ca_cert) + rune_no_getinfo = l1.rpc.createrune(restrictions=[["method/getinfo"]])["rune"] + response = http_session.post( + base_url + "/v1/getinfo", headers={"Rune": rune_no_getinfo}, verify=ca_cert + ) assert response.status_code == 401 - assert response.json()['code'] == 1502 - assert response.json()['message'] == 'Not permitted: method is equal to getinfo' + assert response.json()["code"] == 1502 + assert response.json()["message"] == "Not permitted: method is equal to getinfo" # /v1/getinfo with a correct rune - rune_getinfo = l1.rpc.createrune(restrictions=[["method=getinfo"]])['rune'] - response = http_session.post(base_url + '/v1/getinfo', headers={'Rune': rune_getinfo}, - verify=ca_cert) + rune_getinfo = l1.rpc.createrune(restrictions=[["method=getinfo"]])["rune"] + response = http_session.post( + base_url + "/v1/getinfo", headers={"Rune": rune_getinfo}, verify=ca_cert + ) assert response.status_code == 201 - assert response.json()['id'] == l1.info['id'] + assert response.json()["id"] == l1.info["id"] # /v1/invoice with a correct rune but missing parameters - rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune'] - response = http_session.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice}, - verify=ca_cert) + rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])["rune"] + response = http_session.post( + base_url + "/v1/invoice", headers={"Rune": rune_invoice}, verify=ca_cert + ) assert response.status_code == 500 - assert response.json()['code'] == -32602 + assert response.json()["code"] == -32602 # /v1/invoice with a correct rune but wrong parameters - rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune'] - response = http_session.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice}, - data={'amount_msat': '', - 'label': 'label', - 'description': 'description'}, - verify=ca_cert) + rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])["rune"] + response = http_session.post( + base_url + "/v1/invoice", + headers={"Rune": rune_invoice}, + data={"amount_msat": "", "label": "label", "description": "description"}, + verify=ca_cert, + ) assert response.status_code == 500 - assert response.json()['code'] == -32602 + assert response.json()["code"] == -32602 # l2 pays l1's invoice where the invoice is created with /v1/invoice - rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune'] - response = http_session.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice}, - data={'amount_msat': '50000000', - 'label': 'label', - 'description': 'description'}, - verify=ca_cert) + rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])["rune"] + response = http_session.post( + base_url + "/v1/invoice", + headers={"Rune": rune_invoice}, + data={ + "amount_msat": "50000000", + "label": "label", + "description": "description", + }, + verify=ca_cert, + ) assert response.status_code == 201 - assert 'bolt11' in response.json() + assert "bolt11" in response.json() def test_clnrest_large_response(node_factory): @@ -236,22 +278,31 @@ def test_clnrest_large_response(node_factory): for i in range(NUM_INVOICES): l1.rpc.invoice(amount_msat=100, label=str(i), description="inv") - rune = l1.rpc.createrune()['rune'] - response = http_session.post(base_url + '/v1/listinvoices', headers={'Rune': rune}, - verify=ca_cert) + rune = l1.rpc.createrune()["rune"] + response = http_session.post( + base_url + "/v1/listinvoices", headers={"Rune": rune}, verify=ca_cert + ) # No, this doesn't return JSON, it *parses* it into a Python object! resp = response.json() # Make sure it hasn't turned msat fields into strings! - assert not isinstance(resp['invoices'][0]['amount_msat'], Millisatoshi) - assert len(resp['invoices']) == NUM_INVOICES + assert not isinstance(resp["invoices"][0]["amount_msat"], Millisatoshi) + assert len(resp["invoices"]) == NUM_INVOICES # Tests for websocket are written separately to avoid flake8 # to complain with the errors F811 like this "F811 redefinition of # unused 'message'". -def notifications_received_via_websocket(l1, base_url, http_session, rpc_method='invoice', rpc_params=[100000, 'label', 'description'], expect_error=None): + +def notifications_received_via_websocket( + l1, + base_url, + http_session, + rpc_method="invoice", + rpc_params=[100000, "label", "description"], + expect_error=None, +): """Return the list of notifications received by the websocket client. We try to connect to the websocket server running at `base_url` @@ -269,6 +320,7 @@ def notifications_received_via_websocket(l1, base_url, http_session, rpc_method= @sio.event def message(data): notifications.append(data) + try: sio.connect(base_url) except socketio.exceptions.ConnectionError as e: @@ -299,7 +351,9 @@ def test_clnrest_websocket_no_rune(node_factory): http_session.verify = ca_cert.as_posix() # no rune provided => no websocket connection and no notification received - notifications = notifications_received_via_websocket(l1, base_url, http_session, expect_error="403") + notifications = notifications_received_via_websocket( + l1, base_url, http_session, expect_error="403" + ) assert len(notifications) == 0 @@ -313,11 +367,17 @@ def test_clnrest_websocket_wrong_rune(node_factory): http_session.verify = ca_cert.as_posix() # wrong rune provided => no websocket connection and no notification received - http_session.headers.update({"rune": "jMHrjVJb5l9-mjEd7zwux7Ookra1fgZ8wa9D8QbVT-w9MA=="}) + http_session.headers.update({ + "rune": "jMHrjVJb5l9-mjEd7zwux7Ookra1fgZ8wa9D8QbVT-w9MA==" + }) - notifications = notifications_received_via_websocket(l1, base_url, http_session, expect_error="401") + notifications = notifications_received_via_websocket( + l1, base_url, http_session, expect_error="401" + ) l1.daemon.logsearch_start = 0 - assert l1.daemon.is_in_log(r"Error code 1501: Not authorized: Not derived from master") + assert l1.daemon.is_in_log( + r"Error code 1501: Not authorized: Not derived from master" + ) assert len(notifications) == 0 @@ -331,10 +391,10 @@ def test_clnrest_websocket_unrestricted_rune(node_factory): http_session.verify = ca_cert.as_posix() # unrestricted rune provided => websocket connection and notifications received - rune_unrestricted = l1.rpc.createrune()['rune'] + rune_unrestricted = l1.rpc.createrune()["rune"] http_session.headers.update({"rune": rune_unrestricted}) notifications = notifications_received_via_websocket(l1, base_url, http_session) - assert len([n for n in notifications if not n.get('invoice_creation') is None]) == 1 + assert len([n for n in notifications if not n.get("invoice_creation") is None]) == 1 def test_clnrest_websocket_rune_readonly(node_factory): @@ -347,10 +407,10 @@ def test_clnrest_websocket_rune_readonly(node_factory): http_session.verify = ca_cert.as_posix() # readonly rune provided => websocket connection and notifications received - rune_readonly = l1.rpc.createrune(restrictions="readonly")['rune'] + rune_readonly = l1.rpc.createrune(restrictions="readonly")["rune"] http_session.headers.update({"rune": rune_readonly}) notifications = notifications_received_via_websocket(l1, base_url, http_session) - assert len([n for n in notifications if not n.get('invoice_creation') is None]) == 1 + assert len([n for n in notifications if not n.get("invoice_creation") is None]) == 1 def test_clnrest_websocket_rune_listnotifications(node_factory): @@ -363,10 +423,12 @@ def test_clnrest_websocket_rune_listnotifications(node_factory): http_session.verify = ca_cert.as_posix() # rune authorizing listclnrest-notifications method provided => websocket connection and notifications received - rune_clnrest_notifications = l1.rpc.createrune(restrictions=[["method=listclnrest-notifications"]])['rune'] + rune_clnrest_notifications = l1.rpc.createrune( + restrictions=[["method=listclnrest-notifications"]] + )["rune"] http_session.headers.update({"rune": rune_clnrest_notifications}) notifications = notifications_received_via_websocket(l1, base_url, http_session) - assert len([n for n in notifications if not n.get('invoice_creation') is None]) == 1 + assert len([n for n in notifications if not n.get("invoice_creation") is None]) == 1 def test_clnrest_websocket_rune_no_listnotifications(node_factory): @@ -379,54 +441,76 @@ def test_clnrest_websocket_rune_no_listnotifications(node_factory): http_session.verify = ca_cert.as_posix() # with a rune which doesn't authorized listclnrest-notifications method => no websocket connection and no notification received - rune_no_clnrest_notifications = l1.rpc.createrune(restrictions=[["method/listclnrest-notifications"]])['rune'] + rune_no_clnrest_notifications = l1.rpc.createrune( + restrictions=[["method/listclnrest-notifications"]] + )["rune"] http_session.headers.update({"rune": rune_no_clnrest_notifications}) - notifications = notifications_received_via_websocket(l1, base_url, http_session, expect_error="401") - assert len([n for n in notifications if n.find('invoice_creation') > 0]) == 0 + notifications = notifications_received_via_websocket( + l1, base_url, http_session, expect_error="401" + ) + assert len([n for n in notifications if n.find("invoice_creation") > 0]) == 0 def test_clnrest_numeric_msat_notification(node_factory): """Test that msat fields are integers in notifications also.""" # start a node with clnrest rest_port = str(node_factory.get_unused_port()) - base_url = 'http://127.0.0.1:' + rest_port - l1, l2 = node_factory.get_nodes(2, opts=[{}, {'clnrest-port': rest_port, 'clnrest-protocol': 'http'}]) + base_url = "http://127.0.0.1:" + rest_port + l1, l2 = node_factory.get_nodes( + 2, opts=[{}, {"clnrest-port": rest_port, "clnrest-protocol": "http"}] + ) node_factory.join_nodes([l1, l2], wait_for_announce=True) http_session = http_session_with_retry() # create an invoice on l2 - inv = l2.rpc.invoice(5000000, 'test_invoice_payment_notification', 'test_invoice_payment_notification_description') + inv = l2.rpc.invoice( + 5000000, + "test_invoice_payment_notification", + "test_invoice_payment_notification_description", + ) # create rune authorizing listclnrest-notifications method - rune_clnrest_notifications = l2.rpc.createrune(restrictions=[["method=listclnrest-notifications"]])['rune'] + rune_clnrest_notifications = l2.rpc.createrune( + restrictions=[["method=listclnrest-notifications"]] + )["rune"] http_session.headers.update({"rune": rune_clnrest_notifications}) - notifications = notifications_received_via_websocket(l1, base_url, http_session, 'pay', [inv['bolt11']]) - filtered_notifications = [n for n in notifications if 'invoice_payment' in n] + notifications = notifications_received_via_websocket( + l1, base_url, http_session, "pay", [inv["bolt11"]] + ) + filtered_notifications = [n for n in notifications if "invoice_payment" in n] - assert isinstance(filtered_notifications[0]['invoice_payment']['msat'], int) - assert filtered_notifications[0]['invoice_payment']['msat'] == 5000000 + assert isinstance(filtered_notifications[0]["invoice_payment"]["msat"], int) + assert filtered_notifications[0]["invoice_payment"]["msat"] == 5000000 def test_clnrest_options(node_factory): """Test startup options `clnrest-host`, `clnrest-protocol` and `clnrest-certs`.""" # with invalid port rest_port = 1000 - l1 = node_factory.get_node(options={'clnrest-port': rest_port}) - assert l1.daemon.is_in_log(f'plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` {rest_port}, should be a valid available port between 1024 and 65535.') + l1 = node_factory.get_node(options={"clnrest-port": rest_port}) + assert l1.daemon.is_in_log( + f"plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` {rest_port}, should be a valid available port between 1024 and 65535." + ) # with invalid protocol rest_port = str(node_factory.get_unused_port()) - rest_protocol = 'htttps' - l1 = node_factory.get_node(options={'clnrest-port': rest_port, - 'clnrest-protocol': rest_protocol}) - assert l1.daemon.is_in_log(r'plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-protocol` can either be http or https.') + rest_protocol = "htttps" + l1 = node_factory.get_node( + options={"clnrest-port": rest_port, "clnrest-protocol": rest_protocol} + ) + assert l1.daemon.is_in_log( + r"plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-protocol` can either be http or https." + ) # with invalid host rest_port = str(node_factory.get_unused_port()) - rest_host = '127.0.0.12.15' - l1 = node_factory.get_node(options={'clnrest-port': rest_port, - 'clnrest-host': rest_host}) - assert l1.daemon.is_in_log(r'plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-host` should be a valid IP.') + rest_host = "127.0.0.12.15" + l1 = node_factory.get_node( + options={"clnrest-port": rest_port, "clnrest-host": rest_host} + ) + assert l1.daemon.is_in_log( + r"plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-host` should be a valid IP." + ) def test_clnrest_http_headers(node_factory): @@ -436,37 +520,50 @@ def test_clnrest_http_headers(node_factory): http_session = http_session_with_retry() # Default values for `clnrest-csp` and `clnrest-cors-origins` options - response = http_session.get(base_url + '/v1/list-methods', verify=ca_cert) - assert response.headers['Content-Security-Policy'] == "default-src 'self'; font-src 'self'; img-src 'self' data:; frame-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline';" - assert response.headers['Access-Control-Allow-Origin'] == '*' + response = http_session.get(base_url + "/v1/list-methods", verify=ca_cert) + assert ( + response.headers["Content-Security-Policy"] + == "default-src 'self'; font-src 'self'; img-src 'self' data:; frame-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline';" + ) + assert response.headers["Access-Control-Allow-Origin"] == "*" # This might happen really early! l1.daemon.logsearch_start = 0 - l1.daemon.wait_for_log(f'plugin-clnrest: REST server running at {base_url}') + l1.daemon.wait_for_log(f"plugin-clnrest: REST server running at {base_url}") # Custom values for `clnrest-csp` and `clnrest-cors-origins` options rest_port = str(node_factory.get_unused_port()) - rest_certs = node_factory.directory + '/clnrest-certs' - l2 = node_factory.get_node(options={ - 'clnrest-port': rest_port, - 'clnrest-certs': rest_certs, - 'clnrest-csp': "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';", - 'clnrest-cors-origins': 'https://localhost:5500, http://192.168.1.30:3030, http://192.168.1.10:1010' - }) - base_url = 'https://127.0.0.1:' + rest_port + rest_certs = node_factory.directory + "/clnrest-certs" + l2 = node_factory.get_node( + options={ + "clnrest-port": rest_port, + "clnrest-certs": rest_certs, + "clnrest-csp": "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';", + "clnrest-cors-origins": "https://localhost:5500, http://192.168.1.30:3030, http://192.168.1.10:1010", + } + ) + base_url = "https://127.0.0.1:" + rest_port # This might happen really early! l2.daemon.logsearch_start = 0 - l2.daemon.wait_for_log(f'plugin-clnrest: REST server running at {base_url}') - ca_cert = Path(rest_certs) / 'ca.pem' - - response = http_session.get(base_url + '/v1/list-methods', - headers={'Origin': 'http://192.168.1.30:3030'}, - verify=ca_cert) - assert response.headers['Content-Security-Policy'] == "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';" - assert response.headers['Access-Control-Allow-Origin'] == 'http://192.168.1.30:3030' - response = http_session.get(base_url + '/v1/list-methods', - headers={'Origin': 'http://192.168.1.10:1010'}, - verify=ca_cert) - assert response.headers['Access-Control-Allow-Origin'] == 'http://192.168.1.10:1010' + l2.daemon.wait_for_log(f"plugin-clnrest: REST server running at {base_url}") + ca_cert = Path(rest_certs) / "ca.pem" + + response = http_session.get( + base_url + "/v1/list-methods", + headers={"Origin": "http://192.168.1.30:3030"}, + verify=ca_cert, + ) + assert ( + response.headers["Content-Security-Policy"] + == "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';" + ) + assert response.headers["Access-Control-Allow-Origin"] == "http://192.168.1.30:3030" + response = http_session.get( + base_url + "/v1/list-methods", + headers={"Origin": "http://192.168.1.10:1010"}, + verify=ca_cert, + ) + assert response.headers["Access-Control-Allow-Origin"] == "http://192.168.1.10:1010" + def test_clnrest_manifest(node_factory): """ @@ -474,7 +571,7 @@ def test_clnrest_manifest(node_factory): Does not test if the data is correctly registered in the manifest. """ - l1, _, _ = start_node_with_clnrest(node_factory) + l1, base_url, ca_cert = start_node_with_clnrest(node_factory) plugin_path = os.path.join(os.path.dirname(__file__), "plugins/get_manifest.py") l1.rpc.plugin_start(plugin_path) manifest = l1.rpc.checkmymanifest("checkmymanifest") @@ -485,37 +582,97 @@ def test_clnrest_manifest(node_factory): "rune": True, } + rune = l1.rpc.createrune(restrictions=[["method=checkmymanifest"]])["rune"] + http_session = http_session_with_retry() + response = http_session.post( + base_url + "/v1/path/to/me", headers={"Rune": rune}, verify=ca_cert + ) + assert response.json()["clnrest"] == { + "path": "/path/to/me", + "method": "POST", + "content_type": "application/json", + "rune": True, + } + + rune = l1.rpc.createrune(restrictions=[["method=dyncheckmymanifestpost"]])["rune"] + response = http_session.post( + base_url + "/v1/user/5/me", headers={"Rune": rune}, verify=ca_cert + ) + assert response.json()["clnrest"] == { + "path": r"/user//me", + "method": "POST", + "content_type": "application/json", + "rune": True, + } + assert response.json()["dyn_id_post"] == "5" + + response = http_session.get(base_url + "/v1/stats/5/me", verify=ca_cert) + assert response.json()["clnrest"] == { + "path": r"/stats//me", + "method": "GET", + "content_type": "application/json", + "rune": False, + } + assert response.json()["dyn_id_get"] == "5" + + response = http_session.get(base_url + "/v1/stats/to/me", verify=ca_cert) + assert response.json()["clnrest"] == { + "path": "/stats/to/me", + "method": "GET", + "content_type": "application/json", + "rune": False, + } + assert "dyn_id_get" not in response.json() + def test_clnrest_old_params(node_factory): """Test that we handle the v23.08-style parameters""" rest_port = str(node_factory.get_unused_port()) - rest_host = '127.0.0.1' - base_url = f'https://{rest_host}:{rest_port}' - l1 = node_factory.get_node(options={'rest-port': rest_port, - 'rest-host': rest_host, - 'allow-deprecated-apis': True}) + rest_host = "127.0.0.1" + base_url = f"https://{rest_host}:{rest_port}" + l1 = node_factory.get_node( + options={ + "rest-port": rest_port, + "rest-host": rest_host, + "allow-deprecated-apis": True, + } + ) # This might happen really early! l1.daemon.logsearch_start = 0 - l1.daemon.wait_for_logs([r'UNUSUAL lightningd: Option rest-port=.* deprecated in v23\.11, renaming to clnrest-port', - r'UNUSUAL lightningd: Option rest-host=.* deprecated in v23\.11, renaming to clnrest-host']) - l1.daemon.wait_for_log(r'plugin-clnrest: REST server running at ' + base_url) + l1.daemon.wait_for_logs([ + r"UNUSUAL lightningd: Option rest-port=.* deprecated in v23\.11, renaming to clnrest-port", + r"UNUSUAL lightningd: Option rest-host=.* deprecated in v23\.11, renaming to clnrest-host", + ]) + l1.daemon.wait_for_log(r"plugin-clnrest: REST server running at " + base_url) # Now try one where a plugin (e.g. clightning-rest) registers the option. - plugin = os.path.join(os.path.dirname(__file__), 'plugins/clnrest-use-options.py') - l2 = node_factory.get_node(options={'rest-port': rest_port, - 'rest-host': rest_host, - 'plugin': plugin, - 'allow-deprecated-apis': True}) + plugin = os.path.join(os.path.dirname(__file__), "plugins/clnrest-use-options.py") + l2 = node_factory.get_node( + options={ + "rest-port": rest_port, + "rest-host": rest_host, + "plugin": plugin, + "allow-deprecated-apis": True, + } + ) l2.daemon.logsearch_start = 0 # We still rename this one, since it's for clnrest. - assert l2.daemon.is_in_log(r'UNUSUAL lightningd: Option rest-host=.* deprecated in v23\.11, renaming to clnrest-host') + assert l2.daemon.is_in_log( + r"UNUSUAL lightningd: Option rest-host=.* deprecated in v23\.11, renaming to clnrest-host" + ) # This one does not get renamed! - assert not l2.daemon.is_in_log(r'UNUSUAL lightningd: Option rest-port=.* deprecated in v23\.11, renaming to clnrest-host') - assert [p for p in l2.rpc.plugin('list')['plugins'] if p['name'].endswith('clnrest')] == [] - assert l2.daemon.is_in_log(r'plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` option is not configured') - assert l2.daemon.is_in_log(rf'clnrest-use-options.py: rest-port is {rest_port}') + assert not l2.daemon.is_in_log( + r"UNUSUAL lightningd: Option rest-port=.* deprecated in v23\.11, renaming to clnrest-host" + ) + assert [ + p for p in l2.rpc.plugin("list")["plugins"] if p["name"].endswith("clnrest") + ] == [] + assert l2.daemon.is_in_log( + r"plugin-clnrest: Killing plugin: disabled itself at init: `clnrest-port` option is not configured" + ) + assert l2.daemon.is_in_log(rf"clnrest-use-options.py: rest-port is {rest_port}") def test_clnrest_websocket_upgrade_header(node_factory): @@ -531,13 +688,16 @@ def test_clnrest_websocket_upgrade_header(node_factory): @sio.event def message(data): notifications.append(data) - with pytest.raises(socketio.exceptions.ConnectionError, match="Unexpected response from server"): + + with pytest.raises( + socketio.exceptions.ConnectionError, match="Unexpected response from server" + ): sio.connect(base_url) time.sleep(2) # trigger notification by calling method - rpc_method = 'invoice' - rpc_params = [100000, 'label', 'description'] + rpc_method = "invoice" + rpc_params = [100000, "label", "description"] rpc_call = getattr(l1.rpc, rpc_method) rpc_call(*rpc_params) time.sleep(2)