diff --git a/Cargo.lock b/Cargo.lock index a3dd8c7b3a..23f1af0638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -794,9 +794,9 @@ dependencies = [ [[package]] name = "fil_actor_bundler" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51452516b9dd524ec99d2b38992a0cfed4eda7e2b28fed696e17f05f68fdaf46" +checksum = "80029390c3879b41be9c38907dcfd4ae2638fab5e38d818c22e5f5360d07018b" dependencies = [ "anyhow", "async-std", @@ -1226,30 +1226,17 @@ dependencies = [ ] [[package]] -name = "fil_builtin_actors_bundle" +name = "fil_builtin_actors_builder" version = "16.0.0-rc3" dependencies = [ + "anyhow", "cid", "clap", - "fil_actor_account", "fil_actor_bundler", - "fil_actor_cron", - "fil_actor_datacap", - "fil_actor_eam", - "fil_actor_ethaccount", - "fil_actor_evm", - "fil_actor_init", - "fil_actor_market", - "fil_actor_miner", - "fil_actor_multisig", - "fil_actor_paych", - "fil_actor_placeholder", - "fil_actor_power", - "fil_actor_reward", - "fil_actor_system", - "fil_actor_verifreg", "fil_actors_runtime", "num-traits", + "serde", + "serde_json", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bd731bf3c3..ee725db867 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ edition = "2021" repository = "https://github.com/filecoin-project/builtin-actors" [package] -name = "fil_builtin_actors_bundle" -description = "Bundle of FVM-compatible Wasm bytecode for Filecoin builtin actors" +name = "fil_builtin_actors_builder" +description = "Builder of FVM-compatible Wasm bytecode for Filecoin builtin actors" version.workspace = true license.workspace = true edition.workspace = true @@ -26,33 +26,9 @@ authors = ["Protocol Labs", "Filecoin Core Devs"] keywords = ["filecoin", "web3", "wasm"] exclude = ["examples", ".github"] -# We don't publish the bundle to crates.io anymore. Instead, we build in CI. +# We don't publish the bundle or bundler to crates.io. Instead, we build in CI. publish = false -[target.'cfg(target_arch = "wasm32")'.dependencies] -fil_actor_account = { workspace = true, features = ["fil-actor"] } -fil_actor_cron = { workspace = true, features = ["fil-actor"] } -fil_actor_datacap = { workspace = true, features = ["fil-actor"] } -fil_actor_ethaccount = { workspace = true, features = ["fil-actor"] } -fil_actor_eam = { workspace = true, features = ["fil-actor"] } -fil_actor_evm = { workspace = true, features = ["fil-actor"] } -fil_actor_init = { workspace = true, features = ["fil-actor"] } -fil_actor_market = { workspace = true, features = ["fil-actor"] } -fil_actor_miner = { workspace = true, features = ["fil-actor"] } -fil_actor_multisig = { workspace = true, features = ["fil-actor"] } -fil_actor_paych = { workspace = true, features = ["fil-actor"] } -fil_actor_placeholder = { workspace = true, features = ["fil-actor"] } -fil_actor_power = { workspace = true, features = ["fil-actor"] } -fil_actor_reward = { workspace = true, features = ["fil-actor"] } -fil_actor_system = { workspace = true, features = ["fil-actor"] } -fil_actor_verifreg = { workspace = true, features = ["fil-actor"] } - -[build-dependencies] -fil_actor_bundler = "7.1.0" -cid = { workspace = true } -fil_actors_runtime = { workspace = true } -num-traits = { workspace = true } - [dependencies] clap = { version = "4.3.0", features = [ "derive", @@ -61,6 +37,13 @@ clap = { version = "4.3.0", features = [ "usage", "error-context", ], default-features = false } +serde = { workspace = true } +serde_json = { workspace = true } +fil_actor_bundler = "7.1.1" +fil_actors_runtime = { workspace = true } +num-traits = { workspace = true } +cid = { workspace = true } +anyhow.workspace = true [features] default = [] ## translates to mainnet diff --git a/build.rs b/build.rs deleted file mode 100644 index 7dadee3c6f..0000000000 --- a/build.rs +++ /dev/null @@ -1,187 +0,0 @@ -use fil_actor_bundler::Bundler; -use fil_actors_runtime::runtime::builtins::Type; -use num_traits::cast::FromPrimitive; -use std::error::Error; -use std::io::{BufRead, BufReader}; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; - -/// Cargo package for an actor. -type Package = str; - -/// Technical identifier for the actor in legacy CodeCIDs and else. -type ID = str; - -const ACTORS: &[(&Package, &ID)] = &[ - ("system", "system"), - ("init", "init"), - ("cron", "cron"), - ("account", "account"), - ("power", "storagepower"), - ("miner", "storageminer"), - ("market", "storagemarket"), - ("paych", "paymentchannel"), - ("multisig", "multisig"), - ("reward", "reward"), - ("verifreg", "verifiedregistry"), - ("datacap", "datacap"), - ("placeholder", "placeholder"), - ("evm", "evm"), - ("eam", "eam"), - ("ethaccount", "ethaccount"), -]; - -const NETWORK_ENV: &str = "BUILD_FIL_NETWORK"; - -/// Returns the configured network name, checking both the environment and feature flags. -fn network_name() -> String { - let env_network = std::env::var_os(NETWORK_ENV); - - let feat_network = if cfg!(feature = "mainnet") { - Some("mainnet") - } else if cfg!(feature = "caterpillarnet") { - Some("caterpillarnet") - } else if cfg!(feature = "butterflynet") { - Some("butterflynet") - } else if cfg!(feature = "calibrationnet") { - Some("calibrationnet") - } else if cfg!(feature = "devnet") { - Some("devnet") - } else if cfg!(feature = "testing") { - Some("testing") - } else if cfg!(feature = "testing-fake-proofs") { - Some("testing-fake-proofs") - } else { - None - }; - - // Make sure they match if they're both set. Otherwise, pick the one - // that's set, or fallback on "mainnet". - match (feat_network, &env_network) { - (Some(from_feature), Some(from_env)) => { - assert_eq!(from_feature, from_env, "different target network configured via the features than via the {} environment variable", NETWORK_ENV); - from_feature - } - (Some(net), None) => net, - (None, Some(net)) => net.to_str().expect("network name not utf8"), - (None, None) => "mainnet", - }.to_owned() -} - -fn main() -> Result<(), Box> { - // Cargo executable location. - let cargo = std::env::var_os("CARGO").expect("no CARGO env var"); - println!("cargo:warning=cargo: {:?}", &cargo); - - let out_dir = std::env::var_os("OUT_DIR") - .as_ref() - .map(Path::new) - .map(|p| p.join("bundle")) - .expect("no OUT_DIR env var"); - println!("cargo:warning=out_dir: {:?}", &out_dir); - - // Compute the package names. - let packages = - ACTORS.iter().map(|(pkg, _)| String::from("fil_actor_") + pkg).collect::>(); - - let manifest_path = - Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR unset")) - .join("Cargo.toml"); - println!("cargo:warning=manifest_path={:?}", &manifest_path); - - // Determine the network name. - let network_name = network_name(); - println!("cargo:warning=network name: {}", network_name); - - // Make sure we re-build if the network name changes. - println!("cargo:rerun-if-env-changed={}", NETWORK_ENV); - - // Rerun if the source, dependencies, build options, build script _or_ actors have changed. We - // need to check if the actors have changed because otherwise, when building in a workspace, we - // won't re-run the build script and therefore won't re-compile them. - // - // This _isn't_ an issue when building as a dependency fetched from crates.io (because the crate - // is immutable). - for file in ["actors", "Cargo.toml", "Cargo.lock", "src", "build.rs"] { - println!("cargo:rerun-if-changed={}", file); - } - - // Cargo build command for all actors at once. - let mut cmd = Command::new(&cargo); - cmd.arg("build") - .args(packages.iter().map(|pkg| "-p=".to_owned() + pkg)) - .arg("--target=wasm32-unknown-unknown") - .arg("--profile=wasm") - .arg("--locked") - .arg("--features=fil-actor") - .arg("--manifest-path=".to_owned() + manifest_path.to_str().unwrap()) - .env(NETWORK_ENV, network_name) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - // We are supposed to only generate artifacts under OUT_DIR, - // so set OUT_DIR as the target directory for this build. - .env("CARGO_TARGET_DIR", &out_dir) - // As we are being called inside a build-script, this env variable is set. However, we set - // our own `RUSTFLAGS` and thus, we need to remove this. Otherwise cargo favors this - // env variable. - .env_remove("CARGO_ENCODED_RUSTFLAGS"); - - // Print out the command line we're about to run. - println!("cargo:warning=cmd={:?}", &cmd); - - // Launch the command. - let mut child = cmd.spawn().expect("failed to launch cargo build"); - - // Pipe the output as cargo warnings. Unfortunately this is the only way to - // get cargo build to print the output. - let stdout = child.stdout.take().expect("no stdout"); - let stderr = child.stderr.take().expect("no stderr"); - let j1 = thread::spawn(move || { - for line in BufReader::new(stderr).lines() { - println!("cargo:warning={:?}", line.unwrap()); - } - }); - let j2 = thread::spawn(move || { - for line in BufReader::new(stdout).lines() { - println!("cargo:warning={:?}", line.unwrap()); - } - }); - - j1.join().unwrap(); - j2.join().unwrap(); - - let result = child.wait().expect("failed to wait for build to finish"); - if !result.success() { - return Err("actor build failed".into()); - } - - let dst = Path::new(&out_dir).join("bundle.car"); - let mut bundler = Bundler::new(&dst); - for (&(pkg, name), id) in ACTORS.iter().zip(1u32..) { - assert_eq!( - name, - Type::from_u32(id).expect("type not defined").name(), - "actor types don't match actors included in the bundle" - ); - let bytecode_path = Path::new(&out_dir) - .join("wasm32-unknown-unknown/wasm") - .join(format!("fil_actor_{}.wasm", pkg)); - - // This actor version doesn't force synthetic CIDs; it uses genuine - // content-addressed CIDs. - let forced_cid = None; - - let cid = bundler - .add_from_file(id, name.to_owned(), forced_cid, &bytecode_path) - .unwrap_or_else(|err| { - panic!("failed to add file {:?} to bundle for actor {}: {}", bytecode_path, id, err) - }); - println!("cargo:warning=added {} ({}) to bundle with CID {}", name, id, cid); - } - bundler.finish().expect("failed to finish bundle"); - - println!("cargo:warning=bundle={}", dst.display()); - - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 1d5cc93545..0000000000 --- a/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -/// The bundled CAR embedded as a byte slice for easy consumption by Rust programs. -/// -/// The root CID of the CAR points to an actor index data structure. It is a -/// CBOR-encoded IPLD Map, enumerating actor name and their -/// respective CIDs. -/// -/// The actor names are values from this enumeration: -/// -/// - "account" -/// - "cron" -/// - "init" -/// - "market" -/// - "miner" -/// - "multisig" -/// - "paych" -/// - "power" -/// - "reward" -/// - "system" -/// - "verifreg" -/// - "evm" -/// - "eam" -/// - "ethaccount" -/// - "placeholder" -/// -/// The Filecoin client must import the contents of CAR into the blockstore, but -/// may opt to exclude the index data structure. -pub const BUNDLE_CAR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/bundle/bundle.car")); diff --git a/src/main.rs b/src/main.rs index 8e3538aeda..ac4b3c6a2c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,22 +1,151 @@ +use anyhow::{anyhow, Context}; use clap::Parser; -use std::io::Write; +use fil_actor_bundler::Bundler; +use fil_actors_runtime::runtime::builtins::Type; +use num_traits::cast::FromPrimitive; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; -use fil_builtin_actors_bundle::BUNDLE_CAR; +const ACTORS: &[(&str, &str)] = &[ + ("system", "system"), + ("init", "init"), + ("cron", "cron"), + ("account", "account"), + ("power", "storagepower"), + ("miner", "storageminer"), + ("market", "storagemarket"), + ("paych", "paymentchannel"), + ("multisig", "multisig"), + ("reward", "reward"), + ("verifreg", "verifiedregistry"), + ("datacap", "datacap"), + ("placeholder", "placeholder"), + ("evm", "evm"), + ("eam", "eam"), + ("ethaccount", "ethaccount"), +]; + +const NETWORK_ENV: &str = "BUILD_FIL_NETWORK"; #[derive(Parser)] #[clap(name = env!("CARGO_PKG_NAME"))] #[clap(version = env!("CARGO_PKG_VERSION"))] -#[clap(about = "Writes a CAR file containing Wasm bytecode for Filecoin actors.", long_about = None)] +#[clap(about = "Builds and writes a CAR file containing Wasm bytecode for Filecoin actors.", long_about = None)] struct Cli { - /// The output car path. Defaults to STDOUT. - #[clap(short, long, required = false)] - output: Option, + /// The output car path + #[clap(short, long, required = true)] + output: PathBuf, + + /// Network name to build for: mainnet, calibrationnet, etc. + #[clap(short, long, default_value = "mainnet")] + network: String, } -fn main() -> Result<(), std::io::Error> { - let cli = Cli::parse(); - match cli.output { - Some(path) => std::fs::write(path, BUNDLE_CAR), - None => std::io::stdout().write_all(BUNDLE_CAR), +#[derive(Debug, Deserialize, Serialize)] +struct CargoMessage { + reason: String, + target: Option, + #[serde(default)] + filenames: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct CargoTarget { + name: String, +} + +fn build_bundle(network_name: &str, output_path: &Path) -> anyhow::Result<()> { + // Cargo executable location + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + println!("Using cargo: {:?}", &cargo); + + // Compute the package names + let packages = + ACTORS.iter().map(|(pkg, _)| format!("fil_actor_{pkg}")).collect::>(); + + println!("Building for network: {}", network_name); + + // Cargo build command for all actors at once + let mut cmd = Command::new(&cargo); + cmd.arg("build") + .args(packages.iter().map(|pkg| "-p=".to_owned() + pkg)) + .arg("--target=wasm32-unknown-unknown") + .arg("--profile=wasm") + .arg("--locked") + .arg("--features=fil-actor") + .arg("--message-format=json-render-diagnostics") + .env(NETWORK_ENV, network_name) + .stdout(Stdio::piped()) // json output. + .stderr(Stdio::inherit()); // status messages + + println!("Running: {:?}", &cmd); + + // Run the build command. + let result = cmd.output().context("failed to build the actors")?; + if !result.status.success() { + return Err(anyhow!("actor build failed")); + } + + // Collect the actor bytecode. + let mut actor_bytecode: HashMap<&str, Option> = + packages.iter().map(|n| (&**n, None)).collect(); + let messages = serde_json::Deserializer::from_slice(&result.stdout).into_iter::(); + for m in messages { + let m = m.context("invalid cargo message format")?; + if m.reason != "compiler-artifact" { + continue; + } + + let Some(pkg_name) = m.target.map(|t| t.name) else { continue }; + let Some(fname) = m.filenames.iter().find(|f| f.ends_with(".wasm")) else { continue }; + let Some(entry) = actor_bytecode.get_mut(&*pkg_name) else { continue }; + + if let Some(existing) = entry { + return Err(anyhow!( + "duplicate artifact for {}: {} and {}", + pkg_name, + fname, + existing.display(), + )); + } + *entry = Some(fname.into()); + } + + let mut bundler = Bundler::new(output_path); + for (&(pkg, name), id) in ACTORS.iter().zip(1u32..) { + assert_eq!( + name, + Type::from_u32(id).expect("Type not defined").name(), + "Actor types don't match actors included in the bundle" + ); + let bytecode_path = actor_bytecode + .get(&*format!("fil_actor_{pkg}")) + .unwrap() // we always have an entry in the map. + .as_ref() + .with_context(|| format!("failed to build {pkg}"))?; + + // This actor version doesn't force synthetic CIDs; it uses genuine + // content-addressed CIDs. + let forced_cid = None; + + let cid = bundler + .add_from_file(id, name.to_owned(), forced_cid, bytecode_path) + .unwrap_or_else(|err| { + panic!("Failed to add file {:?} to bundle for actor {}: {}", bytecode_path, id, err) + }); + println!("Added {} ({}) to bundle with CID {}", name, id, cid); } + bundler.finish().expect("Failed to finish bundle"); + + println!("Bundle created at: {}", output_path.display()); + + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + + build_bundle(&cli.network, &cli.output) }