WIT-based environment syntax

itowlson · itowlson · commit c95eadfe284a · 2024-09-24T12:18:42.000+12:00
Signed-off-by: itowlson &lt;ivan.towlson@fermyon.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml
@@ -7,7 +7,10 @@ edition = { workspace = true }
 [dependencies]
 anyhow = { workspace = true }
 async-trait = "0.1"
+bytes = "1.1"
 futures = "0.3"
+futures-util = "0.3"
+id-arena = "2"
 indexmap = "2.2.6"
 miette = "7.2.0"
 oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" }
@@ -26,6 +29,8 @@ wac-parser = "0.6.0"
 wac-resolver = "0.6.0"
 wac-types = "0.6.0"
 wasm-pkg-loader = "0.4.1"
+wit-component = "0.217.0"
+wit-parser = "0.217.0"
 
 [lints]
 workspace = true
diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs
@@ -1,37 +1,112 @@
-use wasm_pkg_loader::PackageRef;
+use anyhow::Context;
 
-#[derive(Debug, serde::Deserialize)]
-pub struct TargetEnvironment {
-    pub name: String,
-    pub environments: std::collections::HashMap<TriggerType, TargetWorld>,
-}
+pub async fn load_environment(env_id: &str) -> anyhow::Result<TargetEnvironment> {
+    use futures_util::TryStreamExt;
+
+    let (pkg_name, pkg_ver) = env_id.split_once('@').unwrap();
+
+    let mut client = wasm_pkg_loader::Client::with_global_defaults()?;
+
+    let package = pkg_name.to_owned().try_into().context("pkg ref parse")?;
+    let version = wasm_pkg_loader::Version::parse(pkg_ver).context("pkg ver parse")?;
+
+    let release = client
+        .get_release(&package, &version)
+        .await
+        .context("get release")?;
+    let stm = client
+        .stream_content(&package, &release)
+        .await
+        .context("stream content")?;
+    let bytes = stm
+        .try_collect::<bytes::BytesMut>()
+        .await
+        .context("collect stm")?
+        .to_vec();
 
-#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)]
-pub struct TargetWorld {
-    wit_package: PackageRef,
-    package_ver: String, // TODO: tidy to semver::Version
-    world_name: WorldNames,
+    TargetEnvironment::new(env_id.to_owned(), bytes)
 }
 
-#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)]
-#[serde(untagged)]
-enum WorldNames {
-    Exactly(String),
-    AnyOf(Vec<String>),
+pub struct TargetEnvironment {
+    name: String,
+    decoded: wit_parser::decoding::DecodedWasm,
+    package: wit_parser::Package, // saves unwrapping it every time
+    package_id: id_arena::Id<wit_parser::Package>,
+    package_bytes: Vec<u8>,
 }
 
-impl TargetWorld {
-    fn versioned_name(&self, world_name: &str) -> String {
-        format!("{}/{}@{}", self.wit_package, world_name, self.package_ver)
+impl TargetEnvironment {
+    fn new(name: String, bytes: Vec<u8>) -> anyhow::Result<Self> {
+        let decoded = wit_component::decode(&bytes).context("decode wasm")?;
+        let package_id = decoded.package();
+        let package = decoded
+            .resolve()
+            .packages
+            .get(package_id)
+            .context("should had a package")?
+            .clone();
+
+        Ok(Self {
+            name,
+            decoded,
+            package,
+            package_id,
+            package_bytes: bytes,
+        })
+    }
+
+    pub fn is_world_for(&self, trigger_type: &TriggerType, world: &wit_parser::World) -> bool {
+        world.name.starts_with(&format!("trigger-{trigger_type}"))
+            && world.package.is_some_and(|p| p == self.package_id)
+    }
+
+    pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool {
+        self.decoded
+            .resolve()
+            .worlds
+            .iter()
+            .any(|(_, world)| self.is_world_for(trigger_type, world))
+    }
+
+    pub fn worlds(&self, trigger_type: &TriggerType) -> Vec<String> {
+        self.decoded
+            .resolve()
+            .worlds
+            .iter()
+            .filter(|(_, world)| self.is_world_for(trigger_type, world))
+            .map(|(_, world)| self.world_qname(world))
+            .collect()
+    }
+
+    /// Fully qualified world name (e.g. fermyon:spin/http-trigger@2.0.0)
+    fn world_qname(&self, world: &wit_parser::World) -> String {
+        let version_suffix = match self.package_version() {
+            Some(version) => format!("@{version}"),
+            None => "".to_owned(),
+        };
+        format!(
+            "{}/{}{version_suffix}",
+            self.package_namespaced_name(),
+            world.name,
+        )
+    }
+
+    /// The environment name for UI purposes
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    /// Namespaced but unversioned package name (e.g. spin:cli)
+    pub fn package_namespaced_name(&self) -> String {
+        format!("{}:{}", self.package.name.namespace, self.package.name.name)
+    }
+
+    pub fn package_version(&self) -> Option<&semver::Version> {
+        self.package.name.version.as_ref()
     }
 
-    pub fn versioned_names(&self) -> Vec<String> {
-        match &self.world_name {
-            WorldNames::Exactly(name) => vec![self.versioned_name(name)],
-            WorldNames::AnyOf(names) => {
-                names.iter().map(|name| self.versioned_name(name)).collect()
-            }
-        }
+    pub fn package_bytes(&self) -> &[u8] {
+        &self.package_bytes
     }
 }
 
diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs
@@ -1,9 +1,9 @@
-use anyhow::{anyhow, Context};
+use anyhow::anyhow;
 
 mod environment_definition;
 mod loader;
 
-use environment_definition::{TargetEnvironment, TargetWorld, TriggerType};
+use environment_definition::{load_environment, TargetEnvironment, TriggerType};
 pub use loader::ResolutionContext;
 use loader::{load_and_resolve_all, ComponentToValidate};
 
@@ -12,57 +12,22 @@ pub async fn validate_application_against_environment_ids(
     app: &spin_manifest::schema::v2::AppManifest,
     resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
-    let envs = join_all_result(env_ids.map(resolve_environment_id)).await?;
+    let envs = join_all_result(env_ids.map(load_environment)).await?;
     validate_application_against_environments(&envs, app, resolution_context).await
 }
 
-async fn resolve_environment_id(id: &str) -> anyhow::Result<TargetEnvironment> {
-    let (name, ver) = id.split_once('@').ok_or(anyhow!(
-        "Target environment '{id}' does not specify a version"
-    ))?;
-    let client = oci_distribution::Client::default();
-    let auth = oci_distribution::secrets::RegistryAuth::Anonymous;
-    let env_def_ref =
-        oci_distribution::Reference::try_from(format!("ghcr.io/itowlson/spinenvs/{name}:{ver}"))?;
-    let (man, _digest) = client
-        .pull_manifest(&env_def_ref, &auth)
-        .await
-        .with_context(|| format!("Failed to find environment '{id}' in registry"))?;
-    let im = match man {
-        oci_distribution::manifest::OciManifest::Image(im) => im,
-        oci_distribution::manifest::OciManifest::ImageIndex(_ind) => {
-            anyhow::bail!("Environment '{id}' definition is unusable - stored in registry in incorrect format")
-        }
-    };
-    let the_layer = &im.layers[0];
-    let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default());
-    client
-        .pull_blob(&env_def_ref, the_layer, &mut out)
-        .await
-        .with_context(|| {
-            format!("Failed to download environment '{id}' definition from registry")
-        })?;
-    let te = serde_json::from_slice(&out).with_context(|| {
-        format!("Failed to load environment '{id}' definition - invalid JSON schema")
-    })?;
-    Ok(te)
-}
-
-pub async fn validate_application_against_environments(
+async fn validate_application_against_environments(
     envs: &[TargetEnvironment],
     app: &spin_manifest::schema::v2::AppManifest,
     resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
     use futures::FutureExt;
 
     for trigger_type in app.triggers.keys() {
-        if let Some(env) = envs
-            .iter()
-            .find(|e| !e.environments.contains_key(trigger_type))
-        {
+        if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) {
             anyhow::bail!(
                 "Environment {} does not support trigger type {trigger_type}",
-                env.name
+                env.name()
             );
         }
     }
@@ -87,28 +52,9 @@ async fn validate_component_against_environments(
     trigger_type: &TriggerType,
     component: &ComponentToValidate<'_>,
 ) -> anyhow::Result<()> {
-    let worlds = envs
-        .iter()
-        .map(|e| {
-            e.environments
-                .get(trigger_type)
-                .ok_or(anyhow!(
-                    "Environment '{}' doesn't support trigger type {trigger_type}",
-                    e.name
-                ))
-                .map(|w| (e.name.as_str(), w))
-        })
-        .collect::<Result<std::collections::HashSet<_>, _>>()?;
-    validate_component_against_worlds(worlds.into_iter(), component).await?;
-    Ok(())
-}
-
-async fn validate_component_against_worlds(
-    target_worlds: impl Iterator<Item = (&str, &TargetWorld)>,
-    component: &ComponentToValidate<'_>,
-) -> anyhow::Result<()> {
-    for (env_name, target_world) in target_worlds {
-        validate_wasm_against_any_world(env_name, target_world, component).await?;
+    for env in envs {
+        let worlds = env.worlds(trigger_type);
+        validate_wasm_against_any_world(env, &worlds, component).await?;
     }
 
     tracing::info!(
@@ -120,21 +66,21 @@ async fn validate_component_against_worlds(
 }
 
 async fn validate_wasm_against_any_world(
-    env_name: &str,
-    target_world: &TargetWorld,
+    env: &TargetEnvironment,
+    world_names: &[String],
     component: &ComponentToValidate<'_>,
 ) -> anyhow::Result<()> {
     let mut result = Ok(());
-    for target_str in target_world.versioned_names() {
-        tracing::info!(
-            "Trying component {} {} against target world {target_str}",
+    for target_world in world_names {
+        tracing::debug!(
+            "Trying component {} {} against target world {target_world}",
             component.id(),
             component.source_description(),
         );
-        match validate_wasm_against_world(env_name, &target_str, component).await {
+        match validate_wasm_against_world(env, target_world, component).await {
             Ok(()) => {
                 tracing::info!(
-                    "Validated component {} {} against target world {target_str}",
+                    "Validated component {} {} against target world {target_world}",
                     component.id(),
                     component.source_description(),
                 );
@@ -143,7 +89,7 @@ async fn validate_wasm_against_any_world(
             Err(e) => {
                 // Record the error, but continue in case a different world succeeds
                 tracing::info!(
-                    "Rejecting component {} {} for target world {target_str} because {e:?}",
+                    "Rejecting component {} {} for target world {target_world} because {e:?}",
                     component.id(),
                     component.source_description(),
                 );
@@ -155,34 +101,52 @@ async fn validate_wasm_against_any_world(
 }
 
 async fn validate_wasm_against_world(
-    env_name: &str,
-    target_str: &str,
+    env: &TargetEnvironment,
+    target_world: &str,
     component: &ComponentToValidate<'_>,
 ) -> anyhow::Result<()> {
-    let comp_name = "root:component";
+    // Because we are abusing a composition tool to do validation, we have to
+    // provide a name by which to refer to the component in the dummy composition.
+    let component_name = "root:component";
+    let component_key = wac_types::BorrowedPackageKey::from_name_and_version(component_name, None);
+
+    // wac is going to get the world from the environment package bytes.
+    // This constructs a key for that mapping.
+    let env_pkg_name = env.package_namespaced_name();
+    let env_pkg_key =
+        wac_types::BorrowedPackageKey::from_name_and_version(&env_pkg_name, env.package_version());
+
+    let env_name = env.name();
 
     let wac_text = format!(
         r#"
-    package validate:component@1.0.0 targets {target_str};
-    let c = new {comp_name} {{ ... }};
+    package validate:component@1.0.0 targets {target_world};
+    let c = new {component_name} {{ ... }};
     export c...;
     "#
     );
 
     let doc = wac_parser::Document::parse(&wac_text)?;
 
-    let compkey = wac_types::BorrowedPackageKey::from_name_and_version(comp_name, None);
+    // TODO: if we end up needing the registry, we need to do this dance
+    // for things we are providing separately, or the registry will try to
+    // hoover them up and will fail.
+    // let mut refpkgs = wac_resolver::packages(&doc)?;
+    // refpkgs.shift_remove(&env_pkg_key);
+    // refpkgs.shift_remove(&component_key);
 
-    let mut refpkgs = wac_resolver::packages(&doc)?;
-    refpkgs.retain(|k, _| k != &compkey);
+    // TODO: determine if this is needed in circumstances other than the simple test
+    // let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?;
+    // let mut packages = reg_resolver
+    //     .resolve(&refpkgs)
+    //     .await
+    //     .context("reg_resolver.resolve failed")?;
 
-    let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?;
-    let mut packages = reg_resolver
-        .resolve(&refpkgs)
-        .await
-        .context("reg_resolver.resolve failed")?;
+    let mut packages: indexmap::IndexMap<wac_types::BorrowedPackageKey, Vec<u8>> =
+        Default::default();
 
-    packages.insert(compkey, component.wasm_bytes().to_vec());
+    packages.insert(env_pkg_key, env.package_bytes().to_vec());
+    packages.insert(component_key, component.wasm_bytes().to_vec());
 
     match doc.resolve(packages) {
         Ok(_) => Ok(()),
@@ -195,7 +159,7 @@ async fn validate_wasm_against_world(
         }
         Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => {
             // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name
-            Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_str} requires an export named {export}, which the component does not provide", component.id(), component.source_description()))
+            Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_world} requires an export named {export}, which the component does not provide", component.id(), component.source_description()))
         }
         Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => {
             Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description()))
