Initial Implementation of OCI push and pull commands (#20)

* implemented pull and push oci commands

* renamed http_server short flag due to conflict

* moved username, password and insecure to top level reigstry struct

* point oci-distribution to git repo

Author: brooksmtownsend

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wash-cli"
-version = "0.1.7"
+version = "0.1.8"
 authors = ["Brooks Townsend <[email protected]>"]
 edition = "2018"
 repository = "https://github.com/wascc/wash"
@@ -19,12 +19,15 @@ env_logger = "0.7.1"
 serde_json = "1.0.59"
 serde = { version = "1.0.117", features = ["derive"] }
 crossbeam = "0.7.3"
+tokio = { version = "0.2.0" }
+spinners = "1.2.0"
 
 nkeys = "0.0.11"
 latticeclient = "0.4.0"
 wascap = "0.5.1"
 term-table = "1.3.0"
 provider-archive = "0.3.0"
+oci-distribution = { git = "https://github.com/brooksmtownsend/krustlet", branch = "oci-push" }
 
 [[bin]]
 name = "wash"

src/claims.rs

@@ -238,7 +238,7 @@ struct ActorMetadata {
     #[structopt(short = "g", long = "msg")]
     msg_broker: bool,
     /// Enable the HTTP server standard capability
-    #[structopt(short = "s", long = "http_server")]
+    #[structopt(short = "q", long = "http_server")]
     http_server: bool,
     /// Enable the HTTP client standard capability
     #[structopt(short = "h", long = "http_client")]

src/main.rs

@@ -9,6 +9,8 @@ mod keys;
 use keys::KeysCli;
 mod par;
 use par::ParCli;
+mod reg;
+use reg::RegCli;
 
 /// This renders appropriately with escape characters
 const ASCII: &str = "
@@ -44,6 +46,9 @@ enum CliCommand {
     /// Utilities for creating, inspecting, and modifying capability provider archive files
     #[structopt(name = "par")]
     Par(ParCli),
+    /// Utilities for interacting with OCI compliant registries
+    #[structopt(name = "reg")]
+    Reg(RegCli),
 }
 
 fn main() {
@@ -55,6 +60,7 @@ fn main() {
         CliCommand::Lattice(latticecli) => lattice::handle_command(latticecli),
         CliCommand::Claims(claimscli) => claims::handle_command(claimscli),
         CliCommand::Par(parcli) => par::handle_command(parcli),
+        CliCommand::Reg(regcli) => reg::handle_command(regcli),
     };
 
     match res {

src/reg.rs

@@ -0,0 +1,336 @@
+extern crate oci_distribution;
+use oci_distribution::client::*;
+use oci_distribution::secrets::RegistryAuth;
+use oci_distribution::Reference;
+use provider_archive::ProviderArchive;
+use spinners::{Spinner, Spinners};
+use std::fs::File;
+use std::io::prelude::*;
+use structopt::clap::AppSettings;
+use structopt::StructOpt;
+use tokio::runtime::*;
+
+const PROVIDER_ARCHIVE_MEDIA_TYPE: &str = "application/vnd.wascc.provider.archive.layer.v1+par";
+const PROVIDER_ARCHIVE_CONFIG_MEDIA_TYPE: &str = "application/vnd.wascc.provider.archive.config";
+const PROVIDER_ARCHIVE_FILE_EXTENSION: &str = ".par.gz";
+const WASM_MEDIA_TYPE: &str = "application/vnd.module.wasm.content.layer.v1+wasm";
+const WASM_CONFIG_MEDIA_TYPE: &str = "application/vnd.wascc.actor.archive.config";
+const WASM_FILE_EXTENSION: &str = ".wasm";
+
+const SHOWER_EMOJI: &str = "\u{1F6BF}";
+
+enum SupportedArtifacts {
+    Par,
+    Wasm,
+}
+
+#[derive(Debug, StructOpt, Clone)]
+#[structopt(
+    global_settings(&[AppSettings::ColoredHelp, AppSettings::VersionlessSubcommands]),
+    name = "reg")]
+pub struct RegCli {
+    #[structopt(flatten)]
+    command: RegCliCommand,
+
+    /// OCI username, if omitted anonymous authentication will be used
+    #[structopt(
+        short = "u",
+        long = "user",
+        env = "WASH_REG_USER",
+        hide_env_values = true
+    )]
+    user: Option<String>,
+
+    /// OCI password, if omitted anonymous authentication will be used
+    #[structopt(
+        short = "p",
+        long = "password",
+        env = "WASH_REG_PASSWORD",
+        hide_env_values = true
+    )]
+    password: Option<String>,
+
+    /// Allow insecure (HTTP) registry connections
+    #[structopt(long = "insecure")]
+    insecure: bool,
+}
+
+#[derive(Debug, Clone, StructOpt)]
+enum RegCliCommand {
+    /// Pull an artifact from an OCI compliant registry
+    #[structopt(name = "pull")]
+    Pull(PullCommand),
+    /// Push an artifact to an OCI compliant registry
+    #[structopt(name = "push")]
+    Push(PushCommand),
+}
+
+#[derive(StructOpt, Debug, Clone)]
+struct PullCommand {
+    /// URL of artifact
+    #[structopt(name = "url")]
+    url: String,
+
+    /// Path to output
+    #[structopt(short = "o", long = "output")]
+    output: Option<String>,
+
+    /// Digest to verify artifact against
+    #[structopt(short = "d", long = "digest")]
+    digest: Option<String>,
+
+    /// Allow latest artifact tags
+    #[structopt(long = "allow-latest")]
+    allow_latest: bool,
+}
+
+#[derive(StructOpt, Debug, Clone)]
+struct PushCommand {
+    /// URL to push artifact to
+    #[structopt(name = "url")]
+    url: String,
+
+    /// Path to artifact to push
+    #[structopt(name = "artifact")]
+    artifact: String,
+
+    /// Path to config file, if omitted will default to a blank configuration
+    #[structopt(short = "c", long = "config")]
+    config: Option<String>,
+
+    /// Allow latest artifact tags
+    #[structopt(long = "allow-latest")]
+    allow_latest: bool,
+}
+
+pub fn handle_command(cli: RegCli) -> Result<(), Box<dyn ::std::error::Error>> {
+    match cli.command {
+        RegCliCommand::Pull(cmd) => handle_pull(cmd, cli.user, cli.password, cli.insecure),
+        RegCliCommand::Push(cmd) => handle_push(cmd, cli.user, cli.password, cli.insecure),
+    }
+}
+
+fn handle_pull(
+    cmd: PullCommand,
+    user: Option<String>,
+    password: Option<String>,
+    insecure: bool,
+) -> Result<(), Box<dyn ::std::error::Error>> {
+    let image: Reference = cmd.url.parse().unwrap();
+
+    if image.tag().unwrap_or("latest") == "latest" && !cmd.allow_latest {
+        return Err(
+            "Pulling artifacts with tag 'latest' is prohibited. This can be overriden with a flag"
+                .into(),
+        );
+    };
+
+    let mut client = Client::new(ClientConfig {
+        protocol: if insecure {
+            ClientProtocol::Http
+        } else {
+            ClientProtocol::Https
+        },
+    });
+
+    let auth = match (user, password) {
+        (Some(user), Some(password)) => RegistryAuth::Basic(user, password),
+        _ => RegistryAuth::Anonymous,
+    };
+
+    let sp = Spinner::new(
+        Spinners::Dots12,
+        format!(" Downloading {} ...", image.whole()),
+    );
+
+    // Asynchronous code from the oci-distribution crate must run on the tokio runtime
+    let mut rt = Runtime::new()?;
+    let image_data = rt.block_on(client.pull(
+        &image,
+        &auth,
+        vec![PROVIDER_ARCHIVE_MEDIA_TYPE, WASM_MEDIA_TYPE],
+    ))?;
+
+    sp.message(format!(" Validating {} ...", image.whole()));
+
+    // Reformatting digest in case the sha256: prefix is left off
+    let digest = match cmd.digest {
+        Some(d) if d.starts_with("sha256:") => Some(d),
+        Some(d) => Some(format!("sha256:{}", d)),
+        None => None,
+    };
+
+    match (digest, image_data.digest) {
+        (Some(digest), Some(image_digest)) if digest != image_digest => {
+            Err("Image digest did not match provided digest, aborting")
+        }
+        _ => Ok(()),
+    }?;
+
+    let artifact = image_data
+        .layers
+        .iter()
+        .map(|l| l.data.clone())
+        .flatten()
+        .collect::<Vec<_>>();
+
+    let file_extension = match validate_artifact(&artifact, image.repository())? {
+        SupportedArtifacts::Par => PROVIDER_ARCHIVE_FILE_EXTENSION,
+        SupportedArtifacts::Wasm => WASM_FILE_EXTENSION,
+    };
+
+    // Output to provided file, or use artifact_name.file_extension
+    let outfile = cmd.output.unwrap_or(format!(
+        "{}{}",
+        image
+            .repository()
+            .to_string()
+            .split('/')
+            .collect::<Vec<_>>()
+            .pop()
+            .unwrap()
+            .to_string(),
+        file_extension
+    ));
+    let mut f = File::create(outfile.clone())?;
+    f.write_all(&artifact)?;
+
+    sp.stop();
+    println!(
+        "\n{} Successfully pulled and validated {}",
+        SHOWER_EMOJI, outfile
+    );
+
+    Ok(())
+}
+
+/// Helper function to determine artifact type and validate that it is
+/// a valid artifact of that type
+fn validate_artifact(
+    artifact: &[u8],
+    name: &str,
+) -> Result<SupportedArtifacts, Box<dyn ::std::error::Error>> {
+    match validate_actor_module(artifact, name) {
+        Ok(_) => Ok(SupportedArtifacts::Wasm),
+        Err(_) => match validate_provider_archive(artifact, name) {
+            Ok(_) => Ok(SupportedArtifacts::Par),
+            Err(_) => Err("Unsupported artifact type".into()),
+        },
+    }
+}
+
+/// Attempts to inspect the claims of an actor module
+/// Will fail without actor claims, or if the artifact is invalid
+fn validate_actor_module(
+    artifact: &[u8],
+    module: &str,
+) -> Result<(), Box<dyn ::std::error::Error>> {
+    match wascap::wasm::extract_claims(&artifact) {
+        Ok(Some(_token)) => Ok(()),
+        Ok(None) => Err(format!("No capabilities discovered in actor module : {}", &module).into()),
+        Err(e) => Err(Box::new(e)),
+    }
+}
+
+/// Attempts to unpack a provider archive
+/// Will fail without claims or if the archive is invalid
+fn validate_provider_archive(
+    artifact: &[u8],
+    archive: &str,
+) -> Result<(), Box<dyn ::std::error::Error>> {
+    match ProviderArchive::try_load(artifact) {
+        Ok(_par) => Ok(()),
+        Err(_e) => Err(format!("Invalid provider archive : {}", archive).into()),
+    }
+}
+
+fn handle_push(
+    cmd: PushCommand,
+    user: Option<String>,
+    password: Option<String>,
+    insecure: bool,
+) -> Result<(), Box<dyn ::std::error::Error>> {
+    let image: Reference = cmd.url.parse().unwrap();
+
+    if image.tag().unwrap() == "latest" && !cmd.allow_latest {
+        return Err(
+            "Pushing artifacts with tag 'latest' is prohibited. This can be overriden with a flag"
+                .into(),
+        );
+    };
+
+    let sp = Spinner::new(Spinners::Dots12, format!(" Loading {} ...", cmd.artifact));
+    let mut config_buf = vec![];
+    match cmd.config {
+        Some(config_file) => {
+            let mut f = File::open(config_file)?;
+            f.read_to_end(&mut config_buf)?;
+        }
+        None => {
+            // If no config provided, send blank config
+            config_buf = b"{}".to_vec();
+        }
+    };
+
+    let mut artifact_buf = vec![];
+    let mut f = File::open(cmd.artifact.clone())?;
+    f.read_to_end(&mut artifact_buf)?;
+
+    sp.message(format!(" Verifying {} ...", cmd.artifact));
+
+    let (artifact_media_type, config_media_type) =
+        match validate_artifact(&artifact_buf, &cmd.artifact)? {
+            SupportedArtifacts::Wasm => (WASM_MEDIA_TYPE, WASM_CONFIG_MEDIA_TYPE),
+            SupportedArtifacts::Par => (
+                PROVIDER_ARCHIVE_MEDIA_TYPE,
+                PROVIDER_ARCHIVE_CONFIG_MEDIA_TYPE,
+            ),
+        };
+
+    let image_data = ImageData {
+        layers: vec![ImageLayer {
+            data: artifact_buf,
+            media_type: artifact_media_type.to_string(),
+        }],
+        digest: None,
+    };
+
+    let mut client = Client::new(ClientConfig {
+        protocol: if insecure {
+            ClientProtocol::Http
+        } else {
+            ClientProtocol::Https
+        },
+    });
+
+    let auth = match (user, password) {
+        (Some(user), Some(password)) => RegistryAuth::Basic(user, password),
+        _ => RegistryAuth::Anonymous,
+    };
+
+    sp.message(format!(
+        " Pushing {} to {} ...",
+        cmd.artifact,
+        image.whole()
+    ));
+
+    // Asynchronous code from the oci-distribution crate must run on the tokio runtime
+    let mut rt = Runtime::new()?;
+    rt.block_on(client.push(
+        &image,
+        &image_data,
+        &config_buf,
+        config_media_type,
+        &auth,
+        None,
+    ))?;
+
+    sp.stop();
+    println!(
+        "\n{} Successfully validated and pushed to {}",
+        SHOWER_EMOJI,
+        image.whole()
+    );
+
+    Ok(())
+}