test(*): hardening codegen tests by adding wasi proposals as submodules (#919)

* test(*): hardening codegen tests by adding wasi proposals as submodules

this commit adds a few git submodules for wasi proposals and placed them
in `tests/codegen` folder for testing purposes. it also changes the structure
now that it requires wit packages to have a `wit` folder inside the directory
similar to how wasi proposal structures.

Signed-off-by: Jiaxiao Zhou (Mossaka) <[email protected]>

* test(rust): skip wasi-http and wasi-cli

both have multiple worlds which require another refactoring of the
current test codebase to adopt that

Signed-off-by: Jiaxiao Zhou (Mossaka) <[email protected]>

* test(*): skip wasi-* tests on teavm and csharp

Signed-off-by: Jiaxiao Zhou (Mossaka) <[email protected]>

---------

Signed-off-by: Jiaxiao Zhou (Mossaka) <[email protected]>

Author: Mossaka

.github/workflows/main.yml

@@ -60,6 +60,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
     - name: Install Rust
       run: rustup update stable --no-self-update && rustup default stable
     - name: Install wasm32-unknown-unknown target

.gitmodules

@@ -0,0 +1,15 @@
+[submodule "tests/codegen/wasi-filesystem"]
+	path = tests/codegen/wasi-filesystem
+	url = https://github.com/WebAssembly/wasi-filesystem
+[submodule "tests/codegen/wasi-http"]
+	path = tests/codegen/wasi-http
+	url = https://github.com/WebAssembly/wasi-http
+[submodule "tests/codegen/wasi-cli"]
+	path = tests/codegen/wasi-cli
+	url = https://github.com/WebAssembly/wasi-cli
+[submodule "tests/codegen/wasi-io"]
+	path = tests/codegen/wasi-io
+	url = https://github.com/WebAssembly/wasi-io
+[submodule "tests/codegen/wasi-clocks"]
+	path = tests/codegen/wasi-clocks
+	url = https://github.com/WebAssembly/wasi-clocks

crates/csharp/tests/codegen.rs

@@ -56,6 +56,11 @@ macro_rules! codegen_test {
                         "worlds-with-types",
                         "variants-unioning-types",
                         "go_params",
+                        "wasi-cli",
+                        "wasi-clocks",
+                        "wasi-filesystem",
+                        "wasi-http",
+                        "wasi-io",
                     ]
                     .contains(&$name)
                     {

crates/rust/tests/codegen.rs

@@ -3,6 +3,8 @@
 
 mod codegen_tests {
     macro_rules! codegen_test {
+        (wasi_cli $name:tt $test:tt) => {};
+        (wasi_http $name:tt $test:tt) => {};
         ($id:ident $name:tt $test:tt) => {
             mod $id {
                 wit_bindgen::generate!({

crates/rust/tests/codegen_no_std.rs

@@ -14,6 +14,8 @@ extern crate alloc;
 
 mod codegen_tests {
     macro_rules! codegen_test {
+        (wasi_cli $name:tt $test:tt) => {};
+        (wasi_http $name:tt $test:tt) => {};
         ($id:ident $name:tt $test:tt) => {
             mod $id {
                 wit_bindgen::generate!({

crates/teavm-java/tests/codegen.rs

@@ -20,6 +20,11 @@ macro_rules! codegen_test {
     (resources_in_aggregates $name:tt $test:tt) => {};
     (issue668 $name:tt $test:tt) => {};
     (multiversion $name:tt $test:tt) => {};
+    (wasi_cli $name:tt $test:tt) => {};
+    (wasi_clocks $name:tt $test:tt) => {};
+    (wasi_filesystem $name:tt $test:tt) => {};
+    (wasi_http $name:tt $test:tt) => {};
+    (wasi_io $name:tt $test:tt) => {};
 
     ($id:ident $name:tt $test:tt) => {
         #[test]

crates/test-helpers/codegen-macro/src/lib.rs

@@ -7,26 +7,31 @@ use std::env;
 /// `codegen_test!` macro then does what's necessary to actually run the test.
 #[proc_macro]
 pub fn codegen_tests(_input: TokenStream) -> TokenStream {
-    let mut tests = Vec::new();
-    for entry in env::current_dir()
-        .unwrap()
-        .join("tests/codegen")
+    let tests_dir = env::current_dir().unwrap().join("tests/codegen");
+    let tests = tests_dir
         .read_dir()
         .unwrap()
-    {
-        let entry = entry.unwrap();
-        let test = entry.path();
+        .filter_map(|entry| {
+            let entry = entry.ok()?;
+            let path = entry.path();
+            let is_dir = entry.file_type().unwrap().is_dir();
+            if is_dir || path.extension().and_then(|s| s.to_str()) == Some("wit") {
+                let test_path = if is_dir {
+                    path.join("wit")
+                } else {
+                    path.clone()
+                };
+                let name = path.file_stem().unwrap().to_str().unwrap();
+                let ident = quote::format_ident!("{}", name.to_snake_case());
+                let path = test_path.to_str().unwrap();
+                Some(quote::quote! {
+                    codegen_test!(#ident #name #path);
+                })
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
 
-        if entry.file_type().unwrap().is_dir()
-            || test.extension().and_then(|s| s.to_str()) == Some("wit")
-        {
-            let name = test.file_stem().unwrap().to_str().unwrap();
-            let path = test.to_str().unwrap();
-            let ident = quote::format_ident!("{}", name.to_snake_case());
-            tests.push(quote::quote! {
-                codegen_test!(#ident #name #path);
-            });
-        }
-    }
     (quote::quote!(#(#tests)*)).into()
 }

crates/test-helpers/src/lib.rs

@@ -66,7 +66,16 @@ pub fn run_world_codegen_test(
     let (resolve, world) = parse_wit(wit_path);
     let world_name = &resolve.worlds[world].name;
 
-    let wit_name = wit_path.file_stem().and_then(|s| s.to_str()).unwrap();
+    let wit_name = if wit_path.is_dir() {
+        wit_path
+            .parent()
+            .unwrap()
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .unwrap()
+    } else {
+        wit_path.file_stem().and_then(|s| s.to_str()).unwrap()
+    };
     let gen_name = format!("{gen_name}-{wit_name}");
     let dir = test_directory("codegen", &gen_name, &world_name);
 
@@ -106,7 +115,16 @@ pub fn run_component_codegen_test(
         .encode()
         .unwrap();
 
-    let wit_name = wit_path.file_stem().and_then(|s| s.to_str()).unwrap();
+    let wit_name = if wit_path.is_dir() {
+        wit_path
+            .parent()
+            .unwrap()
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .unwrap()
+    } else {
+        wit_path.file_stem().and_then(|s| s.to_str()).unwrap()
+    };
 
     let gen_name = format!("{gen_name}-{wit_name}",);
     let dir = test_directory("codegen", &gen_name, &world_name);
@@ -126,6 +144,9 @@ pub fn run_component_codegen_test(
 fn parse_wit(path: &Path) -> (Resolve, WorldId) {
     let mut resolve = Resolve::default();
     let (pkg, _files) = resolve.push_path(path).unwrap();
-    let world = resolve.select_world(pkg, None).unwrap();
+    let world = resolve.select_world(pkg, None).unwrap_or_else(|_| {
+        // note: if there are multiples worlds in the wit package, we assume the "imports" world
+        resolve.select_world(pkg, Some("imports")).unwrap()
+    });
     (resolve, world)
 }

tests/README.md

@@ -41,14 +41,18 @@ Read on to learn more about the testing layout. It's all a bit convoluted so fee
 
 ## Testing wit-bindgen - `codegen`
 
-Any tests placed into the `tests/codegen` directory should be raw `*.wit`
-files. These files will be executed in all code generators by default most
-likely, and the purpose of these files is to execute language-specific
+Any tests placed in to the `tests/codegen` directory should follow either of the following formats:
+1. `*.wit` files that are raw wit files that should be executed by the code generator.
+2. wit package in it's own directory which must contain a `wit` subdirectory with `*.wit` files and `deps` in it. (e.g. see` tests/codegen/issue569`)
+
+The purpose of these files is to execute language-specific
 validation for each bindings generator. Basically if there's a bug where
 something generates invalid code then this is probably where the test should go.
 Note that this directory can have whatever it wants since nothing implements the
 interfaces or tries to call them.
 
+It also contains git submodules for the wasi proposals like `wasi-http`.
+
 The tests are generated by a macro `codegen_tests` in [crates/test-helpers](../crates/test-helpers/).
 
 ## Testing wit-bindgen - `runtime`