WASI runtime

.github/workflows/build-wasm_of_ocaml.yml

@@ -16,6 +16,8 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+        os-name:
+          - Ubuntu
         ocaml-compiler:
           - "4.14"
           - "5.0"
@@ -27,30 +29,50 @@ jobs:
           - false
         all_jane_street_tests:
           - false
+        wasi:
+          - false
         include:
           - os: macos-latest
+            os-name: MacOS
             ocaml-compiler: "5.3"
             separate_compilation: true
             jane_street_tests: false
             all_jane_street_tests: false
+            wasi: false
           - os: windows-latest
+            os-name: Windows
             ocaml-compiler: "5.2"
             separate_compilation: true
             jane_street_tests: true
             all_jane_street_tests: true
+            wasi: false
           - os: ubuntu-latest
+            os-name: Ubuntu
             ocaml-compiler: "5.2"
             separate_compilation: true
             jane_street_tests: true
             all_jane_street_tests: true
+            wasi: false
           - os: ubuntu-latest
+            os-name: Ubuntu
             ocaml-compiler: "5.2"
             separate_compilation: false
             jane_street_tests: true
             all_jane_street_tests: false
+            wasi: false
+          - os: ubuntu-latest
+            os-name: Ubuntu
+            ocaml-compiler: "5.3"
+            separate_compilation: true
+            jane_street_tests: false
+            all_jane_street_tests: false
+            wasi: true
 
     runs-on: ${{ matrix.os }}
 
+    name:
+       ${{ matrix.wasi && 'WASI / ' || '' }}${{ (! matrix.separate_compilation) && 'Whole program / ' || ''}}${{ matrix.ocaml-compiler }} / ${{ matrix.os-name }}${{ matrix.all_jane_street_tests && ' / Jane Street tests' || ''}}
+
     steps:
       - name: Set git to use LF
         if: ${{ matrix.os == 'windows-latest' && matrix.ocaml-compiler < 5.2 }}
@@ -77,6 +99,54 @@ jobs:
         with:
           node-version: latest
 
+      - name: Set-up Rust toolchain
+        if: matrix.wasi
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Checkout Wasmtime
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: bytecodealliance/wasmtime
+          path: wasmtime
+          submodules: true
+
+      - name: Build Wasmtime
+        if: matrix.wasi
+        working-directory: ./wasmtime
+        run: |
+          cargo build
+          echo `pwd`/target/debug >> "$GITHUB_PATH"
+
+      - name: Checkout Virgil
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: titzer/virgil
+          path: virgil
+
+      - name: Build Virgil
+        if: matrix.wasi
+        working-directory: ./virgil
+        run: |
+          export PATH=$PATH:`pwd`/bin
+          echo `pwd`/bin >> "$GITHUB_PATH"
+          make
+
+      - name: Checkout Wizard engine
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: titzer/wizard-engine
+          path: wizard-engine
+
+      - name: Build Wizard engine
+        if: matrix.wasi
+        working-directory: ./wizard-engine
+        run: |
+          make -j 4
+          echo `pwd`/bin >> "$GITHUB_PATH"
+
       - name: Set-up OCaml ${{ matrix.ocaml-compiler }}
         uses: ocaml/setup-ocaml@v3
         with:
@@ -127,7 +197,7 @@ jobs:
           opam install num cohttp-lwt-unix ppx_expect cstruct uucp
 
       - name: Run tests
-        if: ${{ matrix.separate_compilation }}
+        if: ${{ matrix.separate_compilation && ! matrix.wasi }}
         working-directory: ./wasm_of_ocaml
         run: opam exec -- dune build @runtest-wasm
 
@@ -136,11 +206,34 @@ jobs:
       # See https://github.com/libuv/libuv/issues/3622
 
       - name: Run tests with CPS effects
-        if: ${{ matrix.ocaml-compiler >= '5.' && matrix.separate_compilation }}
+        if: ${{ matrix.ocaml-compiler >= '5.' && matrix.separate_compilation && ! matrix.wasi }}
         continue-on-error: ${{ matrix.os == 'windows-latest' }}
         working-directory: ./wasm_of_ocaml
         run: opam exec -- dune build @runtest-wasm --profile with-effects
 
+      - name: Run tests (WASI runtime - node)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
+      - name: Run tests (WASI runtime - wasmtime)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        env:
+          WASM_ENGINE: wasmtime
+          WASI_FLAGS: --enable trap-on-exception
+          RUST_BACKTRACE: 0
+        continue-on-error: true
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
+      - name: Run tests (WASI runtime - Wizard engine)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        env:
+          WASM_ENGINE: wizard-fast
+        continue-on-error: true
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
       - name: Run Base tests
         if: matrix.all_jane_street_tests
         continue-on-error: ${{ matrix.os == 'windows-latest' }}

CHANGES.md

@@ -1,3 +1,9 @@
+# dev
+
+## Features/Changes
+* Runtime: support more Unix functions (#1829)
+* Compiler: use a Wasm text files preprocessor (#1822)
+
 # 6.0.1 (2025-02-07) - Lille
 
 ## Features/Changes

compiler/bin-wasm_of_ocaml/compile.ml

@@ -73,6 +73,54 @@ let output_gen output_file f =
   Code.Var.set_stable (Config.Flag.stable_var ());
   Filename.gen_file output_file f
 
+let with_runtime_files ~runtime_wasm_files f =
+  let inputs =
+    List.map
+      ~f:(fun file ->
+        { Wat_preprocess.module_name = "env"
+        ; file
+        ; source =
+            (if Link.Wasm_binary.check_file ~file
+             then File
+             else Contents (Js_of_ocaml_compiler.Fs.read_file file))
+        })
+      runtime_wasm_files
+  in
+  Wat_preprocess.with_preprocessed_files ~variables:[] ~inputs f
+
+let build_runtime ~runtime_file =
+  (* Keep this variables in sync with gen/gen.ml *)
+  let variables =
+    [ "wasi", Config.Flag.wasi (); "trap-on-exception", Config.Flag.trap_on_exception () ]
+  in
+  match
+    List.find_opt Runtime_files.precompiled_runtimes ~f:(fun (flags, _) ->
+        assert (
+          List.length flags = List.length variables
+          && List.for_all2 ~f:(fun (k, _) (k', _) -> String.equal k k') flags variables);
+        Poly.equal flags variables)
+  with
+  | Some (_, contents) -> Fs.write_file ~name:runtime_file ~contents
+  | None ->
+      let inputs =
+        List.map
+          ~f:(fun (module_name, contents) ->
+            { Wat_preprocess.module_name
+            ; file = module_name ^ ".wat"
+            ; source = Contents contents
+            })
+          (if Config.Flag.wasi ()
+           then ("libc", Runtime_files.wasi_libc) :: Runtime_files.wat_files
+           else Runtime_files.wat_files)
+      in
+      Runtime.build
+        ~link_options:[ "-g" ]
+        ~opt_options:[ "-g"; "-O2" ]
+        ~variables:
+          (List.map ~f:(fun (k, v) : (_ * Wat_preprocess.value) -> k, Bool v) variables)
+        ~inputs
+        ~output_file:runtime_file
+
 let link_and_optimize
     ~profile
     ~sourcemap_root
@@ -91,7 +139,7 @@ let link_and_optimize
   let enable_source_maps = Option.is_some opt_sourcemap_file in
   Fs.with_intermediate_file (Filename.temp_file "runtime" ".wasm")
   @@ fun runtime_file ->
-  Fs.write_file ~name:runtime_file ~contents:Runtime_files.wasm_runtime;
+  build_runtime ~runtime_file;
   Fs.with_intermediate_file (Filename.temp_file "wasm-merged" ".wasm")
   @@ fun temp_file ->
   opt_with
@@ -100,11 +148,15 @@ let link_and_optimize
      then Some (Filename.temp_file "wasm-merged" ".wasm.map")
      else None)
   @@ fun opt_temp_sourcemap ->
+  (with_runtime_files ~runtime_wasm_files
+  @@ fun runtime_inputs ->
   Binaryen.link
-    ~runtime_files:(runtime_file :: runtime_wasm_files)
-    ~input_files:wat_files
+    ~inputs:
+      (({ Binaryen.module_name = "env"; file = runtime_file } :: runtime_inputs)
+      @ List.map ~f:(fun file -> { Binaryen.module_name = "OCaml"; file }) wat_files)
     ~opt_output_sourcemap:opt_temp_sourcemap
-    ~output_file:temp_file;
+    ~output_file:temp_file
+    ());
   Fs.with_intermediate_file (Filename.temp_file "wasm-dce" ".wasm")
   @@ fun temp_file' ->
   opt_with
@@ -113,7 +165,10 @@ let link_and_optimize
   @@ fun opt_temp_sourcemap' ->
   let primitives =
     Binaryen.dead_code_elimination
-      ~dependencies:Runtime_files.dependencies
+      ~dependencies:
+        (if Config.Flag.wasi ()
+         then Runtime_files.wasi_dependencies
+         else Runtime_files.dependencies)
       ~opt_input_sourcemap:opt_temp_sourcemap
       ~opt_output_sourcemap:opt_temp_sourcemap'
       ~input_file:temp_file
@@ -124,7 +179,8 @@ let link_and_optimize
     ~opt_input_sourcemap:opt_temp_sourcemap'
     ~opt_output_sourcemap:opt_sourcemap
     ~input_file:temp_file'
-    ~output_file;
+    ~output_file
+    ();
   Option.iter
     ~f:(update_sourcemap ~sourcemap_root ~sourcemap_don't_inline_content)
     opt_sourcemap_file;
@@ -133,20 +189,23 @@ let link_and_optimize
 let link_runtime ~profile runtime_wasm_files output_file =
   Fs.with_intermediate_file (Filename.temp_file "runtime" ".wasm")
   @@ fun runtime_file ->
-  Fs.write_file ~name:runtime_file ~contents:Runtime_files.wasm_runtime;
+  build_runtime ~runtime_file;
   Fs.with_intermediate_file (Filename.temp_file "wasm-merged" ".wasm")
   @@ fun temp_file ->
+  with_runtime_files ~runtime_wasm_files
+  @@ fun runtime_inputs ->
   Binaryen.link
     ~opt_output_sourcemap:None
-    ~runtime_files:(runtime_file :: runtime_wasm_files)
-    ~input_files:[]
-    ~output_file:temp_file;
+    ~inputs:({ Binaryen.module_name = "env"; file = runtime_file } :: runtime_inputs)
+    ~output_file:temp_file
+    ();
   Binaryen.optimize
     ~profile
     ~opt_input_sourcemap:None
     ~opt_output_sourcemap:None
     ~input_file:temp_file
     ~output_file
+    ()
 
 let generate_prelude ~out_file =
   Filename.gen_file out_file
@@ -186,7 +245,8 @@ let build_prelude z =
     ~input_file:prelude_file
     ~output_file:tmp_prelude_file
     ~opt_input_sourcemap:None
-    ~opt_output_sourcemap:None;
+    ~opt_output_sourcemap:None
+    ();
   Zip.add_file z ~name:"prelude.wasm" ~file:tmp_prelude_file;
   predefined_exceptions
 
@@ -216,7 +276,13 @@ let build_js_runtime ~primitives ?runtime_arguments () =
   in
   let prelude = Link.output_js always_required_js in
   let init_fun =
-    match Parse_js.parse (Parse_js.Lexer.of_string Runtime_files.js_runtime) with
+    match
+      Parse_js.parse
+        (Parse_js.Lexer.of_string
+           (if Config.Flag.wasi ()
+            then Runtime_files.js_wasi_launcher
+            else Runtime_files.js_launcher))
+    with
     | [ (Expression_statement f, _) ] -> f
     | _ -> assert false
   in
@@ -413,7 +479,8 @@ let run
            ~opt_input_sourcemap:None
            ~opt_output_sourcemap:opt_tmp_map_file
            ~input_file:wat_file
-           ~output_file:tmp_wasm_file;
+           ~output_file:tmp_wasm_file
+           ();
          { Link.unit_name; unit_info; strings; fragments }
        in
        cont unit_data unit_name tmp_wasm_file opt_tmp_map_file
@@ -455,9 +522,12 @@ let run
              tmp_wasm_file
          in
          let wasm_name =
-           Printf.sprintf
-             "code-%s"
-             (String.sub (Digest.to_hex (Digest.file tmp_wasm_file)) ~pos:0 ~len:20)
+           if Config.Flag.wasi ()
+           then "code"
+           else
+             Printf.sprintf
+               "code-%s"
+               (String.sub (Digest.to_hex (Digest.file tmp_wasm_file)) ~pos:0 ~len:20)
          in
          let tmp_wasm_file' = Filename.concat tmp_dir (wasm_name ^ ".wasm") in
          Sys.rename tmp_wasm_file tmp_wasm_file';

compiler/bin-wasm_of_ocaml/dune

@@ -1,6 +1,6 @@
-(executable
- (name wasm_of_ocaml)
- (public_name wasm_of_ocaml)
+(executables
+ (names wasm_of_ocaml wasmoo_link_wasm)
+ (public_names wasm_of_ocaml -)
  (package wasm_of_ocaml-compiler)
  (libraries
   jsoo_cmdline
@@ -25,9 +25,13 @@
  (target runtime_files.ml)
  (deps
   gen/gen.exe
-  ../../runtime/wasm/runtime.wasm
   ../../runtime/wasm/runtime.js
-  ../../runtime/wasm/deps.json)
+  ../../runtime/wasm/deps.json
+  ../../runtime/wasm/runtime-wasi.js
+  ../../runtime/wasm/deps-wasi.json
+  ../../runtime/wasm/libc.wasm
+  (glob_files ../../runtime/wasm/*.wat)
+  (glob_files ../../runtime/wasm/runtime-*.wasm))
  (action
   (with-stdout-to
    %{target}