Include erlang/elixir ffi modules in exported erlang app files

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Compiler
 
+- Generated erlang .app files now include ffi modules from elixir and erlang.
+  ([LostKobrakai](https://github.com/lostkobrakai))
+
 ### Language server
 
 ### Formatter

compiler-cli/src/beam_compiler.rs

@@ -35,7 +35,7 @@ impl BeamCompiler {
         lib: &Utf8Path,
         modules: &HashSet<Utf8PathBuf>,
         stdio: Stdio,
-    ) -> Result<(), Error> {
+    ) -> Result<Vec<String>, Error> {
         let inner = match self.inner {
             Some(ref mut inner) => {
                 if let Ok(None) = inner.process.try_wait() {
@@ -66,15 +66,24 @@ impl BeamCompiler {
         })?;
 
         let mut buf = String::new();
+        let mut accumulated_modules: Vec<String> = Vec::new();
         while let (Ok(_), Ok(None)) = (inner.stdout.read_line(&mut buf), inner.process.try_wait()) {
             match buf.trim() {
-                "gleam-compile-result-ok" => return Ok(()),
+                "gleam-compile-result-ok" => {
+                    // Return Ok with the accumulated modules
+                    return Ok(accumulated_modules);
+                }
                 "gleam-compile-result-error" => {
                     return Err(Error::ShellCommand {
                         program: "escript".into(),
                         err: None,
                     })
                 }
+                s if s.starts_with("gleam-compile-module:") => {
+                    if let Some(module_content) = s.strip_prefix("gleam-compile-module:") {
+                        accumulated_modules.push(module_content.to_string());
+                    }
+                }
                 _ => match stdio {
                     Stdio::Inherit => print!("{}", buf),
                     Stdio::Null => {}

compiler-cli/src/fs.rs

@@ -228,7 +228,7 @@ impl BeamCompiler for ProjectIO {
         lib: &Utf8Path,
         modules: &HashSet<Utf8PathBuf>,
         stdio: Stdio,
-    ) -> Result<(), Error> {
+    ) -> Result<Vec<String>, Error> {
         self.beam_compiler
             .lock()
             .as_mut()

compiler-cli/templates/gleam@@compile.erl

@@ -17,8 +17,14 @@ compile_package_loop() ->
             {ok, Tokens, _} = erl_scan:string(Chars),
             {ok, {Lib, Out, Modules}} = erl_parse:parse_term(Tokens),
             case compile_package(Lib, Out, Modules) of
-                ok -> io:put_chars("gleam-compile-result-ok\n");
-                err -> io:put_chars("gleam-compile-result-error\n")
+                {ok, ModuleNames} ->
+                    PrintModuleName = fun(ModuleName) ->
+                        io:put_chars("gleam-compile-module:" ++ atom_to_list(ModuleName) ++ "\n")
+                    end,
+                    lists:map(PrintModuleName, ModuleNames),
+                    io:put_chars("gleam-compile-result-ok\n");
+                err ->
+                    io:put_chars("gleam-compile-result-error\n")
             end,
             compile_package_loop()
     end.
@@ -30,15 +36,18 @@ compile_package(Lib, Out, Modules) ->
     {ElixirModules, ErlangModules} = lists:partition(IsElixirModule, Modules),
     ok = filelib:ensure_dir([Out, $/]),
     ok = add_lib_to_erlang_path(Lib),
-    {ErlangOk, _ErlangBeams} = compile_erlang(ErlangModules, Out),
-    {ElixirOk, _ElixirBeams} = case ErlangOk of
+    {ErlangOk, ErlangBeams} = compile_erlang(ErlangModules, Out),
+    {ElixirOk, ElixirBeams} = case ErlangOk of
         true -> compile_elixir(ElixirModules, Out);
         false -> {false, []}
     end,
     ok = del_lib_from_erlang_path(Lib),
-    case ErlangOk and ElixirOk of
-        true -> ok;
-        false -> err
+    case ErlangOk andalso ElixirOk of
+        true ->
+            ModuleNames = proplists:get_keys(ErlangBeams ++ ElixirBeams),
+            {ok, ModuleNames};
+        false ->
+            err
     end.
 
 compile_erlang(Modules, Out) ->
@@ -48,7 +57,7 @@ compile_erlang(Modules, Out) ->
 
 collect_results(Acc = {Result, Beams}) ->
     receive
-        {compiled, Beam} -> collect_results({Result, [Beam | Beams]});
+        {compiled, ModuleName, Beam} -> collect_results({Result, [{ModuleName, Beam} | Beams]});
         failed -> collect_results({false, Beams})
         after 0 -> Acc
     end.
@@ -84,7 +93,7 @@ worker_loop(Parent, Out) ->
             case compile:file(Module, Options) of
                 {ok, ModuleName} ->
                     Beam = filename:join(Out, ModuleName) ++ ".beam",
-                    Message = {compiled, Beam},
+                    Message = {compiled, ModuleName, Beam},
                     log(Message),
                     erlang:send(Parent, Message);
                 error ->
@@ -133,7 +142,7 @@ do_compile_elixir(Modules, Out) ->
             ToBeam = fun(ModuleAtom) ->
                 Beam = filename:join(Out, atom_to_list(ModuleAtom)) ++ ".beam",
                 log({compiled, Beam}),
-                Beam
+                {ModuleAtom, Beam}
             end,
             {true, lists:map(ToBeam, ModuleAtoms)};
         {error, Errors, _} ->

compiler-core/src/build/package_compiler.rs

@@ -204,16 +204,20 @@ where
         Outcome::Ok(modules)
     }
 
-    fn compile_erlang_to_beam(&mut self, modules: &HashSet<Utf8PathBuf>) -> Result<(), Error> {
+    fn compile_erlang_to_beam(
+        &mut self,
+        modules: &HashSet<Utf8PathBuf>,
+    ) -> Result<Vec<EcoString>, Error> {
         if modules.is_empty() {
             tracing::debug!("no_erlang_to_compile");
-            return Ok(());
+            return Ok(Vec::new());
         }
 
         tracing::debug!("compiling_erlang");
 
         self.io
             .compile_beam(self.out, self.lib, modules, self.subprocess_stdio)
+            .map(|modules| modules.iter().map(|str| EcoString::from(str)).collect())
     }
 
     fn copy_project_native_files(
@@ -336,14 +340,6 @@ where
             tracing::debug!("skipping_native_file_copying");
         }
 
-        if let Some(config) = app_file_config {
-            ErlangApp::new(&self.out.join("ebin"), config).render(
-                io.clone(),
-                &self.config,
-                modules,
-            )?;
-        }
-
         if self.compile_beam_bytecode && self.write_entrypoint {
             self.render_erlang_entrypoint_module(&build_dir, &mut written)?;
         } else {
@@ -354,13 +350,23 @@ where
         // we overwrite any precompiled Erlang that was included in the Hex
         // package. Otherwise we will build the potentially outdated precompiled
         // version and not the newly compiled version.
-        Erlang::new(&build_dir, &include_dir).render(io, modules, self.root)?;
+        Erlang::new(&build_dir, &include_dir).render(io.clone(), modules, self.root)?;
 
-        if self.compile_beam_bytecode {
+        let native_modules: Vec<EcoString> = if self.compile_beam_bytecode {
             written.extend(modules.iter().map(Module::compiled_erlang_path));
-            self.compile_erlang_to_beam(&written)?;
+            self.compile_erlang_to_beam(&written)?
         } else {
             tracing::debug!("skipping_erlang_bytecode_compilation");
+            Vec::new()
+        };
+
+        if let Some(config) = app_file_config {
+            ErlangApp::new(&self.out.join("ebin"), config).render(
+                io,
+                &self.config,
+                modules,
+                native_modules,
+            )?;
         }
         Ok(())
     }

compiler-core/src/codegen.rs

@@ -8,6 +8,8 @@ use crate::{
     line_numbers::LineNumbers,
     Result,
 };
+use ecow::EcoString;
+use erlang::escape_atom_string;
 use itertools::Itertools;
 use std::fmt::Debug;
 
@@ -93,6 +95,7 @@ impl<'a> ErlangApp<'a> {
         writer: Writer,
         config: &PackageConfig,
         modules: &[Module],
+        native_modules: Vec<EcoString>,
     ) -> Result<()> {
         fn tuple(key: &str, value: &str) -> String {
             format!("    {{{key}, {value}}},\n")
@@ -110,7 +113,10 @@ impl<'a> ErlangApp<'a> {
         let modules = modules
             .iter()
             .map(|m| m.name.replace("/", "@"))
+            .chain(native_modules)
+            .unique()
             .sorted()
+            .map(|m| escape_atom_string(m.clone().into()))
             .join(",\n               ");
 
         // TODO: When precompiling for production (i.e. as a precompiled hex

compiler-core/src/erlang.rs

@@ -654,7 +654,7 @@ fn atom(value: &str) -> Document<'_> {
     }
 }
 
-fn escape_atom_string(value: String) -> EcoString {
+pub fn escape_atom_string(value: String) -> EcoString {
     if is_erlang_reserved_word(&value) {
         // Escape because of keyword collision
         eco_format!("'{value}'")

compiler-core/src/io.rs

@@ -340,7 +340,7 @@ pub trait BeamCompiler {
         lib: &Utf8Path,
         modules: &HashSet<Utf8PathBuf>,
         stdio: Stdio,
-    ) -> Result<(), Error>;
+    ) -> Result<Vec<String>, Error>;
 }
 
 /// A trait used to write files.

compiler-core/src/io/memory.rs

@@ -443,8 +443,8 @@ impl BeamCompiler for InMemoryFileSystem {
         _lib: &Utf8Path,
         _modules: &HashSet<Utf8PathBuf>,
         _stdio: Stdio,
-    ) -> Result<(), Error> {
-        Ok(()) // Always succeed.
+    ) -> Result<Vec<String>, Error> {
+        Ok(Vec::new()) // Always succeed.
     }
 }
 

compiler-core/src/language_server/files.rs

@@ -179,7 +179,7 @@ where
         _lib: &Utf8Path,
         _modules: &HashSet<Utf8PathBuf>,
         _stdio: Stdio,
-    ) -> Result<(), Error> {
+    ) -> Result<Vec<String>, Error> {
         panic!("The language server is not permitted to create subprocesses")
     }
 }

compiler-core/src/language_server/tests.rs

@@ -228,7 +228,7 @@ impl BeamCompiler for LanguageServerTestIO {
         lib: &Utf8Path,
         modules: &HashSet<Utf8PathBuf>,
         stdio: crate::io::Stdio,
-    ) -> Result<()> {
+    ) -> Result<Vec<String>> {
         panic!(
             "compile_beam({:?}, {:?}, {:?}, {:?}) is not implemented",
             out, lib, modules, stdio

compiler-wasm/src/wasm_filesystem.rs

@@ -39,8 +39,8 @@ impl BeamCompiler for WasmFileSystem {
         _lib: &Utf8Path,
         _modules: &HashSet<Utf8PathBuf>,
         _stdio: Stdio,
-    ) -> Result<(), Error> {
-        Ok(()) // Always succeed.
+    ) -> Result<Vec<String>, Error> {
+        Ok(Vec::new()) // Always succeed.
     }
 }
 

test/external_only_erlang/test.sh

@@ -33,6 +33,10 @@ if g run --target=javascript; then
   exit 1
 fi
 
+echo Running erlang shipment should succeed
+g export erlang-shipment
+grep "external_only_erlang_ffi" "build/erlang-shipment/external_only_erlang/ebin/external_only_erlang.app"
+
 echo
 echo Success! 💖
 echo