Adds high-level Writer API for defining macros (#938)

* Renames Macro -> MacroDef
* Stubs out ExprGroupWriter, impl WriteAsIon for TemplateMacroRef
* Enables `Writer` to compile macros and add them to the default module
* Introduces `MacroIdLike` trait that allows `&Macro`, addresses, and names to be used for writing E-exps.
* Updates `write_log_events` to use managed writer
* Improves documentation for the `write_log_events` example

Author: zslayton

examples/write_log_events.rs

@@ -1,21 +1,23 @@
 //! This program demonstrates implementing WriteAsIon using Ion 1.1's e-expressions for a more
-//! compact encoding. It uses raw-level writer APIs that end users are unlikely to leverage.
-//! Ion 1.1 is not yet finalized; the encoding produced by this example and the APIs it uses
-//! are very likely to change.
-
+//! compact encoding.
 use ion_rs::*;
 
 fn main() -> IonResult<()> {
     #[cfg(not(feature = "experimental"))]
-    panic!("This example requires the 'experimental' feature to work.");
+    {
+        eprintln!("This example requires the 'experimental' feature to work. Rebuild it with the flag `--features experimental`.");
+    }
 
     #[cfg(feature = "experimental")]
-    example::write_log_events()
+    example::write_log_events()?;
+
+    Ok(())
 }
 
 #[cfg(feature = "experimental")]
 mod example {
     use chrono::{DateTime, FixedOffset};
+    use ion_rs::v1_1::Macro;
     use ion_rs::*;
     use rand::rngs::StdRng;
     use rand::seq::SliceRandom;
@@ -55,29 +57,49 @@ mod example {
             ion_1_1_file.path().to_string_lossy(),
         );
 
-        // Encode the log events as Ion 1.0 data
-        let buf_writer = BufWriter::new(ion_1_0_file.as_file());
-        let mut ion_writer = v1_0::RawBinaryWriter::new(buf_writer)?;
-        for event in &events {
-            ion_writer.write(SerializeWithoutMacros(event))?;
-        }
-        ion_writer.flush()?;
-        drop(ion_writer);
-
-        // Encode the log events as Ion 1.1 data
-        let buf_writer = BufWriter::new(ion_1_1_file.as_file());
-        let mut ion_writer = v1_1::RawBinaryWriter::new(buf_writer)?;
-        for event in &events {
-            ion_writer.write(SerializeWithMacros(event))?;
-        }
-        ion_writer.flush()?;
-        drop(ion_writer);
+        // === Encode the log events as Ion 1.0 data ===
+        // First, we initialize a writer...
+        let mut ion_writer = Writer::new(v1_0::Binary, BufWriter::new(ion_1_0_file.as_file()))?;
+        // ...then we encode all of the events...
+        ion_writer.write_all(events.iter().map(|e| SerializeWithoutMacros(e)))?;
+        // ...finally, we close the writer, consuming it.
+        ion_writer.close()?;
+
+        // === Encode the log events as Ion 1.1 data ===
+        // First, we initialize a writer...
+        let mut ion_writer = Writer::new(v1_1::Binary, BufWriter::new(ion_1_1_file.as_file()))?;
+
+        // ...then we define some macros that we intend to use to encode our log data...
+        let macros = LogEventMacros {
+            thread_name: ion_writer.compile_macro(
+                // This macro includes the prefix common to all thread names, allowing the writer to only encode
+                // the suffix of each.
+                &format!(
+                    r#"
+                    (macro thread_name (suffix) (.make_string {THREAD_NAME_PREFIX} (%suffix) ))
+                "#
+                ),
+            )?,
+            log_statements: log_statements
+                .iter()
+                // As you'll see later, each LogStatement has an associated macro definition in text.
+                .map(|log_statement| ion_writer.compile_macro(&log_statement.macro_source))
+                .collect::<IonResult<Vec<Macro>>>()?,
+        };
+
+        // ...then we encode all of the events using the macros we just defined...
+        ion_writer.write_all(events.iter().map(|e| SerializeWithMacros(e, &macros)))?;
+        // ...finally, we close the writer, consuming it.
+        ion_writer.close()?;
+
+        // === Encoded file size comparison ===
 
         let size_in_1_0 = ion_1_0_file
             .as_file()
             .metadata()
             .expect("failed to read Ion 1.0 file length")
             .len();
+
         let size_in_1_1 = ion_1_1_file
             .as_file()
             .metadata()
@@ -100,18 +122,13 @@ mod example {
 
     // A log statement in the fictional codebase
     #[derive(Debug)]
-    // This struct has several fields that get populated but which are not (yet) used: `logger_name`
-    // `log_level`, and `format`. Currently, the encoded output for Ion 1.0 writes these as symbol
-    // IDs and Ion 1.1 refers to them as part of a macro. In both cases, however, the encoding
-    // context is not written out in the resulting Ion stream.
-    // TODO: Include the symbol/macro table definitions in the resulting output stream.
-    #[allow(dead_code)]
     struct LogStatement {
         index: usize,
         logger_name: String,
         log_level: String,
         format: String,
         parameter_types: Vec<ParameterType>,
+        macro_source: String,
     }
 
     impl LogStatement {
@@ -122,12 +139,37 @@ mod example {
             format: impl Into<String>,
             parameter_types: impl Into<Vec<ParameterType>>,
         ) -> Self {
+            let format = format.into();
+            let macro_source = format!(
+                // Note that there are two levels of interpolation in this string literal.
+                // The `format!` macro will process it first, replacing variable names in single
+                // braces (like `{class_name}`) with the corresponding text, and replacing
+                // double braces (like the `{{...}}` surrounding the template) with single braces.
+                // The resulting string is our macro source, which will be compiled by the Writer.
+                r#"
+(macro
+    ls{index:<42} // Name
+    (timestamp thread_id thread_name parameters) // Signature
+    {{                                            // Template
+        loggerName: "{class_name}",
+        logLevel: {log_level},
+        format: "{format}",
+        timestamp: (%timestamp),
+        thread_id: (%thread_id),
+        thread_name: (%thread_name),
+        parameters: (%parameters)
+    }}
+)
+                "#
+            );
+            println!("{macro_source}");
             Self {
                 index,
                 logger_name: format!("{PACKAGE_NAME}.{class_name}"),
                 log_level: log_level.to_string(),
                 format: format.into(),
                 parameter_types: parameter_types.into(),
+                macro_source,
             }
         }
     }
@@ -160,6 +202,13 @@ mod example {
 
     // ===== Serialization logic for the above types =====
 
+    // A simple container to store macros related to serializing LogEvent
+    struct LogEventMacros {
+        thread_name: Macro,
+        log_statements: Vec<Macro>,
+    }
+
+    // Defines how a `Parameter` is serialized as Ion
     impl WriteAsIon for Parameter {
         fn write_as_ion<V: ValueWriter>(&self, writer: V) -> IonResult<()> {
             match self {
@@ -173,71 +222,79 @@ mod example {
     // the future, as types will be able to define both a macro-ized serialization and a no-macros
     // serialization, allowing the writer to choose whichever is more appropriate.
     struct SerializeWithoutMacros<'a, 'b>(&'a LogEvent<'b>);
-    struct SerializeWithMacros<'a, 'b>(&'a LogEvent<'b>);
+    struct SerializeWithMacros<'a, 'b>(&'a LogEvent<'b>, &'a LogEventMacros);
 
     // When serializing without macros (usually in Ion 1.0), we write out a struct with each
-    // field name/value pair. In the case of recurring strings, we take the liberty of writing
-    // out symbol IDs instead of the full text; this silent type coercion from string to symbol
-    // is technically data loss, but results in a much more compact encoding.
+    // field name/value pair.
     impl WriteAsIon for SerializeWithoutMacros<'_, '_> {
         fn write_as_ion<V: ValueWriter>(&self, writer: V) -> IonResult<()> {
             let event = self.0;
             let mut struct_ = writer.struct_writer()?;
             struct_
                 //            v--- Each field name is a symbol ID
-                .write(10, event.timestamp)?
-                .write(11, event.thread_id)?
-                .write(12, &event.thread_name)?
-                //                 v--- The fixed strings from the log statement are also SIDs
-                .write(13, RawSymbolRef::SymbolId(17))? // logger name
-                .write(14, RawSymbolRef::SymbolId(18))? // log level
-                .write(15, RawSymbolRef::SymbolId(19))? // format
-                .write(16, &event.parameters)?;
+                .write("timestamp", event.timestamp)?
+                .write("threadId", event.thread_id)?
+                .write("threadName", &event.thread_name)?
+                .write(
+                    "loggerName",
+                    SymbolRef::with_text(&event.statement.logger_name),
+                )?
+                .write("logLevel", SymbolRef::with_text(&event.statement.log_level))?
+                .write("format", SymbolRef::with_text(&event.statement.format))?
+                .write("parameters", &event.parameters)?;
             struct_.close()
         }
     }
 
+    impl WriteAsIon for SerializeWithMacros<'_, '_> {
+        fn write_as_ion<V: ValueWriter>(&self, writer: V) -> IonResult<()> {
+            let SerializeWithMacros(event, macros) = *self;
+
+            // Create an e-expression writer to invoke the macro corresponding to this log statement.
+            let mut eexp = writer.eexp_writer(&macros.log_statements[event.statement.index])?;
+            eexp.write(event.timestamp)?
+                .write(event.thread_id)?
+                // Wrap the thread name in the `ThreadName` wrapper to change its serialization.
+                .write(ThreadName(&event.thread_name, &macros.thread_name))?
+                .write(&event.parameters)?;
+            eexp.close()
+        }
+    }
+
     // When leveraging macros, the thread name's recurring prefix can be elided from the output.
-    // This wrapper type is used by the `SerializeWithMacros` type to change to serialization
+    // This wrapper type is used by the `SerializeWithMacros` type to change the serialization
     // behavior for the thread name.
-    struct ThreadName<'a>(&'a str);
+    struct ThreadName<'a>(&'a str, &'a Macro);
 
     impl WriteAsIon for ThreadName<'_> {
         fn write_as_ion<V: ValueWriter>(&self, writer: V) -> IonResult<()> {
+            let thread_name_macro = self.1;
             // ID 12 chosen arbitrarily, but aligns with Ion 1.0 encoding above
-            let mut eexp = writer.eexp_writer(12)?;
+            let mut eexp = writer.eexp_writer(thread_name_macro)?;
             eexp
                 // Ignore the part of the thread name that starts with the recurring prefix.
                 .write(&self.0[THREAD_NAME_PREFIX.len()..])?;
             eexp.close()
         }
     }
 
-    impl WriteAsIon for SerializeWithMacros<'_, '_> {
-        fn write_as_ion<V: ValueWriter>(&self, writer: V) -> IonResult<()> {
-            let event = self.0;
-            let mut eexp = writer.eexp_writer(event.statement.index)?;
-            eexp.write(event.timestamp)?
-                .write(event.thread_id)?
-                // Wrap the thread name in the `ThreadName` wrapper to change its serialization.
-                .write(ThreadName(&event.thread_name))?
-                .write(&event.parameters)?;
-            eexp.close()
-        }
-    }
-
     // ===== Random generation of sample data =====
 
+    // Any time we need an integer, we'll generate a random one between 0 and 5,000.
     const INT_PARAMETER_RANGE: Range<i64> = 0..5_000;
     fn generate_int_parameter(rng: &mut StdRng) -> Parameter {
         Parameter::Int(rng.gen_range(INT_PARAMETER_RANGE))
     }
 
+    // Any time we need a string, we'll select one at random from this collection of plural nouns.
     fn generate_string_parameter(rng: &mut StdRng) -> Parameter {
         const WORDS: &[&str] = &["users", "transactions", "accounts", "customers", "waffles"];
         Parameter::String(WORDS.choose(rng).unwrap().to_string())
     }
 
+    // These are the log statements that our fictional program contains.
+    // Each log event will be associated with a randomly selected log statement and its parameters
+    // will be populated using the methods above.
     fn log_statements() -> Vec<LogStatement> {
         use ParameterType::*;
         vec![

src/constants.rs

@@ -124,6 +124,7 @@ pub(crate) mod v1_1 {
 
         pub const ION: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(1);
         pub const ENCODING: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(10);
+        pub const MACRO_TABLE: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(14);
         pub const SYMBOL_TABLE: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(15);
         pub const MODULE: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(16);
         pub const EMPTY_TEXT: SystemSymbol_1_1 = SystemSymbol_1_1::new_unchecked(33);

src/lazy/binary/raw/v1_1/binary_buffer.rs

@@ -1311,7 +1311,7 @@ mod tests {
     ) -> IonResult<()> {
         let mut context = EncodingContext::for_ion_version(IonVersion::v1_1);
         let template_macro =
-            TemplateCompiler::compile_from_source(context.get_ref(), macro_source)?;
+            TemplateCompiler::compile_from_source(context.macro_table(), macro_source)?;
         let macro_address = context
             .macro_table_mut()
             .add_template_macro(template_macro)?;
@@ -1607,28 +1607,17 @@ mod tests {
 
     #[test]
     fn roundtrip_macro_addresses_up_to_20_bits() -> IonResult<()> {
-        use std::fmt::Write;
-
         // This is a large enough value that many macros will be encoded using 20 bits.
         // However, it is not large enough to fully exercise the 20-bit encoding space. To do that,
         // we would need approximately 1 million macros, which takes too much time to execute in a
         // debug build.
         const MAX_TEST_MACRO_ADDRESS: usize = 6_000;
 
-        // Construct an encoding directive that defines this number of macros. Each macro will expand
-        // to its own address.
-        let mut macro_definitions = String::from("$ion::\n(module _\n  (macro_table _\n");
-        for address in MacroTable::FIRST_USER_MACRO_ID..MAX_TEST_MACRO_ADDRESS {
-            writeln!(macro_definitions, "    (macro m{address} () {address})")?;
-        }
-        macro_definitions.push_str("  )\n)\n");
-        let encoding_directive = Element::read_one(macro_definitions)?;
         let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
-        writer.write(&encoding_directive)?;
-
         // Invoke each of the macros we just defined in order.
         for address in MacroTable::FIRST_USER_MACRO_ID..MAX_TEST_MACRO_ADDRESS {
-            writer.eexp_writer(address)?.close()?;
+            let macro_n = writer.compile_macro(format!("(macro m{address} () {address})"))?;
+            writer.eexp_writer(&macro_n)?.close()?;
         }
         let data = writer.close()?;
 

src/lazy/binary/raw/v1_1/struct.rs

@@ -326,41 +326,30 @@ impl<'top> Iterator for RawBinaryStructIterator_1_1<'top> {
 #[cfg(test)]
 mod tests {
     use crate::{
-        v1_1, AnyEncoding, Element, ElementReader, IonResult, MacroTable, Reader, SequenceWriter,
-        StructWriter, ValueWriter, Writer,
+        v1_1, AnyEncoding, Element, ElementReader, IonResult, Reader, SequenceWriter, StructWriter,
+        ValueWriter, Writer,
     };
 
     #[test]
     fn field_value_eexp() -> IonResult<()> {
         let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
-        let encoding_directive = Element::read_one(
-            r#"
-                $ion::
-                (module _
-                    (symbol_table _)
-                    (macro_table
-                        _
-                        (macro greet (name) (.make_string "hello, " (%name)))
-                    )
-                )
-            "#,
-        )?;
-        writer.write(&encoding_directive)?;
-        let macro_id = MacroTable::FIRST_USER_MACRO_ID;
+
+        let greet =
+            writer.compile_macro(r#"(macro greet (name) (.make_string "hello, " (%name)))"#)?;
         let mut struct_writer = writer.struct_writer()?;
 
         let field_writer = struct_writer.field_writer("Waldo");
-        let mut eexp_writer = field_writer.eexp_writer(macro_id)?;
+        let mut eexp_writer = field_writer.eexp_writer(&greet)?;
         eexp_writer.write("Waldo")?;
         eexp_writer.close()?;
 
         let field_writer = struct_writer.field_writer("Winnifred");
-        let mut eexp_writer = field_writer.eexp_writer(macro_id)?;
+        let mut eexp_writer = field_writer.eexp_writer(&greet)?;
         eexp_writer.write("Winnifred")?;
         eexp_writer.close()?;
 
         let field_writer = struct_writer.field_writer("Winston");
-        let mut eexp_writer = field_writer.eexp_writer(macro_id)?;
+        let mut eexp_writer = field_writer.eexp_writer(&greet)?;
         eexp_writer.write("Winston")?;
         eexp_writer.close()?;
 

src/lazy/encoder/binary/v1_0/value_writer.rs

@@ -1,3 +1,4 @@
+use crate::lazy::text::raw::v1_1::reader::MacroIdLike;
 use std::mem;
 
 use bumpalo::collections::Vec as BumpVec;
@@ -16,7 +17,6 @@ use crate::lazy::encoder::private::Sealed;
 use crate::lazy::encoder::value_writer::ValueWriter;
 use crate::lazy::encoder::value_writer::{delegate_value_writer_to_self, AnnotatableWriter};
 use crate::lazy::never::Never;
-use crate::lazy::text::raw::v1_1::reader::MacroIdRef;
 use crate::raw_symbol_ref::AsRawSymbolRef;
 use crate::result::{EncodingError, IonFailure};
 use crate::{Decimal, Int, IonError, IonResult, IonType, RawSymbolRef, SymbolId, Timestamp};
@@ -437,7 +437,7 @@ impl<'value, 'top> ValueWriter for BinaryAnnotatedValueWriter_1_0<'value, 'top>
         BinaryStructWriter_1_0::new(self.allocator, self.output_buffer)
             .with_annotations(self.annotations)
     }
-    fn eexp_writer<'a>(self, _macro_id: impl Into<MacroIdRef<'a>>) -> IonResult<Self::EExpWriter> {
+    fn eexp_writer<'a>(self, _macro_id: impl MacroIdLike<'a>) -> IonResult<Self::EExpWriter> {
         IonResult::encoding_error("binary Ion 1.0 does not support macros")
     }
 }