Add @doc comment DSL (#238)

This allows us to specify doc-comments for auto-generated types to avoid
users having to manually put in comments afterwards which would be
overwritten every re-gen.

There is support for type-level (after the definition), field-level and
variant-level (for group/type-choices).

Author: rooooooooob

docs/docs/comment_dsl.mdx

@@ -161,6 +161,56 @@ Note that as this is at the field-level it must handle the tag as well as the `u
 
 For more examples see `tests/custom_serialization` (used in the `core` and `core_no_wasm` tests) and `tests/custom_serialization_preserve` (used in the `preserve-encodings` test).
 
+## @doc
+
+This can be placed at field-level, struct-level or variant-level to specify a comment to be placed as a rust doc-comment.
+
+```cddl
+docs = [
+  foo: text, ; @doc this is a field-level comment
+  bar: uint, ; @doc bar is a u64
+] ; @doc struct documentation here
+
+docs_groupchoice = [
+  ; @name first @doc comment-about-first
+  0, uint //
+  ; @doc comments about second @name second
+  text
+] ; @doc type-level comment
+```
+
+Will generate:
+```rust
+/// struct documentation here
+#[derive(Clone, Debug)]
+pub struct Docs {
+    /// this is a field-level comment
+    pub foo: String,
+    /// bar is a u64
+    pub bar: u64,
+}
+
+impl Docs {
+    /// * `foo` - this is a field-level comment
+    /// * `bar` - bar is a u64
+    pub fn new(foo: String, bar: u64) -> Self {
+        Self { foo, bar }
+    }
+}
+
+/// type-level comment
+#[derive(Clone, Debug)]
+pub enum DocsGroupchoice {
+    /// comment-about-first
+    First(u64),
+    /// comments about second
+    Second(String),
+}
+```
+
+Due to the comment dsl parsing this doc comment cannot contain the character `@`.
+
+
 ## _CDDL_CODEGEN_EXTERN_TYPE_
 
 While not as a comment, this allows you to compose in hand-written structs into a cddl spec.

src/comment_ast.rs

@@ -15,6 +15,7 @@ pub struct RuleMetadata {
     pub custom_json: bool,
     pub custom_serialize: Option<String>,
     pub custom_deserialize: Option<String>,
+    pub comment: Option<String>,
 }
 
 pub fn merge_metadata(r1: &RuleMetadata, r2: &RuleMetadata) -> RuleMetadata {
@@ -53,6 +54,13 @@ pub fn merge_metadata(r1: &RuleMetadata, r2: &RuleMetadata) -> RuleMetadata {
             (val @ Some(_), _) => val.cloned(),
             (_, val) => val.cloned(),
         },
+        comment: match (r1.comment.as_ref(), r2.comment.as_ref()) {
+            (Some(val1), Some(val2)) => {
+                panic!("Key \"comment\" specified twice: {:?} {:?}", val1, val2)
+            }
+            (val @ Some(_), _) => val.cloned(),
+            (_, val) => val.cloned(),
+        },
     };
     merged.verify();
     merged
@@ -66,6 +74,7 @@ enum ParseResult {
     CustomJson,
     CustomSerialize(String),
     CustomDeserialize(String),
+    Comment(String),
 }
 
 impl RuleMetadata {
@@ -120,6 +129,14 @@ impl RuleMetadata {
                         }
                     }
                 }
+                ParseResult::Comment(comment) => match base.comment.as_ref() {
+                    Some(old) => {
+                        panic!("Key \"comment\" specified twice: {:?} {:?}", old, comment)
+                    }
+                    None => {
+                        base.comment = Some(comment.to_string());
+                    }
+                },
             }
         }
         base.verify();
@@ -188,6 +205,13 @@ fn tag_custom_deserialize(input: &str) -> IResult<&str, ParseResult> {
     ))
 }
 
+fn tag_comment(input: &str) -> IResult<&str, ParseResult> {
+    let (input, _) = tag("@doc")(input)?;
+    let (input, comment) = take_while1(|c| c != '@')(input)?;
+
+    Ok((input, ParseResult::Comment(comment.trim().to_string())))
+}
+
 fn whitespace_then_tag(input: &str) -> IResult<&str, ParseResult> {
     let (input, _) = take_while(char::is_whitespace)(input)?;
     let (input, result) = alt((
@@ -198,6 +222,7 @@ fn whitespace_then_tag(input: &str) -> IResult<&str, ParseResult> {
         tag_custom_json,
         tag_custom_serialize,
         tag_custom_deserialize,
+        tag_comment,
     ))(input)?;
 
     Ok((input, result))
@@ -242,6 +267,7 @@ fn parse_comment_name() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -261,6 +287,7 @@ fn parse_comment_newtype() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -280,6 +307,7 @@ fn parse_comment_newtype_and_name() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -299,6 +327,7 @@ fn parse_comment_newtype_and_name_and_used_as_key() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -318,6 +347,7 @@ fn parse_comment_used_as_key() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -337,6 +367,7 @@ fn parse_comment_newtype_and_name_inverse() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -356,6 +387,7 @@ fn parse_comment_name_noalias() {
                 custom_json: false,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -375,6 +407,7 @@ fn parse_comment_newtype_and_custom_json() {
                 custom_json: true,
                 custom_serialize: None,
                 custom_deserialize: None,
+                comment: None,
             }
         ))
     );
@@ -400,6 +433,7 @@ fn parse_comment_custom_serialize_deserialize() {
                 custom_json: false,
                 custom_serialize: Some("foo".to_string()),
                 custom_deserialize: Some("bar".to_string()),
+                comment: None,
             }
         ))
     );
@@ -409,7 +443,7 @@ fn parse_comment_custom_serialize_deserialize() {
 #[test]
 fn parse_comment_all_except_no_alias() {
     assert_eq!(
-        rule_metadata("@newtype @name baz @custom_serialize foo @custom_deserialize bar @used_as_key @custom_json"),
+        rule_metadata("@newtype @name baz @custom_serialize foo @custom_deserialize bar @used_as_key @custom_json @doc this is a doc comment"),
         Ok((
             "",
             RuleMetadata {
@@ -420,6 +454,7 @@ fn parse_comment_all_except_no_alias() {
                 custom_json: true,
                 custom_serialize: Some("foo".to_string()),
                 custom_deserialize: Some("bar".to_string()),
+                comment: Some("this is a doc comment".to_string()),
             }
         ))
     );

src/generation.rs

@@ -5162,6 +5162,7 @@ fn codegen_struct(
         }
         wasm_new.vis("pub");
         let mut wasm_new_args = Vec::new();
+        let mut wasm_new_comments = Vec::new();
         for field in &record.fields {
             // Fixed values don't need constructors or getters or fields in the rust code
             if !field.rust_type.is_fixed_value() {
@@ -5249,6 +5250,9 @@ fn codegen_struct(
                             .from_wasm_boundary_clone(types, &field.name, false)
                             .into_iter(),
                     ));
+                    if let Some(comment) = &field.rule_metadata.comment {
+                        wasm_new_comments.push(format!("* `{}` - {}", field.name, comment));
+                    }
                     // do we want setters here later for mandatory types covered by new?
                     // getter
                     let mut getter = codegen::Function::new(&field.name);
@@ -5278,6 +5282,12 @@ fn codegen_struct(
                 wasm_new_args.join(", ")
             ));
         }
+        if !wasm_new_comments.is_empty() {
+            wasm_new.doc(wasm_new_comments.join("\n"));
+        }
+        if let Some(doc) = config.doc.as_ref() {
+            wrapper.s.doc(doc);
+        }
         wrapper.s_impl.push_fn(wasm_new);
         wrapper.push(gen_scope, types);
     }
@@ -5287,6 +5297,9 @@ fn codegen_struct(
     // Struct (fields) + constructor
     let (mut native_struct, mut native_impl) = create_base_rust_struct(types, name, false, cli);
     native_struct.vis("pub");
+    if let Some(doc) = config.doc.as_ref() {
+        native_struct.doc(doc);
+    }
     let mut native_new = codegen::Function::new("new");
     let (ctor_ret, ctor_before) = if new_can_fail {
         ("Result<Self, DeserializeError>", "Ok(Self")
@@ -5298,6 +5311,7 @@ fn codegen_struct(
     if new_can_fail {
         native_new_block.after(")");
     }
+    let mut native_new_comments = Vec::new();
     // for clippy we generate a Default impl if new has no args
     let mut new_arg_count = 0;
     for field in &record.fields {
@@ -5313,37 +5327,35 @@ fn codegen_struct(
         }
         // Fixed values only exist in (de)serialization code (outside of preserve-encodings=true)
         if !field.rust_type.is_fixed_value() {
-            if let Some(default_value) = &field.rust_type.config.default {
-                // field
-                native_struct.field(
-                    &format!("pub {}", field.name),
-                    field.rust_type.for_rust_member(types, false, cli),
-                );
+            let mut codegen_field = if let Some(default_value) = &field.rust_type.config.default {
                 // new
                 native_new_block.line(format!(
                     "{}: {},",
                     field.name,
                     default_value.to_primitive_str_assign()
                 ));
+                // field
+                codegen::Field::new(
+                    &format!("pub {}", field.name),
+                    field.rust_type.for_rust_member(types, false, cli),
+                )
             } else if field.optional {
+                // new
+                native_new_block.line(format!("{}: None,", field.name));
                 // field
-                native_struct.field(
+                codegen::Field::new(
                     &format!("pub {}", field.name),
                     format!(
                         "Option<{}>",
                         field.rust_type.for_rust_member(types, false, cli)
                     ),
-                );
-                // new
-                native_new_block.line(format!("{}: None,", field.name));
+                )
             } else {
-                // field
-                native_struct.field(
-                    &format!("pub {}", field.name),
-                    field.rust_type.for_rust_member(types, false, cli),
-                );
                 // new
                 native_new.arg(&field.name, field.rust_type.for_rust_move(types, cli));
+                if let Some(comment) = &field.rule_metadata.comment {
+                    native_new_comments.push(format!("* `{}` - {}", field.name, comment));
+                }
                 new_arg_count += 1;
                 native_new_block.line(format!("{},", field.name));
                 if let Some(bounds) = field.rust_type.config.bounds.as_ref() {
@@ -5363,9 +5375,21 @@ fn codegen_struct(
                         }
                     }
                 }
+                // field
+                codegen::Field::new(
+                    &format!("pub {}", field.name),
+                    field.rust_type.for_rust_member(types, false, cli),
+                )
+            };
+            if let Some(comment) = &field.rule_metadata.comment {
+                codegen_field.doc(comment);
             }
+            native_struct.push_field(codegen_field);
         }
     }
+    if !native_new_comments.is_empty() {
+        native_new.doc(native_new_comments.join("\n"));
+    }
     let len_encoding_var = if cli.preserve_encodings {
         let encoding_name = RustIdent::new(CDDLIdent::new(format!("{name}Encoding")));
         native_struct.field(
@@ -6850,6 +6874,9 @@ fn generate_enum(
     // rust enum containing the data
     let mut e = codegen::Enum::new(name.to_string());
     e.vis("pub");
+    if let Some(doc) = config.doc.as_ref() {
+        e.doc(doc);
+    }
     let mut e_impl = codegen::Impl::new(name.to_string());
     // instead of using create_serialize_impl() and having the length encoded there, we want to make it easier
     // to offer definite length encoding even if we're mixing plain group members and non-plain group members (or mixed length plain ones)
@@ -6960,6 +6987,10 @@ fn generate_enum(
                 }
             }
         }
+        if let Some(doc) = &variant.doc {
+            // we must repurpose annotations since there is no doc support on enum variants
+            v.annotation(format!("/// {doc}"));
+        }
         e.push_variant(v);
         // new (particularly useful if we have encoding variables)
         let mut new_func = codegen::Function::new(&format!("new_{variant_var_name}"));