Merge pull request #300 from kw217/add-docs

Add extra comments

Author: kevinmehall

README.md

@@ -2,7 +2,7 @@
 
 [Documentation](https://docs.rs/peg) | [Release Notes](https://github.com/kevinmehall/rust-peg/releases)
 
-`rust-peg` is a simple yet flexible parser generator that makes it easy to write robust parsers. Based on the [Parsing Expression Grammar](https://en.wikipedia.org/wiki/Parsing_expression_grammar) formalism, it provides a Rust macro that builds a recursive descent parser from a concise definition of the grammar. 
+`rust-peg` is a simple yet flexible parser generator that makes it easy to write robust parsers. Based on the [Parsing Expression Grammar](https://en.wikipedia.org/wiki/Parsing_expression_grammar) formalism, it provides a Rust macro that builds a recursive descent parser from a concise definition of the grammar.
 
 ## Features
 
@@ -60,3 +60,11 @@ pub fn main() {
 [annotate-snippets]: https://crates.io/crates/annotate-snippets
 [codespan-reporting]: https://crates.io/crates/codespan-reporting
 [codemap-diagnostic]: https://crates.io/crates/codemap-diagnostic
+## Development
+
+The `rust-peg` grammar is written in `rust-peg`: `peg-macros/grammar.rustpeg`. To avoid the circular dependency, a precompiled grammar is checked in as `peg-macros/grammar.rs`. To regenerate this, run the `./bootstrap.sh` script.
+
+There is a large test suite which uses [`trybuild`](https://crates.io/crates/trybuild) to support testing for compilation failure errors.
+Use `cargo test` to run the entire suite,
+or `cargo test -- trybuild trybuild=lifetimes.rs` to test just the indicated file.
+Add `--features trace` to trace these tests.

peg-macros/translate.rs

@@ -196,6 +196,8 @@ fn rule_params_list(context: &Context, rule: &Rule) -> Vec<TokenStream> {
     }).collect()
 }
 
+/// Compile a rule to a function for use internal to the grammar.
+/// Returns `RuleResult<T>`.
 fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
     let span = rule.span.resolved_at(Span::mixed_site());
     let name = format_ident!("__parse_{}", rule.name, span=span);
@@ -305,6 +307,8 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
     }
 }
 
+/// Compile a rule into the parsing function which will be exported.
+/// Returns `Result<T, ParseError>`.
 fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
     let span = rule.span.resolved_at(Span::mixed_site());
 
@@ -329,6 +333,10 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
         quote_spanned!{ span => ::peg::Parse::is_eof(__input, __pos) }
     };
 
+    // Parse once. If it succeeds or throws an error, return that.
+    // If it fails, parse again to determine the set of all tokens
+    // that were expected at the failure position.
+
     quote_spanned! { span =>
         #doc
         #visibility fn #name<'input #(, #grammar_lifetime_params)* #(, #ty_params)*>(__input: #input_ty #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr<#(#grammar_lifetime_params),*>>> {

peg-runtime/error.rs

@@ -38,13 +38,13 @@ impl Display for ExpectedSet {
     }
 }
 
-/// An error from a parse failure
+/// A parse failure.
 #[derive(PartialEq, Eq, Debug, Clone)]
 pub struct ParseError<L> {
-    /// The furthest position the parser reached in the input
+    /// The furthest position the parser reached in the input before failing.
     pub location: L,
 
-    /// The set of literals that failed to match at that position
+    /// The set of literals that failed to match at that position.
     pub expected: ExpectedSet,
 }
 
@@ -66,14 +66,23 @@ impl<L: Display + Debug> ::std::error::Error for ParseError<L> {
 
 #[doc(hidden)]
 pub struct ErrorState {
+    /// Furthest failure we've hit so far.
     pub max_err_pos: usize,
+
+    /// Are we inside a lookahead/quiet block? If so, failure is disabled.
+    /// Non-zero => yes, to support nested blocks.
     pub suppress_fail: usize,
+
+    /// Are we reparsing after a failure? If so, compute and store expected set of all alternative expectations
+    /// when we are at offset `max_err_pos`.
     pub reparsing_on_error: bool,
+
+    /// The set of tokens we expected to find when we hit the failure. Updated when `reparsing_on_error`.
     pub expected: ExpectedSet,
 }
 
 impl ErrorState {
-    pub fn new(initial_pos: usize) -> ErrorState {
+    pub fn new(initial_pos: usize) -> Self {
         ErrorState {
             max_err_pos: initial_pos,
             suppress_fail: 0,
@@ -84,6 +93,7 @@ impl ErrorState {
         }
     }
 
+    /// Set up for reparsing to record the details of the furthest failure.
     pub fn reparse_for_error(&mut self) {
         self.suppress_fail = 0;
         self.reparsing_on_error = true;
@@ -96,6 +106,7 @@ impl ErrorState {
         }
     }
 
+    /// Flag a failure.
     #[inline(always)]
     pub fn mark_failure(&mut self, pos: usize, expected: &'static str) -> RuleResult<()> {
         if self.suppress_fail == 0 {

peg-runtime/lib.rs

@@ -10,7 +10,10 @@ pub mod str;
 /// type. The public API of a parser adapts errors to `std::result::Result`.
 #[derive(Clone, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)]
 pub enum RuleResult<T> {
+    /// Success, with final location
     Matched(usize, T),
+
+    /// Failure (furthest failure location is not yet known)
     Failed,
 }
 

src/lib.rs

@@ -90,7 +90,7 @@
 //!     Rust block returns a `Result<T, &str>` instead of a value directly. On
 //!     `Ok(v)`, it matches successfully and returns `v`. On `Err(e)`, the match
 //!     of the entire expression fails and it tries alternatives or reports a
-//!     parse error with the `&str` `e`.
+//!     parse failure with the `&str` `e`.
 //!   * `e1 / e2 / e3` - _Ordered choice:_ try to match `e1`. If the match succeeds, return its
 //!     result, otherwise try `e2`, and so on.
 //!
@@ -141,7 +141,7 @@
 //!
 //! If your input type is a slice of an enum type, a pattern could match an enum variant like
 //! `[Token::Operator('+')]`.
-//! 
+//!
 //! Variables captured by the pattern are accessible in a subsequent action
 //! block: `[Token::Integer(i)] { i }`
 //!
@@ -214,30 +214,30 @@
 //! To allow matching a prefix of the input, add the `#[no_eof]` attribute before `pub rule`.
 //! Take care to not miss a malformed `x` at the last position if the rule ends with a `x()*`
 //! repeat expression.
-//! 
+//!
 //! ## Rule parameters
-//! 
+//!
 //! Rules can be parameterized with types, lifetimes, and values, just like Rust functions.
 //!
 //! In addition to Rust values, rules can also accept PEG expression fragments as arguments by using
 //! `rule<R>` as a parameter type. When calling such a rule, use `<>` around a PEG expression in the
 //! argument list to capture the expression and pass it to the rule.
-//! 
+//!
 //! For example:
-//! 
+//!
 //! ```rust,no_run
 //! # peg::parser!{grammar doc() for str {
 //! rule num_radix(radix: u32) -> u32
 //!   = n:$(['0'..='9']+) {? u32::from_str_radix(n, radix).or(Err("number")) }
-//! 
+//!
 //! rule list<T>(x: rule<T>) -> Vec<T> = "[" v:(x() ** ",") ","? "]" {v}
-//! 
+//!
 //! pub rule octal_list() -> Vec<u32> = list(<num_radix(8)>)
 //! # }}
 //! # fn main() {}
 //! ```
 //!
-//! ## Error reporting
+//! ## Failure reporting
 //!
 //! When a match fails, position information is automatically recorded to report a set of
 //! "expected" tokens that would have allowed the parser to advance further.
@@ -313,8 +313,8 @@
 //! like `expr() "x" / expr() "y" / expr() "z"`, but this could be rewritten to
 //! `expr() ("x" / "y" / "z")` which would be even faster.
 //!
-//! `#[cache_left_rec]` extends the `#[cache]` mechanism with the ability to resolve 
-//! left-recursive rules, which are otherwise an error. 
+//! `#[cache_left_rec]` extends the `#[cache]` mechanism with the ability to resolve
+//! left-recursive rules, which are otherwise an error.
 //!
 //! The `precedence!{}` syntax is another way to handle nested operators and avoid
 //! repeatedly matching an expression rule.