diff --git a/.gitignore b/.gitignore index 6aa10640..4bf6b093 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target/ +*/target/ **/*.rs.bk Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index de64dbb0..d897918e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ - "fluent", - "fluent-syntax" + "fluent-syntax", + "fluent-bundle", + "fluent-cli" ] diff --git a/fluent/CHANGELOG.md b/fluent-bundle/CHANGELOG.md similarity index 100% rename from fluent/CHANGELOG.md rename to fluent-bundle/CHANGELOG.md diff --git a/fluent/Cargo.toml b/fluent-bundle/Cargo.toml similarity index 67% rename from fluent/Cargo.toml rename to fluent-bundle/Cargo.toml index d9102f09..66c40972 100644 --- a/fluent/Cargo.toml +++ b/fluent-bundle/Cargo.toml @@ -1,10 +1,11 @@ [package] -name = "fluent" +name = "fluent-bundle" description = """ -A localization library designed to unleash the entire expressive power of +A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" +edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " @@ -17,9 +18,9 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] -clap = "2.32" fluent-locale = "^0.4.1" -fluent-syntax = "0.1.1" -failure = "0.1" -failure_derive = "0.1" -intl_pluralrules = "1.0" +fluent-syntax = { path = "../fluent-syntax" } +failure = "^0.1" +failure_derive = "^0.1" +intl_pluralrules = "^1.0" +rental = "^0.5.2" diff --git a/fluent/README.md b/fluent-bundle/README.md similarity index 98% rename from fluent/README.md rename to fluent-bundle/README.md index b459ee37..200f7f5d 100644 --- a/fluent/README.md +++ b/fluent-bundle/README.md @@ -22,7 +22,7 @@ Usage ----- ```rust -use fluent::bundle::FluentBundle; +use fluent_bundle::FluentBundle; fn main() { let mut bundle = FluentBundle::new(&["en-US"]); diff --git a/fluent/benches/lib.rs b/fluent-bundle/benches/lib.rs similarity index 57% rename from fluent/benches/lib.rs rename to fluent-bundle/benches/lib.rs index a7abdb30..d7512af8 100644 --- a/fluent/benches/lib.rs +++ b/fluent-bundle/benches/lib.rs @@ -1,43 +1,46 @@ #![feature(test)] -extern crate fluent; -extern crate fluent_syntax; extern crate test; -use fluent::bundle::FluentBundle; -use fluent_syntax::{ast, parser::parse}; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use fluent_syntax::ast; use std::fs::File; use std::io; use std::io::Read; use test::Bencher; fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); + let mut f = File::open(path)?; let mut s = String::new(); - try!(f.read_to_string(&mut s)); + f.read_to_string(&mut s)?; Ok(s) } #[bench] fn bench_simple_format(b: &mut Bencher) { let source = read_file("./benches/simple.ftl").expect("Couldn't load file"); - let resource = parse(&source).unwrap(); + let res = FluentResource::try_new(source).expect("Couldn't parse an FTL source"); let mut ids = Vec::new(); - for entry in resource.body { + for entry in &res.ast().body { match entry { - ast::Entry::Message(ast::Message { id, .. }) => ids.push(id.name), + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { + ids.push(id.name) + } _ => continue, }; } let mut bundle = FluentBundle::new(&["x-testing"]); - bundle.add_messages(&source).unwrap(); + bundle + .add_resource(&res) + .expect("Couldn't add FluentResource to the FluentBundle"); b.iter(|| { for id in &ids { - bundle.format(id.as_str(), None); + bundle.format(id, None); } }); } @@ -45,19 +48,23 @@ fn bench_simple_format(b: &mut Bencher) { #[bench] fn bench_menubar_format(b: &mut Bencher) { let source = read_file("./benches/menubar.ftl").expect("Couldn't load file"); - let resource = parse(&source).unwrap(); + let res = FluentResource::try_new(source).expect("Couldn't parse an FTL source"); let mut ids = Vec::new(); - for entry in resource.body { + for entry in &res.ast().body { match entry { - ast::Entry::Message(ast::Message { id, .. }) => ids.push(id.name), + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { + ids.push(id.name) + } _ => continue, }; } let mut bundle = FluentBundle::new(&["x-testing"]); - bundle.add_messages(&source).unwrap(); + bundle + .add_resource(&res) + .expect("Couldn't add FluentResource to the FluentBundle"); b.iter(|| { for id in &ids { @@ -66,7 +73,7 @@ fn bench_menubar_format(b: &mut Bencher) { // widgets may only expect attributes and they shouldn't be forced to display a value. // Here however it doesn't matter because we know for certain that the message for `id` // exists. - bundle.format_message(id.as_str(), None); + bundle.compound(id, None); } }); } diff --git a/fluent/benches/menubar.ftl b/fluent-bundle/benches/menubar.ftl similarity index 100% rename from fluent/benches/menubar.ftl rename to fluent-bundle/benches/menubar.ftl diff --git a/fluent/benches/simple.ftl b/fluent-bundle/benches/simple.ftl similarity index 100% rename from fluent/benches/simple.ftl rename to fluent-bundle/benches/simple.ftl diff --git a/fluent/examples/README.md b/fluent-bundle/examples/README.md similarity index 100% rename from fluent/examples/README.md rename to fluent-bundle/examples/README.md diff --git a/fluent/examples/external_arguments.rs b/fluent-bundle/examples/external_arguments.rs similarity index 78% rename from fluent/examples/external_arguments.rs rename to fluent-bundle/examples/external_arguments.rs index eda4e350..8e09e91d 100644 --- a/fluent/examples/external_arguments.rs +++ b/fluent-bundle/examples/external_arguments.rs @@ -1,14 +1,11 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["en"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " hello-world = Hello { $name } ref = The previous message says { hello-world } unread-emails = @@ -16,9 +13,12 @@ unread-emails = [one] You have { $emailCount } unread email *[other] You have { $emailCount } unread emails } -", - ) - .unwrap(); +" + .to_owned(), + ) + .unwrap(); + let mut bundle = FluentBundle::new(&["en"]); + bundle.add_resource(&res).unwrap(); let mut args = HashMap::new(); args.insert("name", FluentValue::from("John")); diff --git a/fluent/examples/functions.rs b/fluent-bundle/examples/functions.rs similarity index 70% rename from fluent/examples/functions.rs rename to fluent-bundle/examples/functions.rs index e0bf1db3..ba50dda6 100644 --- a/fluent/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -1,9 +1,18 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use fluent_bundle::types::FluentValue; fn main() { + // We define the resources here so that they outlive + // the bundle. + let res1 = FluentResource::try_new("hello-world = Hey there! { HELLO() }".to_owned()).unwrap(); + let res2 = + FluentResource::try_new("meaning-of-life = { MEANING_OF_LIFE(42) }".to_owned()).unwrap(); + let res3 = FluentResource::try_new( + "all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }".to_owned(), + ) + .unwrap(); + let mut bundle = FluentBundle::new(&["en-US"]); // Test for a simple function that returns a string @@ -40,15 +49,9 @@ fn main() { }) .unwrap(); - bundle - .add_messages("hello-world = Hey there! { HELLO() }") - .unwrap(); - bundle - .add_messages("meaning-of-life = { MEANING_OF_LIFE(42) }") - .unwrap(); - bundle - .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") - .unwrap(); + bundle.add_resource(&res1).unwrap(); + bundle.add_resource(&res2).unwrap(); + bundle.add_resource(&res3).unwrap(); let value = bundle.format("hello-world", None); assert_eq!( diff --git a/fluent-bundle/examples/hello.rs b/fluent-bundle/examples/hello.rs new file mode 100644 index 00000000..c800a571 --- /dev/null +++ b/fluent-bundle/examples/hello.rs @@ -0,0 +1,10 @@ +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; + +fn main() { + let res = FluentResource::try_new("hello-world = Hello, world!".to_owned()).unwrap(); + let mut bundle = FluentBundle::new(&["en-US"]); + bundle.add_resource(&res).unwrap(); + let (value, _) = bundle.format("hello-world", None).unwrap(); + assert_eq!(&value, "Hello, world!"); +} diff --git a/fluent/examples/message_reference.rs b/fluent-bundle/examples/message_reference.rs similarity index 63% rename from fluent/examples/message_reference.rs rename to fluent-bundle/examples/message_reference.rs index 21e33ac9..6b0438ab 100644 --- a/fluent/examples/message_reference.rs +++ b/fluent-bundle/examples/message_reference.rs @@ -1,18 +1,19 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " foo = Foo foobar = { foo } Bar bazbar = { baz } Bar -", - ) - .unwrap(); +" + .to_owned(), + ) + .unwrap(); + + let mut bundle = FluentBundle::new(&["x-testing"]); + bundle.add_resource(&res).unwrap(); match bundle.format("foobar", None) { Some((value, _)) => println!("{}", value), diff --git a/fluent/examples/resources/en-US/simple.ftl b/fluent-bundle/examples/resources/en-US/simple.ftl similarity index 100% rename from fluent/examples/resources/en-US/simple.ftl rename to fluent-bundle/examples/resources/en-US/simple.ftl diff --git a/fluent/examples/resources/pl/simple.ftl b/fluent-bundle/examples/resources/pl/simple.ftl similarity index 100% rename from fluent/examples/resources/pl/simple.ftl rename to fluent-bundle/examples/resources/pl/simple.ftl diff --git a/fluent/examples/selector.rs b/fluent-bundle/examples/selector.rs similarity index 70% rename from fluent/examples/selector.rs rename to fluent-bundle/examples/selector.rs index a747f97b..c2e83c58 100644 --- a/fluent/examples/selector.rs +++ b/fluent-bundle/examples/selector.rs @@ -1,14 +1,11 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " hello-world = Hello { *[one] World [two] Moon @@ -18,9 +15,12 @@ hello-world2 = Hello { $name -> *[world] World [moon] Moon } -", - ) - .unwrap(); + " + .to_owned(), + ) + .unwrap(); + let mut bundle = FluentBundle::new(&["x-testing"]); + bundle.add_resource(&res).unwrap(); match bundle.format("hello-world", None) { Some((value, _)) => println!("{}", value), diff --git a/fluent/examples/simple.rs b/fluent-bundle/examples/simple-app.rs similarity index 84% rename from fluent/examples/simple.rs rename to fluent-bundle/examples/simple-app.rs index ff3b597b..79f2803f 100644 --- a/fluent/examples/simple.rs +++ b/fluent-bundle/examples/simple-app.rs @@ -17,11 +17,9 @@ //! //! If the second argument is omitted, `en-US` locale is used as the //! default one. -extern crate fluent; -extern crate fluent_locale; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use fluent_bundle::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; @@ -37,9 +35,9 @@ use std::str::FromStr; /// The resource files are stored in /// `./examples/resources/{locale}` directory. fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); + let mut f = File::open(path)?; let mut s = String::new(); - try!(f.read_to_string(&mut s)); + f.read_to_string(&mut s)?; Ok(s) } @@ -95,7 +93,10 @@ fn main() { // 1. Get the command line arguments. let args: Vec = env::args().collect(); - // 2. If the argument length is more than 1, + // 2. Allocate resources. + let mut resources: Vec = vec![]; + + // 3. If the argument length is more than 1, // take the second argument as a comma-separated // list of requested locales. // @@ -104,38 +105,41 @@ fn main() { .get(2) .map_or(vec!["en-US"], |arg| arg.split(",").collect()); - // 3. Negotiate it against the avialable ones + // 4. Negotiate it against the avialable ones let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); - // 4. Create a new Fluent FluentBundle using the + // 5. Create a new Fluent FluentBundle using the // resolved locales. let mut bundle = FluentBundle::new(&locales); - // 5. Load the localization resource + // 6. Load the localization resource for path in L10N_RESOURCES { let full_path = format!( "./examples/resources/{locale}/{path}", locale = locales[0], path = path ); - let res = read_file(&full_path).unwrap(); - // 5.1 Insert the loaded resource into the - // Fluent FluentBundle. - bundle.add_messages(&res).unwrap(); + let source = read_file(&full_path).unwrap(); + let resource = FluentResource::try_new(source).unwrap(); + resources.push(resource); + } + + for res in &resources { + bundle.add_resource(res).unwrap(); } - // 6. Check if the input is provided. + // 7. Check if the input is provided. match args.get(1) { Some(input) => { - // 6.1. Cast it to a number. + // 7.1. Cast it to a number. match isize::from_str(&input) { Ok(i) => { - // 6.2. Construct a map of arguments + // 7.2. Construct a map of arguments // to format the message. let mut args = HashMap::new(); args.insert("input", FluentValue::from(i)); args.insert("value", FluentValue::from(collatz(i))); - // 6.3. Format the message. + // 7.3. Format the message. println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); } Err(err) => { diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs new file mode 100644 index 00000000..ebf0b8df --- /dev/null +++ b/fluent-bundle/src/bundle.rs @@ -0,0 +1,451 @@ +//! `FluentBundle` is a collection of localization messages in Fluent. +//! +//! It stores a list of messages in a single locale which can reference one another, use the same +//! internationalization formatters, functions, environmental variables and are expected to be used +//! together. + +use std::cell::RefCell; +use std::collections::hash_map::{Entry as HashEntry, HashMap}; + +use super::entry::{Entry, GetEntry}; +pub use super::errors::FluentError; +use super::resolve::{Env, ResolveValue}; +use super::resource::FluentResource; +use super::types::FluentValue; +use fluent_locale::{negotiate_languages, NegotiationStrategy}; +use fluent_syntax::ast; +use intl_pluralrules::{IntlPluralRules, PluralRuleType}; + +#[derive(Debug, PartialEq)] +pub struct Message { + pub value: Option, + pub attributes: HashMap, +} + +/// A collection of localization messages for a single locale, which are meant +/// to be used together in a single view, widget or any other UI abstraction. +/// +/// # Examples +/// +/// ``` +/// use fluent_bundle::bundle::FluentBundle; +/// use fluent_bundle::resource::FluentResource; +/// use fluent_bundle::types::FluentValue; +/// use std::collections::HashMap; +/// +/// let resource = FluentResource::try_new("intro = Welcome, { $name }.".to_string()).unwrap(); +/// let mut bundle = FluentBundle::new(&["en-US"]); +/// bundle.add_resource(&resource); +/// +/// let mut args = HashMap::new(); +/// args.insert("name", FluentValue::from("Rustacean")); +/// +/// let value = bundle.format("intro", Some(&args)); +/// assert_eq!(value, Some(("Welcome, Rustacean.".to_string(), vec![]))); +/// +/// ``` +/// +/// # `FluentBundle` Life Cycle +/// +/// To create a bundle, call [`FluentBundle::new`] with a locale list that represents the best +/// possible fallback chain for a given locale. The simplest case is a one-locale list. +/// +/// Next, call [`add_resource`] one or more times, supplying translations in the FTL syntax. The +/// `FluentBundle` instance is now ready to be used for localization. +/// +/// To format a translation, call [`format`] with the path of a message or attribute in order to +/// retrieve the translated string. Alternately, [`compound`] provides a convenient way of +/// formatting all attributes of a message at once. +/// +/// The result of `format` is an [`Option`] wrapping a `(String, Vec)`. On success, +/// the string is a formatted value that should be displayed in the UI. It is +/// recommended to treat the result as opaque from the perspective of the program and use it only +/// to display localized messages. Do not examine it or alter in any way before displaying. This +/// is a general good practice as far as all internationalization operations are concerned. +/// +/// +/// # Locale Fallback Chain +/// +/// `FluentBundle` stores messages in a single locale, but keeps a locale fallback chain for the +/// purpose of language negotiation with i18n formatters. For instance, if date and time formatting +/// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain +/// to negotiate a sensible fallback for date and time formatting. +/// +/// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource +/// [`FluentBundle::new`]: ./struct.FluentBundle.html#method.new +/// [`fluent::bundle::Message`]: ./struct.FluentBundle.html#method.new +/// [`format`]: ./struct.FluentBundle.html#method.format +/// [`compound`]: ./struct.FluentBundle.html#method.compound +/// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource +/// [`Option`]: http://doc.rust-lang.org/std/option/enum.Option.html +pub struct FluentBundle<'bundle> { + pub locales: Vec, + pub entries: HashMap>, + pub plural_rules: IntlPluralRules, +} + +impl<'bundle> FluentBundle<'bundle> { + /// Constructs a FluentBundle. `locales` is the fallback chain of locales + /// to use for formatters like date and time. `locales` does not influence + /// message selection. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// ``` + /// + /// # Errors + /// + /// This will panic if no formatters can be found for the locales. + pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { + let locales = locales.iter().map(|s| s.to_string()).collect::>(); + let pr_locale = negotiate_languages( + &locales, + IntlPluralRules::get_locales(PluralRuleType::CARDINAL), + Some("en"), + &NegotiationStrategy::Lookup, + )[0] + .to_owned(); + + let pr = IntlPluralRules::create(&pr_locale, PluralRuleType::CARDINAL).unwrap(); + FluentBundle { + locales, + entries: HashMap::new(), + plural_rules: pr, + } + } + + /// Returns true if this bundle contains a message with the given id. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// + /// let resource = FluentResource::try_new("hello = Hi!".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); + /// assert_eq!(true, bundle.has_message("hello")); + /// + /// ``` + pub fn has_message(&self, id: &str) -> bool { + self.entries.get_message(id).is_some() + } + + /// Makes the provided rust function available to messages with the name `id`. See + /// the [FTL syntax guide] to learn how these are used in messages. + /// + /// FTL functions accept both positional and named args. The rust function you + /// provide therefore has two parameters: a slice of values for the positional + /// args, and a HashMap of values for named args. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; + /// + /// let resource = FluentResource::try_new("length = { STRLEN(\"12345\") }".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); + /// + /// // Register a fn that maps from string to string length + /// bundle.add_function("STRLEN", |positional, _named| match positional { + /// [Some(FluentValue::String(str))] => Some(FluentValue::Number(str.len().to_string())), + /// _ => None, + /// }).unwrap(); + /// + /// let (value, _) = bundle.format("length", None).unwrap(); + /// assert_eq!(&value, "5"); + /// ``` + /// + /// [FTL syntax guide]: https://projectfluent.org/fluent/guide/functions.html + pub fn add_function(&mut self, id: &str, func: F) -> Result<(), FluentError> + where + F: 'bundle + + Fn(&[Option], &HashMap) -> Option + + Sync + + Send, + { + match self.entries.entry(id.to_owned()) { + HashEntry::Vacant(entry) => { + entry.insert(Entry::Function(Box::new(func))); + Ok(()) + } + HashEntry::Occupied(_) => Err(FluentError::Overriding { + kind: "function", + id: id.to_owned(), + }), + } + } + + /// Adds the message or messages, in [FTL syntax], to the bundle, returning an + /// empty [`Result`] on success. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// + /// let resource = FluentResource::try_new(" + /// hello = Hi! + /// goodbye = Bye! + /// ".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); + /// assert_eq!(true, bundle.has_message("hello")); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result`]: https://doc.rust-lang.org/std/result/enum.Result.html + pub fn add_resource(&mut self, res: &'bundle FluentResource) -> Result<(), Vec> { + let mut errors = vec![]; + + for entry in &res.ast().body { + let id = match entry { + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { ref id, .. })) + | ast::ResourceEntry::Entry(ast::Entry::Term(ast::Term { ref id, .. })) => id.name, + _ => continue, + }; + + let (entry, kind) = match entry { + ast::ResourceEntry::Entry(ast::Entry::Message(message)) => { + (Entry::Message(message), "message") + } + ast::ResourceEntry::Entry(ast::Entry::Term(term)) => (Entry::Term(term), "term"), + _ => continue, + }; + + match self.entries.entry(id.to_string()) { + HashEntry::Vacant(empty) => { + empty.insert(entry); + } + HashEntry::Occupied(_) => { + errors.push(FluentError::Overriding { + kind, + id: id.to_string(), + }); + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + /// Formats the message value identified by `path` using `args` to + /// provide variables. `path` is either a message id ("hello"), or + /// message id plus attribute ("hello.tooltip"). + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; + /// use std::collections::HashMap; + /// + /// let resource = FluentResource::try_new("intro = Welcome, { $name }.".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); + /// + /// let mut args = HashMap::new(); + /// args.insert("name", FluentValue::from("Rustacean")); + /// + /// let value = bundle.format("intro", Some(&args)); + /// assert_eq!(value, Some(("Welcome, Rustacean.".to_string(), vec![]))); + /// + /// ``` + /// + /// An example with attributes and no args: + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// + /// let resource = FluentResource::try_new(" + /// hello = + /// .title = Hi! + /// .tooltip = This says 'Hi!' + /// ".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); + /// + /// let value = bundle.format("hello.title", None); + /// assert_eq!(value, Some(("Hi!".to_string(), vec![]))); + /// ``` + /// + /// # Errors + /// + /// For some cases where `format` can't find a message it will return `None`. + /// + /// In all other cases `format` returns a string even if it + /// encountered errors. Generally, during partial errors `format` will + /// use `'___'` to replace parts of the formatted message that it could + /// not successfuly build. For more fundamental errors `format` will return + /// the path itself as the translation. + /// + /// The second term of the tuple will contain any extra error information + /// gathered during formatting. A caller may safely ignore the extra errors + /// if the fallback formatting policies are acceptable. + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// + /// // Create a message with bad cyclic reference + /// let mut res = FluentResource::try_new("foo = a { foo } b".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&res); + /// + /// // The result falls back to "___" + /// let value = bundle.format("foo", None); + /// assert_eq!(value, Some(("___".to_string(), vec![]))); + /// ``` + pub fn format( + &self, + path: &str, + args: Option<&HashMap<&str, FluentValue>>, + ) -> Option<(String, Vec)> { + let env = Env { + bundle: self, + args, + travelled: RefCell::new(Vec::new()), + }; + + let mut errors = vec![]; + + if let Some(ptr_pos) = path.find('.') { + let message_id = &path[..ptr_pos]; + let message = self.entries.get_message(message_id)?; + let attr_name = &path[(ptr_pos + 1)..]; + for attribute in message.attributes.iter() { + if attribute.id.name == attr_name { + match attribute.to_value(&env) { + Ok(val) => { + return Some((val.format(self), errors)); + } + Err(err) => { + errors.push(FluentError::ResolverError(err)); + // XXX: In the future we'll want to get the partial + // value out of resolver and return it here. + // We also expect to get a Vec or errors out of resolver. + return Some((path.to_string(), errors)); + } + } + } + } + } else { + let message_id = path; + let message = self.entries.get_message(message_id)?; + match message.to_value(&env) { + Ok(val) => { + let s = val.format(self); + return Some((s, errors)); + } + Err(err) => { + errors.push(FluentError::ResolverError(err)); + return Some((message_id.to_string(), errors)); + } + } + } + + None + } + + /// Formats both the message value and attributes identified by `message_id` + /// using `args` to provide variables. This is useful for cases where a UI + /// element requires multiple related text fields, such as a button that has + /// both display text and assistive text. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; + /// use std::collections::HashMap; + /// + /// let mut res = FluentResource::try_new(" + /// login-input = Predefined value + /// .placeholder = example@email.com + /// .aria-label = Login input value + /// .title = Type your login email".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&res); + /// + /// let (message, _) = bundle.compound("login-input", None).unwrap(); + /// assert_eq!(message.value, Some("Predefined value".to_string())); + /// assert_eq!(message.attributes.get("title"), Some(&"Type your login email".to_string())); + /// ``` + /// + /// # Errors + /// + /// For some cases where `compound` can't find a message it will return `None`. + /// + /// In all other cases `compound` returns a message even if it + /// encountered errors. Generally, during partial errors `compound` will + /// use `'___'` to replace parts of the formatted message that it could + /// not successfuly build. For more fundamental errors `compound` will return + /// the path itself as the translation. + /// + /// The second term of the tuple will contain any extra error information + /// gathered during formatting. A caller may safely ignore the extra errors + /// if the fallback formatting policies are acceptable. + pub fn compound( + &self, + message_id: &str, + args: Option<&HashMap<&str, FluentValue>>, + ) -> Option<(Message, Vec)> { + let mut errors = vec![]; + + let env = Env { + bundle: self, + args, + travelled: RefCell::new(Vec::new()), + }; + let message = self.entries.get_message(message_id)?; + + let value = match message.to_value(&env) { + Ok(value) => Some(value.format(self)), + Err(err) => { + errors.push(FluentError::ResolverError(err)); + None + } + }; + + let mut attributes = HashMap::new(); + + for attr in message.attributes.iter() { + match attr.to_value(&env) { + Ok(value) => { + let val = value.format(self); + attributes.insert(attr.id.name.to_owned(), val); + } + Err(err) => { + errors.push(FluentError::ResolverError(err)); + } + } + } + + Some((Message { value, attributes }, errors)) + } +} diff --git a/fluent/src/entry.rs b/fluent-bundle/src/entry.rs similarity index 86% rename from fluent/src/entry.rs rename to fluent-bundle/src/entry.rs index 467599c9..b378dd1d 100644 --- a/fluent/src/entry.rs +++ b/fluent-bundle/src/entry.rs @@ -12,8 +12,8 @@ type FluentFunction<'bundle> = Box< >; pub enum Entry<'bundle> { - Message(ast::Message), - Term(ast::Term), + Message(&'bundle ast::Message<'bundle>), + Term(&'bundle ast::Term<'bundle>), Function(FluentFunction<'bundle>), } @@ -25,14 +25,14 @@ pub trait GetEntry<'bundle> { impl<'bundle> GetEntry<'bundle> for HashMap> { fn get_term(&self, id: &str) -> Option<&ast::Term> { - self.get(id).and_then(|entry| match entry { + self.get(id).and_then(|entry| match *entry { Entry::Term(term) => Some(term), _ => None, }) } fn get_message(&self, id: &str) -> Option<&ast::Message> { - self.get(id).and_then(|entry| match entry { + self.get(id).and_then(|entry| match *entry { Entry::Message(message) => Some(message), _ => None, }) diff --git a/fluent/src/errors.rs b/fluent-bundle/src/errors.rs similarity index 92% rename from fluent/src/errors.rs rename to fluent-bundle/src/errors.rs index 66fafa8d..9b7cf418 100644 --- a/fluent/src/errors.rs +++ b/fluent-bundle/src/errors.rs @@ -1,5 +1,5 @@ use super::resolve::ResolverError; -use fluent_syntax::parser::errors::ParserError; +use fluent_syntax::parser::ParserError; #[derive(Debug, Fail, PartialEq)] pub enum FluentError { diff --git a/fluent/src/lib.rs b/fluent-bundle/src/lib.rs similarity index 77% rename from fluent/src/lib.rs rename to fluent-bundle/src/lib.rs index 019069a3..c36d5f58 100644 --- a/fluent/src/lib.rs +++ b/fluent-bundle/src/lib.rs @@ -11,17 +11,19 @@ //! # Example //! //! ``` -//! use fluent::bundle::FluentBundle; -//! use fluent::types::FluentValue; +//! use fluent_bundle::bundle::FluentBundle; +//! use fluent_bundle::types::FluentValue; +//! use fluent_bundle::resource::FluentResource; //! use std::collections::HashMap; //! -//! let mut bundle = FluentBundle::new(&["en-US"]); -//! bundle.add_messages( -//! " +//! let res = FluentResource::try_new(" //! hello-world = Hello, world! //! intro = Welcome, { $name }. -//! " -//! ); +//! ".to_owned()).expect("Failed to parse FTL."); +//! +//! let mut bundle = FluentBundle::new(&["en-US"]); +//! +//! bundle.add_resource(&res).expect("Failed to add FluentResource to Bundle."); //! //! let value = bundle.format("hello-world", None); //! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); @@ -33,12 +35,10 @@ //! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); //! ``` -extern crate failure; +#[macro_use] +extern crate rental; #[macro_use] extern crate failure_derive; -extern crate fluent_locale; -extern crate fluent_syntax; -extern crate intl_pluralrules; pub mod bundle; pub mod entry; diff --git a/fluent/src/resolve.rs b/fluent-bundle/src/resolve.rs similarity index 50% rename from fluent/src/resolve.rs rename to fluent-bundle/src/resolve.rs index e093cfc8..a04e4806 100644 --- a/fluent/src/resolve.rs +++ b/fluent-bundle/src/resolve.rs @@ -62,7 +62,7 @@ pub trait ResolveValue { fn to_value(&self, env: &Env) -> Result; } -impl ResolveValue for ast::Message { +impl<'source> ResolveValue for ast::Message<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || { self.value @@ -73,19 +73,31 @@ impl ResolveValue for ast::Message { } } -impl ResolveValue for ast::Term { +impl<'source> ResolveValue for ast::Term<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl ResolveValue for ast::Attribute { +impl<'source> ResolveValue for ast::Attribute<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl ResolveValue for ast::Pattern { +impl<'source> ResolveValue for ast::Value<'source> { + fn to_value(&self, env: &Env) -> Result { + match self { + ast::Value::Pattern(p) => p.to_value(env), + ast::Value::VariantList { variants } => select_default(variants) + .ok_or(ResolverError::None)? + .value + .to_value(env), + } + } +} + +impl<'source> ResolveValue for ast::Pattern<'source> { fn to_value(&self, env: &Env) -> Result { let mut string = String::with_capacity(128); for elem in &self.elements { @@ -107,79 +119,44 @@ impl ResolveValue for ast::Pattern { } } -impl ResolveValue for ast::PatternElement { +impl<'source> ResolveValue for ast::PatternElement<'source> { fn to_value(&self, env: &Env) -> Result { match self { - ast::PatternElement::TextElement(s) => Ok(FluentValue::from(s.clone())), + ast::PatternElement::TextElement(s) => Ok(FluentValue::from(*s)), ast::PatternElement::Placeable(p) => p.to_value(env), } } } -impl ResolveValue for ast::Number { - fn to_value(&self, _env: &Env) -> Result { - FluentValue::as_number(&self.value).map_err(|_| ResolverError::None) - } -} - -impl ResolveValue for ast::VariantName { +impl<'source> ResolveValue for ast::VariantKey<'source> { fn to_value(&self, _env: &Env) -> Result { - Ok(FluentValue::from(self.name.clone())) + match self { + ast::VariantKey::Identifier { name } => Ok(FluentValue::from(*name)), + ast::VariantKey::NumberLiteral { value } => { + FluentValue::as_number(value).map_err(|_| ResolverError::None) + } + } } } -impl ResolveValue for ast::Expression { +impl<'source> ResolveValue for ast::Expression<'source> { fn to_value(&self, env: &Env) -> Result { match self { - ast::Expression::StringExpression { value } => Ok(FluentValue::from(value.clone())), - ast::Expression::NumberExpression { value } => value.to_value(env), - ast::Expression::MessageReference { ref id } if id.name.starts_with('-') => env - .bundle - .entries - .get_term(&id.name) - .ok_or(ResolverError::None)? - .to_value(env), - ast::Expression::MessageReference { ref id } => env - .bundle - .entries - .get_message(&id.name) - .ok_or(ResolverError::None)? - .to_value(env), - ast::Expression::ExternalArgument { ref id } => env - .args - .and_then(|args| args.get(&id.name.as_ref())) - .cloned() - .ok_or(ResolverError::None), - ast::Expression::SelectExpression { - expression: None, - variants, - } => select_default(variants) - .ok_or(ResolverError::None)? - .value - .to_value(env), - ast::Expression::SelectExpression { - expression, - variants, - } => { - let selector = expression - .as_ref() - .ok_or(ResolverError::None)? - .to_value(env); - - if let Ok(ref selector) = selector { + ast::Expression::InlineExpression(exp) => exp.to_value(env), + ast::Expression::SelectExpression { selector, variants } => { + if let Ok(ref selector) = selector.to_value(env) { for variant in variants { match variant.key { - ast::VarKey::VariantName(ref symbol) => { - let key = FluentValue::from(symbol.name.clone()); + ast::VariantKey::Identifier { name } => { + let key = FluentValue::from(name); if key.matches(env.bundle, selector) { return variant.value.to_value(env); } } - ast::VarKey::Number(ref number) => { - if let Ok(key) = number.to_value(env) { - if key.matches(env.bundle, selector) { - return variant.value.to_value(env); - } + ast::VariantKey::NumberLiteral { value } => { + let key = FluentValue::as_number(value).unwrap(); + if key.matches(env.bundle, selector) { + return variant.value.to_value(env); } } } @@ -191,100 +168,130 @@ impl ResolveValue for ast::Expression { .value .to_value(env) } - ast::Expression::AttributeExpression { id, name } => { - let attributes = if id.name.starts_with('-') { - env.bundle + } + } +} + +impl<'source> ResolveValue for ast::InlineExpression<'source> { + fn to_value(&self, env: &Env) -> Result { + match self { + ast::InlineExpression::StringLiteral { raw } => { + // XXX: We need to decode the raw into unicode here. + Ok(FluentValue::from(*raw)) + } + ast::InlineExpression::NumberLiteral { value } => { + Ok(FluentValue::as_number(*value).unwrap()) + } + ast::InlineExpression::VariableReference { id } => env + .args + .and_then(|args| args.get(&id.name)) + .cloned() + .ok_or(ResolverError::None), + ast::InlineExpression::CallExpression { + ref callee, + ref positional, + ref named, + } => { + let mut resolved_positional_args = Vec::new(); + let mut resolved_named_args = HashMap::new(); + + for expression in positional { + resolved_positional_args.push(expression.to_value(env).ok()); + } + + for arg in named { + resolved_named_args + .insert(arg.name.name.to_string(), arg.value.to_value(env).unwrap()); + } + + let func = match **callee { + ast::InlineExpression::FunctionReference { ref id } => { + env.bundle.entries.get_function(id.name) + } + _ => panic!(), + }; + + func.ok_or(ResolverError::None).and_then(|func| { + func(resolved_positional_args.as_slice(), &resolved_named_args) + .ok_or(ResolverError::None) + }) + } + ast::InlineExpression::AttributeExpression { reference, name } => { + let attributes: &Vec = match reference.as_ref() { + ast::InlineExpression::MessageReference { ref id } => env + .bundle .entries - .get_term(&id.name) + .get_message(&id.name) .ok_or(ResolverError::None)? .attributes - .as_ref() - } else { - env.bundle + .as_ref(), + ast::InlineExpression::TermReference { ref id } => env + .bundle .entries - .get_message(&id.name) + .get_term(&id.name) .ok_or(ResolverError::None)? .attributes - .as_ref() + .as_ref(), + _ => unimplemented!(), }; - if let Some(attributes) = attributes { - for attribute in attributes { - if attribute.id.name == name.name { - return attribute.to_value(env); - } + for attribute in attributes { + if attribute.id.name == name.name { + return attribute.to_value(env); } } Err(ResolverError::None) } - ast::Expression::VariantExpression { id, key } if id.name.starts_with('-') => { - let term = env - .bundle - .entries - .get_term(&id.name) - .ok_or(ResolverError::None)?; - let variants = match term.value.elements.as_slice() { - [ast::PatternElement::Placeable(ast::Expression::SelectExpression { - expression: None, - ref variants, - })] => variants, - _ => return term.value.to_value(env), - }; - - for variant in variants { - if variant.key == *key { - return variant.value.to_value(env); - } - } - - select_default(variants) - .ok_or(ResolverError::None)? - .value - .to_value(env) - } - ast::Expression::CallExpression { - ref callee, - ref args, - } => { - let resolved_unnamed_args = &mut Vec::new(); - let resolved_named_args = &mut HashMap::new(); - - for arg in args { - env.scope(|| match arg { - ast::Argument::Expression(ref expression) => { - resolved_unnamed_args.push(expression.to_value(env).ok()); - } - ast::Argument::NamedArgument { ref name, ref val } => { - let mut fluent_val: FluentValue; + ast::InlineExpression::VariantExpression { reference, key } => { + if let ast::InlineExpression::TermReference { ref id } = reference.as_ref() { + let term = env + .bundle + .entries + .get_term(&id.name) + .ok_or(ResolverError::None)?; - match val { - ast::ArgValue::Number(ref num) => { - fluent_val = num.to_value(env).unwrap(); - } - ast::ArgValue::String(ref string) => { - fluent_val = FluentValue::from(string.as_str()); + match term.value { + ast::Value::VariantList { ref variants } => { + for variant in variants { + if variant.key == *key { + return variant.value.to_value(env); } - }; + } - resolved_named_args.insert(name.name.clone(), fluent_val); + select_default(variants) + .ok_or(ResolverError::None)? + .value + .to_value(env) } - }); + ast::Value::Pattern(ref p) => p.to_value(env), + } + } else { + unimplemented!() } - - env.bundle - .entries - .get_function(&callee.name) - .ok_or(ResolverError::None) - .and_then(|func| { - func(resolved_unnamed_args.as_slice(), &resolved_named_args) - .ok_or(ResolverError::None) - }) } - _ => unimplemented!(), + ast::InlineExpression::FunctionReference { .. } => panic!(), + ast::InlineExpression::MessageReference { ref id } => env + .bundle + .entries + .get_message(&id.name) + .ok_or(ResolverError::None)? + .to_value(env), + ast::InlineExpression::TermReference { ref id } => env + .bundle + .entries + .get_term(&id.name) + .ok_or(ResolverError::None)? + .to_value(env), + ast::InlineExpression::Placeable { ref expression } => { + let exp = expression.as_ref(); + exp.to_value(env) + } } } } -fn select_default(variants: &[ast::Variant]) -> Option<&ast::Variant> { +fn select_default<'source>( + variants: &'source [ast::Variant<'source>], +) -> Option<&ast::Variant<'source>> { for variant in variants { if variant.default { return Some(variant); diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs new file mode 100644 index 00000000..cb6d167d --- /dev/null +++ b/fluent-bundle/src/resource.rs @@ -0,0 +1,40 @@ +use fluent_syntax::ast; +use fluent_syntax::parser::parse; +use fluent_syntax::parser::ParserError; + +rental! { + mod rentals { + use super::*; + #[rental(covariant, debug)] + pub struct FluentResource { + string: String, + ast: ast::Resource<'string>, + } + } +} + +#[derive(Debug)] +pub struct FluentResource(rentals::FluentResource); + +impl FluentResource { + pub fn try_new(source: String) -> Result)> { + let mut errors = None; + let res = rentals::FluentResource::new(source, |s| match parse(s) { + Ok(ast) => ast, + Err((ast, err)) => { + errors = Some(err); + ast + } + }); + + if let Some(errors) = errors { + return Err((Self(res), errors)); + } else { + return Ok(Self(res)); + } + } + + pub fn ast(&self) -> &ast::Resource { + self.0.all().ast + } +} diff --git a/fluent/src/types.rs b/fluent-bundle/src/types.rs similarity index 100% rename from fluent/src/types.rs rename to fluent-bundle/src/types.rs diff --git a/fluent/tests/bundle.rs b/fluent-bundle/tests/bundle.rs similarity index 95% rename from fluent/tests/bundle.rs rename to fluent-bundle/tests/bundle.rs index fc9a9bd3..2dd4ed3a 100644 --- a/fluent/tests/bundle.rs +++ b/fluent-bundle/tests/bundle.rs @@ -1,6 +1,4 @@ -extern crate fluent; - -use self::fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn bundle_new_from_str() { diff --git a/fluent/tests/format_message.rs b/fluent-bundle/tests/compound.rs similarity index 53% rename from fluent/tests/format_message.rs rename to fluent-bundle/tests/compound.rs index 472557e1..7b40a786 100644 --- a/fluent/tests/format_message.rs +++ b/fluent-bundle/tests/compound.rs @@ -1,29 +1,28 @@ -extern crate fluent; - mod helpers; -use fluent::bundle::FluentBundle; -use fluent::bundle::Message; -use helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; +use self::helpers::{ + assert_compound_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; +use fluent_bundle::bundle::Message; use std::collections::HashMap; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Attribute .attr2 = Attribute 2 -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut attrs = HashMap::new(); attrs.insert("attr".to_string(), "Attribute".to_string()); attrs.insert("attr2".to_string(), "Attribute 2".to_string()); - assert_format_message_no_errors( - bundle.format_message("foo", None), + assert_compound_no_errors( + bundle.compound("foo", None), Message { value: Some("Foo".to_string()), attributes: attrs, diff --git a/fluent/tests/format.rs b/fluent-bundle/tests/format.rs similarity index 68% rename from fluent/tests/format.rs rename to fluent-bundle/tests/format.rs index 7b5e2f50..a51a5152 100644 --- a/fluent/tests/format.rs +++ b/fluent-bundle/tests/format.rs @@ -1,20 +1,20 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; +use self::helpers::{ + assert_format_no_errors, assert_format_none, assert_get_bundle_no_errors, + assert_get_resource_from_str_no_errors, +}; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Attribute -term = Term -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo"); diff --git a/fluent-bundle/tests/helpers/mod.rs b/fluent-bundle/tests/helpers/mod.rs new file mode 100644 index 00000000..595170d2 --- /dev/null +++ b/fluent-bundle/tests/helpers/mod.rs @@ -0,0 +1,35 @@ +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::bundle::FluentError; +use fluent_bundle::bundle::Message; +use fluent_bundle::resource::FluentResource; + +#[allow(dead_code)] +pub fn assert_format_none(result: Option<(String, Vec)>) { + assert!(result.is_none()); +} + +#[allow(dead_code)] +pub fn assert_format_no_errors(result: Option<(String, Vec)>, expected: &str) { + assert!(result.is_some()); + assert_eq!(result, Some((expected.to_string(), vec![]))); +} + +#[allow(dead_code)] +pub fn assert_compound_no_errors(result: Option<(Message, Vec)>, expected: Message) { + assert_eq!(result, Some((expected, vec![]))); +} + +pub fn assert_get_resource_from_str_no_errors(s: &str) -> FluentResource { + FluentResource::try_new(s.to_owned()).unwrap() +} + +pub fn assert_get_bundle_no_errors<'a>( + res: &'a FluentResource, + locale: Option<&str>, +) -> FluentBundle<'a> { + let mut bundle = FluentBundle::new(&[locale.unwrap_or("x-testing")]); + bundle + .add_resource(res) + .expect("Failed to add FluentResource to FluentBundle."); + bundle +} diff --git a/fluent/tests/resolve_attribute_expression.rs b/fluent-bundle/tests/resolve_attribute_expression.rs similarity index 74% rename from fluent/tests/resolve_attribute_expression.rs rename to fluent-bundle/tests/resolve_attribute_expression.rs index cbd5edc8..e8c58631 100644 --- a/fluent/tests/resolve_attribute_expression.rs +++ b/fluent-bundle/tests/resolve_attribute_expression.rs @@ -1,15 +1,12 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn attribute_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Foo Attr @@ -23,8 +20,9 @@ use-bar-attr = { bar.attr } missing-attr = { foo.missing } missing-missing = { missing.missing } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent/tests/resolve_external_argument.rs b/fluent-bundle/tests/resolve_external_argument.rs similarity index 53% rename from fluent/tests/resolve_external_argument.rs rename to fluent-bundle/tests/resolve_external_argument.rs index cb22fa94..854cbf22 100644 --- a/fluent/tests/resolve_external_argument.rs +++ b/fluent-bundle/tests/resolve_external_argument.rs @@ -1,18 +1,16 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_string() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages("hello-world = Hello { $name }")); + let res = assert_get_resource_from_str_no_errors("hello-world = Hello { $name }"); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("name", FluentValue::from("John")); @@ -22,14 +20,13 @@ fn external_argument_string() { #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors( - bundle.add_messages("unread-emails = You have { $emailsCount } unread emails."), - ); - assert_add_messages_no_errors( - bundle.add_messages("unread-emails-dec = You have { $emailsCountDec } unread emails."), + let res = assert_get_resource_from_str_no_errors( + " +unread-emails = You have { $emailsCount } unread emails. +unread-emails-dec = You have { $emailsCountDec } unread emails. + ", ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(5)); @@ -48,12 +45,13 @@ fn external_argument_number() { #[test] fn reference_message_with_external_argument() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages("greetings = Hello, { $userName }")); - assert_add_messages_no_errors( - bundle.add_messages("click-on = Click on the `{ greetings }` label."), + let res = assert_get_resource_from_str_no_errors( + " +greetings = Hello, { $userName } +click-on = Click on the `{ greetings }` label. + ", ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("userName", FluentValue::from("Mary")); diff --git a/fluent/tests/resolve_message_reference.rs b/fluent-bundle/tests/resolve_message_reference.rs similarity index 53% rename from fluent/tests/resolve_message_reference.rs rename to fluent-bundle/tests/resolve_message_reference.rs index 7a7af037..0bcda5b1 100644 --- a/fluent/tests/resolve_message_reference.rs +++ b/fluent-bundle/tests/resolve_message_reference.rs @@ -1,81 +1,79 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn message_reference() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar"); } #[test] fn term_reference() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo bar = { -foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar"); } #[test] fn message_reference_nested() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar baz = { bar } Baz -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("baz", None), "Foo Bar Baz"); } #[test] fn message_reference_missing() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages("bar = { foo } Bar")); - + let res = assert_get_resource_from_str_no_errors("bar = { foo } Bar"); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "___ Bar"); } #[test] fn message_reference_cyclic() { { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo { bar } bar = { foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo ___"); assert_format_no_errors(bundle.format("bar", None), "___ Bar"); } { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { bar } bar = { foo } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "___"); assert_format_no_errors(bundle.format("bar", None), "___"); @@ -84,13 +82,13 @@ bar = { foo } #[test] fn message_reference_multiple() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar { foo } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar Foo"); } diff --git a/fluent/tests/resolve_plural_rule.rs b/fluent-bundle/tests/resolve_plural_rule.rs similarity index 80% rename from fluent/tests/resolve_plural_rule.rs rename to fluent-bundle/tests/resolve_plural_rule.rs index c22b3483..a0caa7fd 100644 --- a/fluent/tests/resolve_plural_rule.rs +++ b/fluent-bundle/tests/resolve_plural_rule.rs @@ -1,17 +1,15 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " unread-emails = { $emailsCount -> @@ -24,9 +22,9 @@ unread-emails-dec = [one] You have { $emailsCountDec } unread email. *[other] You have { $emailsCountDec } unread emails. } - -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(1)); @@ -45,8 +43,7 @@ unread-emails-dec = #[test] fn exact_match() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " unread-emails = { $emailsCount -> @@ -61,9 +58,9 @@ unread-emails-dec = [one] You have { $emailsCountDec } unread email. *[other] You have { $emailsCountDec } unread emails. } - -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(1)); diff --git a/fluent/tests/resolve_select_expression.rs b/fluent-bundle/tests/resolve_select_expression.rs similarity index 70% rename from fluent/tests/resolve_select_expression.rs rename to fluent-bundle/tests/resolve_select_expression.rs index 48309369..7cda0a73 100644 --- a/fluent/tests/resolve_select_expression.rs +++ b/fluent-bundle/tests/resolve_select_expression.rs @@ -1,42 +1,15 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; - -#[test] -fn select_expression_without_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages( - " -foo = - { - *[nominative] Foo - [genitive] Foo's - } - -bar = - { - [genitive] Bar's - *[nominative] Bar - } -", - )); - - assert_format_no_errors(bundle.format("foo", None), "Foo"); - - assert_format_no_errors(bundle.format("bar", None), "Bar"); -} +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; +use fluent_bundle::types::FluentValue; #[test] fn select_expression_string_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { \"genitive\" -> @@ -49,8 +22,9 @@ bar = *[nominative] Bar [genitive] Bar's } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo's"); @@ -59,8 +33,7 @@ bar = #[test] fn select_expression_number_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { 3 -> @@ -80,8 +53,9 @@ baz = [3] Baz 3 [3.14] Baz Pi } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -92,8 +66,7 @@ baz = #[test] fn select_expression_plurals() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { 3 -> @@ -115,8 +88,9 @@ baz = [3] Bar 3 *[other] Bar Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -127,8 +101,7 @@ baz = #[test] fn select_expression_external_argument_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo-hit = { $str -> @@ -183,8 +156,9 @@ baz-unknown = *[1] Baz 1 [2] Baz 2 } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("str", FluentValue::from("qux")); @@ -212,38 +186,38 @@ baz-unknown = #[test] fn select_expression_message_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -bar = Bar - .attr = attr val + .attr = attr_val use-bar = { -bar.attr -> - [attr val] Bar + [attr_val] Bar *[other] Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-bar", None), "Bar"); } #[test] fn select_expression_attribute_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo - .attr = Foo Attr + .attr = FooAttr use-foo = { -foo.attr -> - [Foo Attr] Foo + [FooAttr] Foo *[other] Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); } diff --git a/fluent-bundle/tests/resolve_value.rs b/fluent-bundle/tests/resolve_value.rs new file mode 100644 index 00000000..e0b1c04d --- /dev/null +++ b/fluent-bundle/tests/resolve_value.rs @@ -0,0 +1,30 @@ +mod helpers; + +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; + +#[test] +fn format_message() { + let res = assert_get_resource_from_str_no_errors( + " +foo = Foo + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); + + assert_format_no_errors(bundle.format("foo", None), "Foo"); +} + +#[test] +fn format_attribute() { + let res = assert_get_resource_from_str_no_errors( + " +foo = Foo + .attr = Foo Attr + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); + + assert_format_no_errors(bundle.format("foo.attr", None), "Foo Attr"); +} diff --git a/fluent/tests/resolve_variant_expression.rs b/fluent-bundle/tests/resolve_variant_expression.rs similarity index 73% rename from fluent/tests/resolve_variant_expression.rs rename to fluent-bundle/tests/resolve_variant_expression.rs index 3828b20a..978c0ec0 100644 --- a/fluent/tests/resolve_variant_expression.rs +++ b/fluent-bundle/tests/resolve_variant_expression.rs @@ -1,14 +1,12 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn variant_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo -bar = @@ -16,7 +14,7 @@ fn variant_expression() { *[nominative] Bar [genitive] Bar's } -bar = { -bar } +baz = { -bar } use-foo = { -foo } use-foo-missing = { -foo[missing] } @@ -27,10 +25,11 @@ use-bar-genitive = { -bar[genitive] } use-bar-missing = { -bar[missing] } missing-missing = { -missing[missing] } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); - assert_format_no_errors(bundle.format("bar", None), "Bar"); + assert_format_no_errors(bundle.format("baz", None), "Bar"); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent-cli/Cargo.toml b/fluent-cli/Cargo.toml new file mode 100644 index 00000000..0751bbee --- /dev/null +++ b/fluent-cli/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "fluent-cli" +description = """ +A collection of command line interface programs +for Fluent Localization System. +""" +version = "0.0.1" +edition = "2018" +authors = [ + "Zibi Braniecki ", + "Staś Małolepszy " +] +homepage = "http://www.projectfluent.org" +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] + +[dependencies] +annotate-snippets = {version = "^0.5", features = ["color"]} +clap = "^2.32" +fluent-syntax = { path = "../fluent-syntax" } diff --git a/fluent-cli/src/main.rs b/fluent-cli/src/main.rs new file mode 100644 index 00000000..454c3b50 --- /dev/null +++ b/fluent-cli/src/main.rs @@ -0,0 +1,150 @@ +use std::cmp; +use std::fs::File; +use std::io; +use std::io::Read; + +use clap::App; + +use annotate_snippets::display_list::DisplayList; +use annotate_snippets::formatter::DisplayListFormatter; +use annotate_snippets::snippet::{Annotation, AnnotationType, Slice, Snippet, SourceAnnotation}; +use fluent_syntax::ast::Resource; +use fluent_syntax::parser::errors::ErrorKind; +use fluent_syntax::parser::parse; + +fn read_file(path: &str) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn print_entries_resource(res: &Resource) { + println!("{:#?}", res); +} + +fn main() { + let matches = App::new("Fluent Parser") + .version("1.0") + .about("Parses FTL file into an AST") + .args_from_usage( + "-s, --silent 'Disables error reporting' + 'FTL file to parse'", + ) + .get_matches(); + + let input = matches.value_of("INPUT").unwrap(); + + let source = read_file(&input).expect("Read file failed"); + + let res = parse(&source); + + match res { + Ok(res) => print_entries_resource(&res), + Err((res, errors)) => { + print_entries_resource(&res); + + if matches.is_present("silent") { + return; + }; + + println!("==============================\n"); + if errors.len() == 1 { + println!("Parser encountered one error:"); + } else { + println!("Parser encountered {} errors:", errors.len()); + } + println!("-----------------------------"); + for err in errors { + println!("{:#?}", err); + if let Some(slice) = err.slice { + let (id, desc) = get_error_info(err.kind); + let end_pos = cmp::min(err.pos.1, slice.1); + let snippet = Snippet { + slices: vec![Slice { + source: source[slice.0..slice.1].to_string(), + line_start: get_line_num(&source, err.pos.0) + 1, + origin: Some(input.to_string()), + fold: false, + annotations: vec![SourceAnnotation { + label: desc.to_string(), + annotation_type: AnnotationType::Error, + range: (err.pos.0 - slice.0, end_pos - slice.0 + 1), + }], + }], + title: Some(Annotation { + label: Some(desc.to_string()), + id: Some(id.to_string()), + annotation_type: AnnotationType::Error, + }), + footer: vec![], + }; + let dl = DisplayList::from(snippet); + let dlf = DisplayListFormatter::new(true); + println!("{}", dlf.format(&dl)); + println!("-----------------------------"); + } + } + } + }; +} + +fn get_line_num(source: &str, pos: usize) -> usize { + let mut ptr = 0; + let mut i = 0; + + let lines = source.lines(); + + for line in lines { + let lnlen = line.chars().count(); + ptr += lnlen + 1; + + if ptr > pos { + break; + } + i += 1; + } + + i +} + +fn get_error_info(kind: ErrorKind) -> (&'static str, String) { + match kind { + ErrorKind::Generic => ("E0001", "Generic error".to_string()), + ErrorKind::ExpectedEntry => ("E0002", "Expected an entry start".to_string()), + ErrorKind::ExpectedToken(ch) => ("E0003", format!("Expected token: \"{}\"", ch)), + ErrorKind::ExpectedCharRange { range } => ( + "E0004", + format!("Expected a character from range: \"{}\"", range), + ), + ErrorKind::ExpectedMessageField { entry_id } => ( + "E0005", + format!( + "Expected message \"{}\" to have a value or attributes", + entry_id + ), + ), + ErrorKind::ExpectedTermField { entry_id } => ( + "E0006", + format!("Expected term \"{}\" to have a value", entry_id), + ), + ErrorKind::ForbiddenWhitespace => { + ("E0007", "Keyword cannot end with a whitespace".to_string()) + } + ErrorKind::ForbiddenCallee => ( + "E0008", + "The callee has to be a simple, upper-case identifier".to_string(), + ), + ErrorKind::ForbiddenKey => ("E0009", "The key has to be a simple identifier".to_string()), + ErrorKind::MissingDefaultVariant => ( + "E0010", + "Expected one of the variants to be marked as default **)".to_string(), + ), + ErrorKind::MissingValue => ("E0012", "Expected value.".to_string()), + ErrorKind::TermAttributeAsPlaceable => ( + "E0019", + "Attributes of terms cannot be used as placeables".to_string(), + ), + _ => ("E0000", "Unknown Error.".to_string()), + } +} diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index 56dcbee1..9b8bd953 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -4,6 +4,7 @@ description = """ Parser/Serializer tools for Fluent Syntax. """ version = "0.1.1" +edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " @@ -15,8 +16,9 @@ readme = "README.md" keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] -[dependencies] -annotate-snippets = {version = "0.1", features = ["color"]} - [dev-dependencies] -glob = "0.2" +serde = "^1.0" +serde_derive = "^1.0" +serde_json = "^1.0" +glob = "^0.2" +assert-json-diff = "^0.2.1" diff --git a/fluent-syntax/benches/lib.rs b/fluent-syntax/benches/lib.rs index e0d7e4fd..d5736c5e 100644 --- a/fluent-syntax/benches/lib.rs +++ b/fluent-syntax/benches/lib.rs @@ -1,6 +1,5 @@ #![feature(test)] -extern crate fluent_syntax; extern crate test; use fluent_syntax::parser::parse; @@ -10,9 +9,9 @@ use std::io::Read; use test::Bencher; fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); + let mut f = File::open(path)?; let mut s = String::new(); - try!(f.read_to_string(&mut s)); + f.read_to_string(&mut s)?; Ok(s) } diff --git a/fluent-syntax/src/ast.rs b/fluent-syntax/src/ast.rs index 77a22188..30dae45f 100644 --- a/fluent-syntax/src/ast.rs +++ b/fluent-syntax/src/ast.rs @@ -1,129 +1,137 @@ #[derive(Debug, PartialEq)] -pub struct Resource { - pub body: Vec, +pub struct Resource<'ast> { + pub body: Vec>, } #[derive(Debug, PartialEq)] -pub enum Entry { - Message(Message), - Term(Term), - Comment(Comment), - Junk { content: String }, +pub enum ResourceEntry<'ast> { + Entry(Entry<'ast>), + Junk(&'ast str), } #[derive(Debug, PartialEq)] -pub struct Message { - pub id: Identifier, - pub value: Option, - pub attributes: Option>, - pub comment: Option, +pub enum Entry<'ast> { + Message(Message<'ast>), + Term(Term<'ast>), + Comment(Comment<'ast>), } #[derive(Debug, PartialEq)] -pub struct Term { - pub id: Identifier, - pub value: Pattern, - pub attributes: Option>, - pub comment: Option, +pub struct Message<'ast> { + pub id: Identifier<'ast>, + pub value: Option>, + pub attributes: Vec>, + pub comment: Option>, } #[derive(Debug, PartialEq)] -pub struct Pattern { - pub elements: Vec, +pub struct Term<'ast> { + pub id: Identifier<'ast>, + pub value: Value<'ast>, + pub attributes: Vec>, + pub comment: Option>, } #[derive(Debug, PartialEq)] -pub enum PatternElement { - TextElement(String), - Placeable(Expression), +pub enum Value<'ast> { + Pattern(Pattern<'ast>), + VariantList { variants: Vec> }, } #[derive(Debug, PartialEq)] -pub enum Expression { - StringExpression { - value: String, - }, - NumberExpression { - value: Number, - }, - MessageReference { - id: Identifier, - }, - ExternalArgument { - id: Identifier, - }, - SelectExpression { - expression: Option>, - variants: Vec, - }, - AttributeExpression { - id: Identifier, - name: Identifier, - }, - VariantExpression { - id: Identifier, - key: VarKey, - }, - CallExpression { - callee: Function, - args: Vec, - }, +pub struct Pattern<'ast> { + pub elements: Vec>, } #[derive(Debug, PartialEq)] -pub struct Attribute { - pub id: Identifier, - pub value: Pattern, +pub enum PatternElement<'ast> { + TextElement(&'ast str), + Placeable(Expression<'ast>), } #[derive(Debug, PartialEq)] -pub struct Variant { - pub key: VarKey, - pub value: Pattern, - pub default: bool, +pub struct Attribute<'ast> { + pub id: Identifier<'ast>, + pub value: Pattern<'ast>, } #[derive(Debug, PartialEq)] -pub enum VarKey { - VariantName(VariantName), - Number(Number), +pub struct Identifier<'ast> { + pub name: &'ast str, } #[derive(Debug, PartialEq)] -pub enum Argument { - Expression(Expression), - NamedArgument { name: Identifier, val: ArgValue }, +pub struct Variant<'ast> { + pub key: VariantKey<'ast>, + pub value: Pattern<'ast>, + pub default: bool, } #[derive(Debug, PartialEq)] -pub enum ArgValue { - Number(Number), - String(String), -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Identifier { - pub name: String, +pub enum VariantKey<'ast> { + Identifier { name: &'ast str }, + NumberLiteral { value: &'ast str }, } #[derive(Debug, PartialEq)] -pub struct Number { - pub value: String, +pub enum Comment<'ast> { + Comment { content: Vec<&'ast str> }, + GroupComment { content: Vec<&'ast str> }, + ResourceComment { content: Vec<&'ast str> }, } #[derive(Debug, PartialEq)] -pub struct VariantName { - pub name: String, +pub enum InlineExpression<'ast> { + StringLiteral { + raw: &'ast str, + }, + NumberLiteral { + value: &'ast str, + }, + VariableReference { + id: Identifier<'ast>, + }, + CallExpression { + callee: Box>, + positional: Vec>, + named: Vec>, + }, + AttributeExpression { + reference: Box>, + name: Identifier<'ast>, + }, + VariantExpression { + reference: Box>, + key: VariantKey<'ast>, + }, + MessageReference { + id: Identifier<'ast>, + }, + TermReference { + id: Identifier<'ast>, + }, + // This node is standalone in EBNF, but it + // is more convinient for us to store it a + // variant of the InlineExpression in Rust. + FunctionReference { + id: Identifier<'ast>, + }, + Placeable { + expression: Box>, + }, } #[derive(Debug, PartialEq)] -pub enum Comment { - Comment { content: String }, - GroupComment { content: String }, - ResourceComment { content: String }, +pub struct NamedArgument<'ast> { + pub name: Identifier<'ast>, + pub value: InlineExpression<'ast>, } #[derive(Debug, PartialEq)] -pub struct Function { - pub name: String, +pub enum Expression<'ast> { + InlineExpression(InlineExpression<'ast>), + SelectExpression { + selector: InlineExpression<'ast>, + variants: Vec>, + }, } diff --git a/fluent-syntax/src/parser/errors.rs b/fluent-syntax/src/parser/errors.rs new file mode 100644 index 00000000..7b8ab028 --- /dev/null +++ b/fluent-syntax/src/parser/errors.rs @@ -0,0 +1,53 @@ +#[derive(Debug, PartialEq)] +pub struct ParserError { + pub pos: (usize, usize), + pub slice: Option<(usize, usize)>, + pub kind: ErrorKind, +} + +macro_rules! error { + ($kind:expr, $start:expr) => {{ + Err(ParserError { + pos: ($start, $start + 1), + slice: None, + kind: $kind, + }) + }}; + ($kind:expr, $start:expr, $end:expr) => {{ + Err(ParserError { + pos: ($start, $end), + slice: None, + kind: $kind, + }) + }}; +} + +#[derive(Debug, PartialEq)] +pub enum ErrorKind { + Generic, + ExpectedEntry, + ExpectedToken(char), + ExpectedCharRange { range: String }, + ExpectedMessageField { entry_id: String }, + ExpectedTermField { entry_id: String }, + ForbiddenWhitespace, + ForbiddenCallee, + ForbiddenKey, + MissingDefaultVariant, + MissingVariants, + MissingValue, + MissingVariantKey, + MissingLiteral, + MultipleDefaultVariants, + MessageReferenceAsSelector, + VariantAsSelector, + MessageAttributeAsSelector, + TermAttributeAsPlaceable, + UnterminatedStringExpression, + PositionalArgumentFollowsNamed, + DuplicatedNamedArgument(String), + VariantListInExpression, + ForbiddenVariantAccessor, + UnknownEscapeSequence(String), + InvalidUnicodeEscapeSequence(String), +} diff --git a/fluent-syntax/src/parser/errors/display.rs b/fluent-syntax/src/parser/errors/display.rs deleted file mode 100644 index 401a142c..00000000 --- a/fluent-syntax/src/parser/errors/display.rs +++ /dev/null @@ -1,41 +0,0 @@ -extern crate annotate_snippets; - -use super::list::get_error_desc; -use super::ParserError; - -use self::annotate_snippets::display_list::DisplayList; -use self::annotate_snippets::formatter::DisplayListFormatter; -use self::annotate_snippets::snippet; - -pub fn annotate_error(err: &ParserError, file_name: &Option, color: bool) -> String { - let desc = get_error_desc(&err.kind); - - let (source, line_start, pos) = if let Some(ref info) = err.info { - (info.slice.clone(), info.line, info.pos) - } else { - panic!() - }; - - let snippet = snippet::Snippet { - slices: vec![snippet::Slice { - source, - line_start, - origin: file_name.clone(), - fold: false, - annotations: vec![snippet::SourceAnnotation { - label: desc.2.to_string(), - annotation_type: snippet::AnnotationType::Error, - range: (pos, pos + 1), - }], - }], - title: Some(snippet::Annotation { - label: Some(desc.1), - id: Some(desc.0.to_string()), - annotation_type: snippet::AnnotationType::Error, - }), - footer: vec![], - }; - let dl = DisplayList::from(snippet); - let dlf = DisplayListFormatter::new(color); - dlf.format(&dl) -} diff --git a/fluent-syntax/src/parser/errors/list.rs b/fluent-syntax/src/parser/errors/list.rs deleted file mode 100644 index bcf6d5a1..00000000 --- a/fluent-syntax/src/parser/errors/list.rs +++ /dev/null @@ -1,122 +0,0 @@ -#[derive(Debug, PartialEq)] -pub struct ParserError { - pub info: Option, - pub kind: ErrorKind, -} - -#[derive(Debug, PartialEq)] -pub struct ErrorInfo { - pub slice: String, - pub line: usize, - pub pos: usize, -} - -#[derive(Debug, PartialEq)] -pub enum ErrorKind { - Generic, - ExpectedEntry, - ExpectedToken { token: char }, - ExpectedCharRange { range: String }, - ExpectedMessageField { entry_id: String }, - ExpectedTermField { entry_id: String }, - ForbiddenWhitespace, - ForbiddenCallee, - ForbiddenKey, - MissingDefaultVariant, - MissingVariants, - MissingValue, - MissingVariantKey, - MissingLiteral, - MultipleDefaultVariants, - MessageReferenceAsSelector, - VariantAsSelector, - MessageAttributeAsSelector, - TermAttributeAsSelector, - UnterminatedStringExpression, -} - -pub fn get_error_desc(err: &ErrorKind) -> (&'static str, String, &'static str) { - match err { - ErrorKind::Generic => ("E0001", "generic error".to_owned(), ""), - ErrorKind::ExpectedEntry => ( - "E0002", - "Expected an entry start".to_owned(), - "Expected one of ('a'...'Z' | '_' | #') here", - ), - ErrorKind::ExpectedToken { token } => ("E0003", format!("expected token `{}`", token), ""), - ErrorKind::ExpectedCharRange { range } => ( - "E0004", - format!("Expected a character from range ({})", range), - "", - ), - ErrorKind::ExpectedMessageField { entry_id } => ( - "E0005", - format!( - "Expected message `{}` to have a value or attributes", - entry_id - ), - "", - ), - ErrorKind::ExpectedTermField { entry_id } => ( - "E0006", - format!("Expected term `{}` to have a value", entry_id), - "", - ), - ErrorKind::ForbiddenWhitespace => ( - "E0007", - "Keyword cannot end with a whitespace".to_owned(), - "", - ), - ErrorKind::ForbiddenCallee => ( - "E0008", - "The callee has to be a simple, upper-case, identifier".to_owned(), - "", - ), - ErrorKind::ForbiddenKey => ( - "E0009", - "The key has to be a simple identifier".to_owned(), - "", - ), - ErrorKind::MissingDefaultVariant => ( - "E0010", - "Expected one of the variants to be marked as default (*).".to_owned(), - "", - ), - ErrorKind::MissingVariants => ( - "E0011", - "Expected at least one variant after \"->\".".to_owned(), - "", - ), - ErrorKind::MissingValue => ("E0012", "Expected value".to_owned(), ""), - ErrorKind::MissingVariantKey => ("E0013", "Expected variant key".to_owned(), ""), - ErrorKind::MissingLiteral => ("E0014", "Expected literal".to_owned(), ""), - ErrorKind::MultipleDefaultVariants => ( - "E0015", - "Only one variant can be marked as default (*)".to_owned(), - "", - ), - ErrorKind::MessageReferenceAsSelector => ( - "E0016", - "Message references cannot be used as selectors".to_owned(), - "", - ), - ErrorKind::VariantAsSelector => ( - "E0017", - "Variants cannot be used as selectors".to_owned(), - "", - ), - ErrorKind::MessageAttributeAsSelector => ( - "E0018", - "Attributes of messages cannot be used as selectors.".to_owned(), - "", - ), - ErrorKind::TermAttributeAsSelector => ( - "E0019", - "Attributes of terms cannot be used as selectors.".to_owned(), - "", - ), - ErrorKind::UnterminatedStringExpression => { - ("E0020", "Underminated string expression".to_owned(), "") - } - } -} diff --git a/fluent-syntax/src/parser/errors/mod.rs b/fluent-syntax/src/parser/errors/mod.rs deleted file mode 100644 index 133dc3b0..00000000 --- a/fluent-syntax/src/parser/errors/mod.rs +++ /dev/null @@ -1,96 +0,0 @@ -pub mod display; -mod list; - -pub use self::list::get_error_desc; -pub use self::list::ErrorInfo; -pub use self::list::ErrorKind; -pub use self::list::ParserError; - -macro_rules! error { - ($kind:expr) => {{ - Err(ParserError { - info: None, - kind: $kind, - }) - }}; -} - -fn get_line_num(source: &str, pos: usize) -> usize { - let mut ptr = 0; - let mut i = 0; - - let lines = source.lines(); - - for line in lines { - let lnlen = line.chars().count(); - ptr += lnlen + 1; - - if ptr > pos { - break; - } - i += 1; - } - - i -} - -pub fn get_error_lines(source: &str, start: usize, end: usize) -> String { - let l = if start < end { end - start } else { 1 }; - - let lines = source.lines().skip(start).take(l); - - let mut s = String::new(); - - for line in lines { - s.push_str(line); - s.push('\n'); - } - - String::from(s.trim_right()) -} - -pub fn get_error_slice(source: &str, start: usize, end: usize) -> &str { - let len = source.chars().count(); - - let start_pos; - let mut slice_len = end - start; - - if len <= slice_len { - start_pos = 0; - slice_len = len; - } else if start + slice_len >= len { - start_pos = len - slice_len - 1; - } else { - start_pos = start; - } - - let mut iter = source.chars(); - if start_pos > 0 { - iter.by_ref().nth(start_pos - 1); - } - let slice = iter.as_str(); - let endp = slice - .char_indices() - .nth(slice_len) - .map(|(n, _)| n) - .unwrap_or(len); - &slice[..endp] -} - -pub fn get_error_info( - source: &str, - pos: usize, - entry_start: usize, - next_entry_start: usize, -) -> Option { - let first_line_num = get_line_num(source, entry_start); - let next_entry_line = get_line_num(source, next_entry_start); - - let slice = get_error_lines(source, first_line_num, next_entry_line); - - Some(ErrorInfo { - slice, - line: first_line_num, - pos: pos - entry_start, - }) -} diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index cdee2c44..d7338153 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -1,394 +1,214 @@ -use super::errors::ErrorKind; -use super::errors::ParserError; -use super::parser::Result; -use super::stream::ParserStream; - -pub trait FTLParserStream { - fn skip_inline_ws(&mut self); - fn peek_inline_ws(&mut self); - fn skip_blank_lines(&mut self); - fn peek_blank_lines(&mut self); - fn skip_indent(&mut self); - fn expect_char(&mut self, ch: char) -> Result<()>; - fn expect_indent(&mut self) -> Result<()>; - fn take_char_if(&mut self, ch: char) -> bool; - - fn take_char(&mut self, f: F) -> Option - where - F: Fn(char) -> bool; - - fn is_char_id_start(&mut self, ch: Option) -> bool; - fn is_entry_id_start(&mut self) -> bool; - fn is_number_start(&mut self) -> bool; - fn is_char_pattern_continuation(&self, ch: Option) -> bool; - fn is_peek_pattern_start(&mut self) -> bool; - fn is_peek_next_line_zero_four_style_comment(&mut self) -> bool; - fn is_peek_next_line_comment(&mut self, level: i8) -> bool; - fn is_peek_next_line_variant_start(&mut self) -> bool; - fn is_peek_next_line_attribute_start(&mut self) -> bool; - fn is_peek_next_line_pattern_start(&mut self) -> bool; - fn skip_to_next_entry_start(&mut self); - fn take_id_start(&mut self, allow_private: bool) -> Result; - fn take_id_char(&mut self) -> Option; - fn take_variant_name_char(&mut self) -> Option; - fn take_digit(&mut self) -> Option; +use super::errors::{ErrorKind, ParserError}; +use super::Result; +use std::str; + +pub struct ParserStream<'p> { + pub source: &'p [u8], + pub length: usize, + pub ptr: usize, } -static INLINE_WS: [char; 2] = [' ', '\t']; -static SPECIAL_LINE_START_CHARS: [char; 4] = ['}', '.', '[', '*']; - -impl FTLParserStream for ParserStream -where - I: Iterator, -{ - fn skip_inline_ws(&mut self) { - while let Some(ch) = self.ch { - if !INLINE_WS.contains(&ch) { - break; - } - self.next(); +impl<'p> ParserStream<'p> { + pub fn new(stream: &'p str) -> Self { + ParserStream { + source: stream.as_bytes(), + length: stream.len(), + ptr: 0, } } - fn peek_inline_ws(&mut self) { - while let Some(ch) = self.current_peek() { - if !INLINE_WS.contains(&ch) { - break; - } - self.peek(); - } - } - - fn skip_blank_lines(&mut self) { - loop { - self.peek_inline_ws(); - - if self.current_peek() == Some('\n') { - self.skip_to_peek(); - self.next(); - } else { - self.reset_peek(None); - break; - } - } - } - - fn peek_blank_lines(&mut self) { - loop { - let line_start = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.current_peek_is('\n') { - self.peek(); - } else { - self.reset_peek(Some(line_start)); - break; - } - } + pub fn is_current_byte(&self, b: u8) -> bool { + self.source.get(self.ptr) == Some(&b) } - fn skip_indent(&mut self) { - self.skip_blank_lines(); - self.skip_inline_ws(); + pub fn is_byte_at(&self, b: u8, pos: usize) -> bool { + self.source.get(pos) == Some(&b) } - fn expect_char(&mut self, ch: char) -> Result<()> { - if self.ch == Some(ch) { - self.next(); - return Ok(()); - } - - if self.ch == Some('\n') { - // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) - return error!(ErrorKind::ExpectedToken { token: '\u{2424}' }); + pub fn expect_byte(&mut self, b: u8) -> Result<()> { + if !self.is_current_byte(b) { + return error!(ErrorKind::ExpectedToken(b as char), self.ptr); } - - error!(ErrorKind::ExpectedToken { token: ch }) - } - - fn expect_indent(&mut self) -> Result<()> { - self.expect_char('\n')?; - self.skip_blank_lines(); - self.expect_char(' ')?; - self.skip_inline_ws(); + self.ptr += 1; Ok(()) } - fn take_char_if(&mut self, ch: char) -> bool { - if self.ch == Some(ch) { - self.next(); - return true; + pub fn take_byte_if(&mut self, b: u8) -> bool { + if self.is_current_byte(b) { + self.ptr += 1; + true + } else { + false } - - false } - fn take_char(&mut self, f: F) -> Option - where - F: Fn(char) -> bool, - { - if let Some(ch) = self.ch { - if f(ch) { - self.next(); - return Some(ch); + pub fn skip_blank_block(&mut self) -> usize { + let mut count = 0; + loop { + let start = self.ptr; + self.skip_blank_inline(); + if !self.skip_eol() { + self.ptr = start; + break; } + count += 1; } - None + count } - fn is_char_id_start(&mut self, ch: Option) -> bool { - match ch { - Some('a'...'z') | Some('A'...'Z') => true, - _ => false, - } - } - - fn is_entry_id_start(&mut self) -> bool { - if let Some('-') = self.ch { - self.peek(); - } - let ch = self.current_peek(); - let is_id = self.is_char_id_start(ch); - self.reset_peek(None); - is_id - } - - fn is_number_start(&mut self) -> bool { - if let Some('-') = self.ch { - self.peek(); + pub fn skip_blank(&mut self) { + loop { + match self.source.get(self.ptr) { + Some(b) if [b' ', b'\n'].contains(b) => self.ptr += 1, + _ => break, + } } - let ch = self.current_peek(); - let is_digit = match ch { - Some('0'...'9') => true, - _ => false, - }; - self.reset_peek(None); - is_digit } - fn is_char_pattern_continuation(&self, ch: Option) -> bool { - match ch { - Some(ch) => !SPECIAL_LINE_START_CHARS.contains(&ch), - _ => false, + pub fn skip_blank_inline(&mut self) -> usize { + let start = self.ptr; + while let Some(b' ') = self.source.get(self.ptr) { + self.ptr += 1; } + self.ptr - start } - fn is_peek_pattern_start(&mut self) -> bool { - self.peek_inline_ws(); + pub fn skip_to_next_entry_start(&mut self) { + while let Some(b) = self.source.get(self.ptr) { + let new_line = self.ptr == 0 || self.source.get(self.ptr - 1) == Some(&b'\n'); - if let Some(ch) = self.current_peek() { - if ch != '\n' { - return true; + if new_line && (self.is_byte_alphabetic(*b) || [b'-', b'#'].contains(b)) { + break; } - } - - self.is_peek_next_line_pattern_start() - } - fn is_peek_next_line_zero_four_style_comment(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; + self.ptr += 1; } - self.peek(); - - if self.current_peek_is('/') { - self.peek(); - if self.current_peek_is('/') { - self.reset_peek(None); - return true; - } - } - self.reset_peek(None); - false } - fn is_peek_next_line_comment(&mut self, level: i8) -> bool { - if !self.current_peek_is('\n') { - return false; - } - - let mut i = 0; - - while i <= level && (level == -1 && i < 3) { - self.peek(); - if !self.current_peek_is('#') { - if i != level && level != -1 { - self.reset_peek(None); - return false; - } - break; + pub fn skip_eol(&mut self) -> bool { + match self.source.get(self.ptr) { + Some(b'\n') => { + self.ptr += 1; + true } - i += 1; - } - - self.peek(); - - if let Some(ch) = self.current_peek() { - if [' ', '\n'].contains(&ch) { - self.reset_peek(None); - return true; + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => { + self.ptr += 2; + true } + _ => false, } - self.reset_peek(None); - false } - fn is_peek_next_line_variant_start(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; - } - self.peek(); - - self.peek_blank_lines(); + pub fn skip_to_value_start(&mut self) -> Option { + self.skip_blank_inline(); - let ptr = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); - return false; + if !self.is_eol() { + return Some(true); } - if self.current_peek_is('*') { - self.peek(); - } + self.skip_blank_block(); - if self.current_peek_is('[') && !self.peek_char_is('[') { - self.reset_peek(None); - return true; - } - self.reset_peek(None); - false - } + let inline = self.skip_blank_inline(); - fn is_peek_next_line_attribute_start(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; + if self.is_current_byte(b'{') { + return Some(true); } - self.peek(); - - self.peek_blank_lines(); - let ptr = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); - return false; + if inline == 0 { + return None; } - if self.current_peek_is('.') { - self.reset_peek(None); - return true; + if !self.is_char_pattern_continuation() { + return None; } - - self.reset_peek(None); - false + self.ptr -= inline; + Some(false) } - fn is_peek_next_line_pattern_start(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; + pub fn skip_unicode_escape_sequence(&mut self, length: usize) -> Result<()> { + let start = self.ptr; + for _ in 0..length { + match self.source.get(self.ptr) { + Some(b) if b.is_ascii_hexdigit() => self.ptr += 1, + _ => break, + } } - self.peek(); - - self.peek_blank_lines(); - - let ptr = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); - return false; + if self.ptr - start != length { + let end = if self.ptr >= self.length { + self.ptr + } else { + self.ptr + 1 + }; + return error!( + ErrorKind::InvalidUnicodeEscapeSequence(self.get_slice(start, end).to_owned()), + self.ptr + ); } + Ok(()) + } - if !self.is_char_pattern_continuation(self.current_peek()) { - self.reset_peek(None); - return false; + pub fn is_char_pattern_continuation(&self) -> bool { + match self.source.get(self.ptr) { + Some(b) => self.is_byte_pattern_continuation(*b), + _ => false, } - - self.reset_peek(None); - true } - fn skip_to_next_entry_start(&mut self) { - while let Some(_) = self.next() { - if self.current_is('\n') && !self.peek_char_is('\n') { - self.next(); - - if self.ch.is_none() - || self.is_entry_id_start() - || self.current_is('#') - || (self.current_is('/') && self.peek_char_is('/')) - || (self.current_is('[') && self.peek_char_is('[')) - { - break; - } - } + pub fn is_identifier_start(&self) -> bool { + match self.source.get(self.ptr) { + Some(b) if self.is_byte_alphabetic(*b) => true, + _ => false, } } - fn take_id_start(&mut self, allow_term: bool) -> Result { - if allow_term && self.current_is('-') { - self.next(); - return Ok('-'); - } + pub fn is_byte_alphabetic(&self, b: u8) -> bool { + (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') + } - if let Some(ch) = self.ch { - if self.is_char_id_start(Some(ch)) { - let ret = self.ch.unwrap(); - self.next(); - return Ok(ret); - } - } + pub fn is_byte_digit(&self, b: u8) -> bool { + b >= b'0' && b <= b'9' + } - let allowed_range = if allow_term { - "'a'...'z' | 'A'...'Z' | '-'" - } else { - "'a'...'z' | 'A'...'Z'" - }; - error!(ErrorKind::ExpectedCharRange { - range: String::from(allowed_range), - }) + pub fn is_byte_pattern_continuation(&self, b: u8) -> bool { + ![b'}', b'.', b'[', b'*'].contains(&b) } - fn take_id_char(&mut self) -> Option { - let closure = |x| match x { - 'a'...'z' | 'A'...'Z' | '0'...'9' | '_' | '-' => true, + pub fn is_number_start(&self) -> bool { + match self.source.get(self.ptr) { + Some(b) if (b == &b'-') || self.is_byte_digit(*b) => true, _ => false, - }; - - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, } } - fn take_variant_name_char(&mut self) -> Option { - let closure = |x| match x { - 'a'...'z' | 'A'...'Z' | '0'...'9' | '_' | '-' | ' ' => true, + pub fn is_eol(&self) -> bool { + match self.source.get(self.ptr) { + Some(b'\n') => true, + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => true, _ => false, - }; - - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, } } - fn take_digit(&mut self) -> Option { - let closure = |x| match x { - '0'...'9' => true, - _ => false, - }; + pub fn get_slice(&self, start: usize, end: usize) -> &'p str { + str::from_utf8(&self.source[start..end]).expect("Slicing the source failed") + } - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, + pub fn skip_digits(&mut self) -> Result<()> { + let start = self.ptr; + loop { + match self.source.get(self.ptr) { + Some(b) if self.is_byte_digit(*b) => self.ptr += 1, + _ => break, + } + } + if start == self.ptr { + error!( + ErrorKind::ExpectedCharRange { + range: "0-9".to_string() + }, + self.ptr + ) + } else { + Ok(()) } } } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index ac133c9f..406ecb31 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -1,14 +1,857 @@ -//! AST, parser and serializer operations -//! -//! This is an internal API used by `FluentBundle` for parsing an FTL syntax -//! into an AST that can be then resolved by the `Resolver`. -//! -//! This module may be useful for tooling that operates on FTL syntax. - #[macro_use] pub mod errors; -pub mod ftlstream; -pub mod parser; -pub mod stream; +mod ftlstream; + +use std::cmp; +use std::result; +use std::str; + +use self::errors::ErrorKind; +pub use self::errors::ParserError; +use self::ftlstream::ParserStream; +use super::ast; + +pub type Result = result::Result; + +pub fn parse(source: &str) -> result::Result)> { + let mut errors = vec![]; + + let mut ps = ParserStream::new(source); + + let mut body = vec![]; + + ps.skip_blank_block(); + let mut last_comment = None; + let mut last_blank_count = 0; + + while ps.ptr < ps.length { + let entry_start = ps.ptr; + let mut entry = get_entry(&mut ps, entry_start); + + if let Some(comment) = last_comment.take() { + match entry { + Ok(ast::Entry::Message(ref mut msg)) if last_blank_count < 2 => { + msg.comment = Some(comment); + } + Ok(ast::Entry::Term(ref mut term)) if last_blank_count < 2 => { + term.comment = Some(comment); + } + _ => { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment(comment))); + } + } + } + + match entry { + Ok(ast::Entry::Comment(comment @ ast::Comment::Comment { .. })) => { + last_comment = Some(comment); + } + Ok(entry) => { + body.push(ast::ResourceEntry::Entry(entry)); + } + Err(mut err) => { + ps.skip_to_next_entry_start(); + err.slice = Some((entry_start, ps.ptr)); + errors.push(err); + let slice = ps.get_slice(entry_start, ps.ptr); + body.push(ast::ResourceEntry::Junk(slice)); + } + } + last_blank_count = ps.skip_blank_block(); + } + + if let Some(last_comment) = last_comment.take() { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment(last_comment))); + } + if errors.is_empty() { + Ok(ast::Resource { body }) + } else { + Err((ast::Resource { body }, errors)) + } +} + +fn get_entry<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { + let entry = match ps.source[ps.ptr] { + b'#' => ast::Entry::Comment(get_comment(ps)?), + b'-' => ast::Entry::Term(get_term(ps, entry_start)?), + _ => ast::Entry::Message(get_message(ps, entry_start)?), + }; + Ok(entry) +} + +fn get_message<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + + let pattern = get_pattern(ps)?; + + ps.skip_blank_block(); + + let attributes = get_attributes(ps); + + if pattern.is_none() && attributes.is_empty() { + return error!( + ErrorKind::ExpectedMessageField { + entry_id: id.name.to_string() + }, + entry_start, ps.ptr + ); + } + + Ok(ast::Message { + id, + value: pattern, + attributes, + comment: None, + }) +} + +fn get_term<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { + ps.expect_byte(b'-')?; + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + ps.skip_blank_inline(); + + let value = get_value(ps)?; + + ps.skip_blank_block(); + + let attributes = get_attributes(ps); + + if let Some(value) = value { + Ok(ast::Term { + id, + value, + attributes, + comment: None, + }) + } else { + error!( + ErrorKind::ExpectedTermField { + entry_id: id.name.to_string() + }, + entry_start, ps.ptr + ) + } +} + +fn get_value<'p>(ps: &mut ParserStream<'p>) -> Result>> { + if ps.skip_to_value_start().is_none() { + return Ok(None); + } + + if ps.is_current_byte(b'{') { + let start = ps.ptr; + ps.ptr += 1; + ps.skip_blank(); + if ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { + let variants = get_variants(ps)?; + ps.expect_byte(b'}')?; + return Ok(Some(ast::Value::VariantList { variants })); + } + ps.ptr = start; + } + + let pattern = get_pattern(ps)?; + + Ok(pattern.map(ast::Value::Pattern)) +} + +fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Vec> { + let mut attributes = vec![]; + + loop { + let line_start = ps.ptr; + ps.skip_blank_inline(); + if !ps.is_current_byte(b'.') { + ps.ptr = line_start; + break; + } + + match get_attribute(ps) { + Ok(attr) => attributes.push(attr), + Err(_) => { + ps.ptr = line_start; + break; + } + } + } + attributes +} + +fn get_attribute<'p>(ps: &mut ParserStream<'p>) -> Result> { + ps.expect_byte(b'.')?; + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + let pattern = get_pattern(ps)?; + + match pattern { + Some(pattern) => Ok(ast::Attribute { id, value: pattern }), + None => error!(ErrorKind::MissingValue, ps.ptr), + } +} + +fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> { + let mut ptr = ps.ptr; + + match ps.source.get(ptr) { + Some(b) if ps.is_byte_alphabetic(*b) => { + ptr += 1; + } + _ => { + return error!( + ErrorKind::ExpectedCharRange { + range: "a-zA-Z".to_string() + }, + ptr + ); + } + } + + while let Some(b) = ps.source.get(ptr) { + if ps.is_byte_alphabetic(*b) || ps.is_byte_digit(*b) || [b'_', b'-'].contains(b) { + ptr += 1; + } else { + break; + } + } + + let name = ps.get_slice(ps.ptr, ptr); + ps.ptr = ptr; + + Ok(ast::Identifier { name }) +} + +fn get_variant_key<'p>(ps: &mut ParserStream<'p>) -> Result> { + if !ps.take_byte_if(b'[') { + return error!(ErrorKind::ExpectedToken('['), ps.ptr); + } + ps.skip_blank(); + + let key = if ps.is_number_start() { + ast::VariantKey::NumberLiteral { + value: get_number_literal(ps)?, + } + } else { + ast::VariantKey::Identifier { + name: get_identifier(ps)?.name, + } + }; + + ps.skip_blank(); + + ps.expect_byte(b']')?; + + Ok(key) +} + +fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> { + let mut variants = vec![]; + let mut has_default = false; + + while ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { + let default = ps.take_byte_if(b'*'); + + if default { + if has_default { + return error!(ErrorKind::MultipleDefaultVariants, ps.ptr); + } else { + has_default = true; + } + } + + let key = get_variant_key(ps)?; + + let value = get_pattern(ps)?; + + if let Some(value) = value { + variants.push(ast::Variant { + key, + value, + default, + }); + ps.skip_blank(); + } else { + return error!(ErrorKind::MissingValue, ps.ptr); + } + } + + if !has_default { + error!(ErrorKind::MissingDefaultVariant, ps.ptr) + } else { + Ok(variants) + } +} + +// This enum tracks the reason for which a text slice ended. +// It is used by the pattern to set the proper state for the next line. +// +// CRLF variant is specific because we want to skip the CR but keep the LF in text elements +// production. +// For example `a\r\n b` will produce (`a`, `\n` and ` b`) TextElements. +#[derive(Debug, PartialEq)] +enum TextElementTermination { + LineFeed, + CRLF, + PlaceableStart, + EOF, +} + +// This enum tracks the placement of the text element in the pattern, which is needed for +// dedentation logic. +#[derive(Debug, PartialEq)] +enum TextElementPosition { + InitialLineStart, + LineStart, + Continuation, +} + +// This enum allows us to mark pointers in the source which will later become text elements +// but without slicing them out of the source string. This makes the indentation adjustments +// cheaper since they'll happen on the pointers, rather than extracted slices. +#[derive(Debug)] +enum PatternElementPlaceholders<'a> { + Placeable(ast::Expression<'a>), + // (start, end, indent, position) + TextElement(usize, usize, usize, TextElementPosition), +} + +// This enum tracks whether the text element is blank or not. +// This is important to identify text elements which should not be taken into account +// when calculating common indent. +#[derive(Debug, PartialEq)] +enum TextElementType { + Blank, + NonBlank, +} + +fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result>> { + let mut elements = vec![]; + let mut last_non_blank = None; + let mut common_indent = None; + + ps.skip_blank_inline(); + + let mut text_element_role = if ps.skip_eol() { + ps.skip_blank_block(); + TextElementPosition::LineStart + } else { + TextElementPosition::InitialLineStart + }; + + while ps.ptr < ps.length { + if ps.is_current_byte(b'{') { + if text_element_role == TextElementPosition::LineStart { + common_indent = Some(0); + } + let exp = get_placeable(ps)?; + last_non_blank = Some(elements.len()); + elements.push(PatternElementPlaceholders::Placeable(exp)); + text_element_role = TextElementPosition::Continuation; + } else { + let slice_start = ps.ptr; + let mut indent = 0; + if text_element_role == TextElementPosition::LineStart { + indent = ps.skip_blank_inline(); + if ps.ptr >= ps.length { + break; + } + let b = ps.source[ps.ptr]; + if indent == 0 { + if b != b'\n' { + break; + } + } else if !ps.is_byte_pattern_continuation(b) { + ps.ptr = slice_start; + break; + } + } + let (start, end, text_element_type, termination_reason) = get_text_slice(ps)?; + if start != end { + if text_element_role == TextElementPosition::LineStart + && text_element_type == TextElementType::NonBlank + { + if let Some(common) = common_indent { + if indent < common { + common_indent = Some(indent); + } + } else { + common_indent = Some(indent); + } + } + if text_element_role != TextElementPosition::LineStart + || text_element_type == TextElementType::NonBlank + || termination_reason == TextElementTermination::LineFeed + { + if text_element_type == TextElementType::NonBlank { + last_non_blank = Some(elements.len()); + } + elements.push(PatternElementPlaceholders::TextElement( + slice_start, + end, + indent, + text_element_role, + )); + } + } + + text_element_role = match termination_reason { + TextElementTermination::LineFeed => TextElementPosition::LineStart, + TextElementTermination::CRLF => TextElementPosition::Continuation, + TextElementTermination::PlaceableStart => TextElementPosition::Continuation, + TextElementTermination::EOF => TextElementPosition::Continuation, + }; + } + } + + if let Some(last_non_blank) = last_non_blank { + let elements = elements + .into_iter() + .take(last_non_blank + 1) + .enumerate() + .filter_map(|(i, elem)| match elem { + PatternElementPlaceholders::Placeable(exp) => { + Some(ast::PatternElement::Placeable(exp)) + } + PatternElementPlaceholders::TextElement(start, end, indent, role) => { + let start = if role == TextElementPosition::LineStart { + if let Some(common_indent) = common_indent { + start + cmp::min(indent, common_indent) + } else { + start + indent + } + } else { + start + }; + let slice = ps.get_slice(start, end); + if last_non_blank == i { + if slice == "\n" { + return None; + } + Some(ast::PatternElement::TextElement(slice.trim_end())) + } else { + Some(ast::PatternElement::TextElement(slice)) + } + } + }) + .collect(); + return Ok(Some(ast::Pattern { elements })); + } + + Ok(None) +} + +fn get_text_slice<'p>( + ps: &mut ParserStream<'p>, +) -> Result<(usize, usize, TextElementType, TextElementTermination)> { + let start_pos = ps.ptr; + let mut text_element_type = TextElementType::Blank; + + while ps.ptr < ps.length { + match ps.source[ps.ptr] { + b' ' => ps.ptr += 1, + b'\n' => { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::LineFeed, + )); + } + b'\r' if ps.is_byte_at(b'\n', ps.ptr + 1) => { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr - 1, + text_element_type, + TextElementTermination::CRLF, + )); + } + b'{' => { + return Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::PlaceableStart, + )); + } + b'}' => { + return error!(ErrorKind::Generic, ps.ptr); + } + b'\\' => { + text_element_type = TextElementType::NonBlank; + match ps.source.get(ps.ptr) { + Some(b'\\') => ps.ptr += 1, + Some(b'u') => { + ps.ptr += 1; + ps.skip_unicode_escape_sequence(4)?; + } + _ => {} + } + } + _ => { + text_element_type = TextElementType::NonBlank; + ps.ptr += 1 + } + } + } + Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::EOF, + )) +} + +fn get_comment<'p>(ps: &mut ParserStream<'p>) -> Result> { + let mut level = None; + let mut content = vec![]; + + while ps.ptr < ps.length { + let line_level = get_comment_level(ps); + if line_level == 0 { + ps.ptr -= 1; + break; + } + if level.is_some() && Some(line_level) != level { + ps.ptr -= line_level; + break; + } + + level = Some(line_level); + + if ps.is_current_byte(b'\n') { + content.push(get_comment_line(ps)?); + } else { + ps.expect_byte(b' ')?; + content.push(get_comment_line(ps)?); + } + ps.skip_eol(); + } + + let comment = match level { + Some(3) => ast::Comment::ResourceComment { content }, + Some(2) => ast::Comment::GroupComment { content }, + _ => ast::Comment::Comment { content }, + }; + Ok(comment) +} + +fn get_comment_level<'p>(ps: &mut ParserStream<'p>) -> usize { + let mut chars = 0; + + while ps.take_byte_if(b'#') { + chars += 1; + } + + chars +} + +fn get_comment_line<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { + let start_pos = ps.ptr; + + while ps.ptr < ps.length && !ps.is_eol() { + ps.ptr += 1; + } + + Ok(ps.get_slice(start_pos, ps.ptr)) +} + +fn get_placeable<'p>(ps: &mut ParserStream<'p>) -> Result> { + ps.expect_byte(b'{')?; + ps.skip_blank(); + let exp = get_expression(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'}')?; + + let invalid_expression_found = match &exp { + ast::Expression::InlineExpression(ast::InlineExpression::AttributeExpression { + ref reference, + .. + }) => { + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false + } + } + ast::Expression::InlineExpression(ast::InlineExpression::CallExpression { + callee, .. + }) => { + if let ast::InlineExpression::AttributeExpression { .. } = **callee { + true + } else { + false + } + } + _ => false, + }; + if invalid_expression_found { + return error!(ErrorKind::TermAttributeAsPlaceable, ps.ptr); + } + + Ok(exp) +} + +fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let exp = get_call_expression(ps)?; + + ps.skip_blank(); + + if !ps.is_current_byte(b'-') || !ps.is_byte_at(b'>', ps.ptr + 1) { + return Ok(ast::Expression::InlineExpression(exp)); + } + + let is_valid = match exp { + ast::InlineExpression::StringLiteral { .. } => true, + ast::InlineExpression::NumberLiteral { .. } => true, + ast::InlineExpression::VariableReference { .. } => true, + ast::InlineExpression::AttributeExpression { ref reference, .. } => { + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false + } + } + ast::InlineExpression::CallExpression { ref callee, .. } => { + if let ast::InlineExpression::FunctionReference { .. } = **callee { + true + } else if let ast::InlineExpression::AttributeExpression { .. } = **callee { + true + } else { + false + } + } + _ => false, + }; + + if !is_valid { + //XXX: Give more specific error type + return error!(ErrorKind::MessageReferenceAsSelector, ps.ptr); + } + + ps.ptr += 2; // -> + + ps.skip_blank_inline(); + ps.expect_byte(b'\n')?; + ps.skip_blank(); + + let variants = get_variants(ps)?; + + Ok(ast::Expression::SelectExpression { + selector: exp, + variants, + }) +} + +fn get_call_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let mut callee = get_attribute_expression(ps)?; + + let expr = if ps.is_current_byte(b'(') { + let is_valid = match callee { + ast::InlineExpression::AttributeExpression { ref reference, .. } => { + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false + } + } + ast::InlineExpression::TermReference { .. } => true, + ast::InlineExpression::MessageReference { ref id, .. } => { + id.name.find(|c: char| c.is_ascii_lowercase()).is_none() + } + _ => false, + }; + + if !is_valid { + return error!(ErrorKind::ForbiddenCallee, ps.ptr); + } + + if let ast::InlineExpression::MessageReference { id } = callee { + callee = ast::InlineExpression::FunctionReference { id }; + } + let (positional, named) = get_call_args(ps)?; + ast::InlineExpression::CallExpression { + callee: Box::new(callee), + positional, + named, + } + } else { + callee + }; + + Ok(expr) +} + +fn get_attribute_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let reference = get_simple_expression(ps)?; + + match reference { + ast::InlineExpression::MessageReference { .. } + | ast::InlineExpression::TermReference { .. } => { + if ps.is_current_byte(b'.') { + ps.ptr += 1; // . + let attr = get_identifier(ps)?; + Ok(ast::InlineExpression::AttributeExpression { + reference: Box::new(reference), + name: attr, + }) + } else { + Ok(reference) + } + } + _ => Ok(reference), + } +} + +fn get_simple_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + match ps.source.get(ps.ptr) { + Some(b'"') => { + ps.ptr += 1; // " + let start = ps.ptr; + while ps.ptr < ps.length { + match ps.source[ps.ptr] { + b'\\' => match ps.source.get(ps.ptr + 1) { + Some(b'\\') => ps.ptr += 2, + Some(b'{') => ps.ptr += 2, + Some(b'"') => ps.ptr += 2, + Some(b'u') => { + ps.ptr += 2; + ps.skip_unicode_escape_sequence(4)?; + } + Some(b'U') => { + ps.ptr += 2; + ps.skip_unicode_escape_sequence(6)?; + } + _ => return error!(ErrorKind::Generic, ps.ptr), + }, + b'"' => { + break; + } + b'\n' => { + return error!(ErrorKind::Generic, ps.ptr); + } + _ => ps.ptr += 1, + } + } + + ps.expect_byte(b'"')?; + let slice = ps.get_slice(start, ps.ptr - 1); + Ok(ast::InlineExpression::StringLiteral { raw: slice }) + } + Some(b) if ps.is_byte_digit(*b) => { + let num = get_number_literal(ps)?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + Some(b'-') => { + ps.ptr += 1; // - + if ps.is_identifier_start() { + let id = get_identifier(ps)?; + match ps.source.get(ps.ptr) { + Some(b'[') => { + let key = get_variant_key(ps)?; + Ok(ast::InlineExpression::VariantExpression { + reference: Box::new(ast::InlineExpression::TermReference { id }), + key, + }) + } + _ => Ok(ast::InlineExpression::TermReference { id }), + } + } else { + ps.ptr -= 1; + let num = get_number_literal(ps)?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + } + Some(b'$') => { + ps.ptr += 1; // - + let id = get_identifier(ps)?; + Ok(ast::InlineExpression::VariableReference { id }) + } + Some(b) if ps.is_byte_alphabetic(*b) => { + let id = get_identifier(ps)?; + Ok(ast::InlineExpression::MessageReference { id }) + } + Some(b'{') => { + let exp = get_placeable(ps)?; + Ok(ast::InlineExpression::Placeable { + expression: Box::new(exp), + }) + } + _ => error!(ErrorKind::MissingLiteral, ps.ptr), + } +} + +fn get_call_args<'p>( + ps: &mut ParserStream<'p>, +) -> Result<(Vec>, Vec>)> { + let mut positional = vec![]; + let mut named = vec![]; + let mut argument_names = vec![]; + + ps.expect_byte(b'(')?; + ps.skip_blank(); + + while ps.ptr < ps.length { + if ps.is_current_byte(b')') { + break; + } + + let expr = get_call_expression(ps)?; + + match expr { + ast::InlineExpression::MessageReference { ref id } => { + ps.skip_blank(); + if ps.is_current_byte(b':') { + if argument_names.contains(&id.name.to_owned()) { + return error!( + ErrorKind::DuplicatedNamedArgument(id.name.to_owned()), + ps.ptr + ); + } + ps.ptr += 1; + ps.skip_blank(); + let val = get_call_expression(ps)?; + argument_names.push(id.name.to_owned()); + named.push(ast::NamedArgument { + name: ast::Identifier { name: id.name }, + value: val, + }); + } else { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, ps.ptr); + } + positional.push(expr); + } + } + _ => { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, ps.ptr); + } + positional.push(expr); + } + } + + ps.skip_blank(); + ps.take_byte_if(b','); + ps.skip_blank(); + } + + ps.expect_byte(b')')?; + Ok((positional, named)) +} + +fn get_number_literal<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { + let start = ps.ptr; + ps.take_byte_if(b'-'); + ps.skip_digits()?; + if ps.take_byte_if(b'.') { + ps.skip_digits()?; + } -pub use self::parser::parse; + Ok(ps.get_slice(start, ps.ptr)) +} diff --git a/fluent-syntax/src/parser/parser.rs b/fluent-syntax/src/parser/parser.rs deleted file mode 100644 index 9286a407..00000000 --- a/fluent-syntax/src/parser/parser.rs +++ /dev/null @@ -1,685 +0,0 @@ -pub use super::errors::get_error_info; -pub use super::errors::get_error_slice; -pub use super::errors::ErrorKind; -pub use super::errors::ParserError; - -use super::ftlstream::FTLParserStream; -use super::stream::ParserStream; - -use std::result; - -use super::super::ast; - -pub type Result = result::Result; - -pub fn parse(source: &str) -> result::Result)> { - let mut errors = vec![]; - - let mut ps = ParserStream::new(source.chars()); - - ps.skip_blank_lines(); - - let mut entries = vec![]; - - while ps.current().is_some() { - let entry_start_pos = ps.get_index(); - - match get_entry(&mut ps) { - Ok(entry) => { - entries.push(entry); - } - Err(mut e) => { - let error_pos = ps.get_index(); - entries.push(get_junk_entry(&mut ps, source, entry_start_pos)); - - e.info = get_error_info(source, error_pos, entry_start_pos, ps.get_index()); - errors.push(e); - } - } - - ps.skip_blank_lines(); - } - - if errors.is_empty() { - Ok(ast::Resource { body: entries }) - } else { - Err((ast::Resource { body: entries }, errors)) - } -} - -fn get_entry(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let comment = if ps.current_is('#') { - Some(get_comment(ps)?) - } else { - None - }; - - if ps.is_entry_id_start() { - match comment { - None | Some(ast::Comment::Comment { .. }) => { - return Ok(get_message(ps, comment)?); - } - _ => {} - }; - } - - match comment { - Some(comment) => Ok(ast::Entry::Comment(comment)), - None => error!(ErrorKind::ExpectedEntry), - } -} - -fn get_comment(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut level = -1; - let mut content = String::new(); - - loop { - let mut i = -1; - while ps.current_is('#') && ((level == -1 && i < 2) || (level != -1 && i < level)) { - ps.next(); - i += 1; - } - - if level == -1 { - level = i; - } - - if !ps.current_is('\n') { - ps.expect_char(' ')?; - while let Some(ch) = ps.take_char(|x| x != '\n') { - content.push(ch); - } - } - - if ps.is_peek_next_line_comment(level) { - content.push('\n'); - ps.next(); - } else { - break; - } - } - - match level { - 0 => Ok(ast::Comment::Comment { content }), - 1 => Ok(ast::Comment::GroupComment { content }), - 2 => Ok(ast::Comment::ResourceComment { content }), - _ => panic!("Unknown comment level!"), - } -} - -fn get_message(ps: &mut ParserStream, comment: Option) -> Result -where - I: Iterator, -{ - let id = get_entry_identifier(ps)?; - - ps.skip_inline_ws(); - - ps.expect_char('=')?; - - let pattern = if ps.is_peek_pattern_start() { - ps.skip_indent(); - get_pattern(ps)? - } else { - None - }; - - let attributes = if ps.is_peek_next_line_attribute_start() { - Some(get_attributes(ps)?) - } else { - None - }; - - if id.name.starts_with('-') { - match pattern { - Some(pattern) => { - return Ok(ast::Entry::Term(ast::Term { - id, - value: pattern, - attributes, - comment, - })); - } - None => { - return error!(ErrorKind::ExpectedTermField { entry_id: id.name }); - } - } - } - - if pattern.is_none() && attributes.is_none() { - return error!(ErrorKind::ExpectedMessageField { entry_id: id.name }); - } - - Ok(ast::Entry::Message(ast::Message { - id, - value: pattern, - attributes, - comment, - })) -} - -fn get_attribute(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - ps.expect_char('.')?; - - let key = get_identifier(ps, false)?; - - ps.skip_inline_ws(); - ps.expect_char('=')?; - - if ps.is_peek_pattern_start() { - ps.skip_indent(); - let value = get_pattern(ps)?; - if let Some(value) = value { - return Ok(ast::Attribute { id: key, value }); - } - } - error!(ErrorKind::MissingValue) -} - -fn get_attributes(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut attributes = vec![]; - loop { - ps.expect_indent()?; - let attr = get_attribute(ps)?; - attributes.push(attr); - - if !ps.is_peek_next_line_attribute_start() { - break; - } - } - Ok(attributes) -} - -fn get_entry_identifier(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - get_identifier(ps, true) -} - -fn get_identifier(ps: &mut ParserStream, allow_term: bool) -> Result -where - I: Iterator, -{ - let mut name = String::new(); - - name.push(ps.take_id_start(allow_term)?); - - while let Some(ch) = ps.take_id_char() { - name.push(ch); - } - - Ok(ast::Identifier { name }) -} - -fn get_variant_key(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if let Some(ch) = ps.current() { - match ch { - '0'...'9' | '-' => { - return Ok(ast::VarKey::Number(get_number(ps)?)); - } - _ => { - return Ok(ast::VarKey::VariantName(get_variant_name(ps)?)); - } - } - } else { - return error!(ErrorKind::MissingVariantKey); - } -} - -fn get_variant(ps: &mut ParserStream, has_default: bool) -> Result -where - I: Iterator, -{ - let default_index = if ps.current_is('*') { - if has_default { - return error!(ErrorKind::MultipleDefaultVariants); - } - ps.next(); - true - } else { - false - }; - - ps.expect_char('[')?; - - let key = get_variant_key(ps)?; - - ps.expect_char(']')?; - - if ps.is_peek_pattern_start() { - ps.skip_indent(); - if let Some(value) = get_pattern(ps)? { - return Ok(ast::Variant { - key, - value, - default: default_index, - }); - } - } - return error!(ErrorKind::MissingValue); -} - -fn get_variants(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut variants = vec![]; - let mut has_default = false; - - loop { - ps.expect_indent()?; - let variant = get_variant(ps, has_default)?; - - if variant.default { - has_default = true; - } - - variants.push(variant); - - if !ps.is_peek_next_line_variant_start() { - break; - } - } - if !has_default { - return error!(ErrorKind::MissingDefaultVariant); - } - Ok(variants) -} - -fn get_variant_name(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut name = String::new(); - - name.push(ps.take_id_start(false)?); - - while let Some(ch) = ps.take_variant_name_char() { - name.push(ch); - } - - while name.ends_with(' ') { - name.pop(); - } - - Ok(ast::VariantName { name }) -} - -fn get_digits(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut num = String::new(); - - while let Some(ch) = ps.take_digit() { - num.push(ch); - } - - if num.is_empty() { - return error!(ErrorKind::ExpectedCharRange { - range: "0...9".to_owned(), - }); - } - - Ok(num) -} - -fn get_number(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut num = String::new(); - - if ps.current_is('-') { - num.push('-'); - ps.next(); - } - - num.push_str(&get_digits(ps)?); - - if ps.current_is('.') { - num.push('.'); - ps.next(); - num.push_str(&get_digits(ps)?); - } - Ok(ast::Number { value: num }) -} - -fn get_pattern(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut elements = vec![]; - - ps.skip_inline_ws(); - - while let Some(ch) = ps.current() { - if ch == '\n' && !ps.is_peek_next_line_pattern_start() { - break; - } - - match ch { - '{' => { - elements.push(get_placeable(ps)?); - } - _ => { - elements.push(get_text_element(ps)?); - } - } - } - - Ok(Some(ast::Pattern { elements })) -} - -fn get_text_element(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut buf = String::new(); - - while let Some(ch) = ps.current() { - match ch { - '{' => return Ok(ast::PatternElement::TextElement(buf)), - '\n' => { - if !ps.is_peek_next_line_pattern_start() { - return Ok(ast::PatternElement::TextElement(buf)); - } - ps.next(); - ps.skip_inline_ws(); - - // Add the new line to the buffer - buf.push(ch); - continue; - } - '\\' => { - if let Some(ch2) = ps.next() { - if ch2 == '{' || ch2 == '"' { - buf.push(ch2); - } else { - buf.push(ch); - buf.push(ch2); - } - } - } - _ => buf.push(ch), - } - ps.next(); - } - - Ok(ast::PatternElement::TextElement(buf)) -} - -fn get_placeable(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - ps.expect_char('{')?; - let expression = get_expression(ps)?; - ps.expect_char('}')?; - Ok(ast::PatternElement::Placeable(expression)) -} - -fn get_expression(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if ps.is_peek_next_line_variant_start() { - let variants = get_variants(ps)?; - - ps.expect_indent()?; - - return Ok(ast::Expression::SelectExpression { - expression: None, - variants, - }); - } - - ps.skip_inline_ws(); - - let selector = get_selector_expression(ps)?; - - ps.skip_inline_ws(); - - if ps.current_is('-') { - ps.peek(); - - if !ps.current_peek_is('>') { - ps.reset_peek(None); - return Ok(selector); - } - - match selector { - ast::Expression::MessageReference { .. } => { - return error!(ErrorKind::MessageReferenceAsSelector) - } - ast::Expression::AttributeExpression { ref id, .. } => { - if !id.name.starts_with('-') { - return error!(ErrorKind::MessageAttributeAsSelector); - } - } - ast::Expression::VariantExpression { .. } => { - return error!(ErrorKind::VariantAsSelector) - } - _ => {} - }; - - ps.next(); - ps.next(); - - ps.skip_inline_ws(); - - let variants = get_variants(ps)?; - - if variants.is_empty() { - return error!(ErrorKind::MissingVariants); - } - - ps.expect_indent()?; - - return Ok(ast::Expression::SelectExpression { - expression: Some(Box::new(selector)), - variants, - }); - } else if let ast::Expression::AttributeExpression { ref id, .. } = selector { - if id.name.starts_with('-') { - return error!(ErrorKind::TermAttributeAsSelector); - } - } - - Ok(selector) -} - -fn get_selector_expression(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let literal = get_literal(ps)?; - - match literal { - ast::Expression::MessageReference { id } => match ps.ch { - Some('.') => { - ps.next(); - let attr = get_identifier(ps, false)?; - Ok(ast::Expression::AttributeExpression { id, name: attr }) - } - Some('[') => { - ps.next(); - let key = get_variant_key(ps)?; - ps.expect_char(']')?; - - Ok(ast::Expression::VariantExpression { id, key }) - } - Some('(') => { - if id.name.starts_with('-') || id.name.chars().any(|c| c.is_lowercase()) { - return error!(ErrorKind::ForbiddenCallee); - } - ps.next(); - let args = get_call_args(ps)?; - ps.expect_char(')')?; - - // XXX Make sure that id.name is [A-Z][A-Z_?-]* - Ok(ast::Expression::CallExpression { - callee: ast::Function { name: id.name }, - args, - }) - } - _ => Ok(ast::Expression::MessageReference { id }), - }, - _ => Ok(literal), - } -} - -fn get_call_arg(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let exp = get_selector_expression(ps)?; - - if !ps.current_is(':') { - return Ok(ast::Argument::Expression(exp)); - } - - match exp { - ast::Expression::MessageReference { id } => { - ps.next(); - ps.skip_inline_ws(); - - let val = get_arg_val(ps)?; - Ok(ast::Argument::NamedArgument { name: id, val }) - } - _ => error!(ErrorKind::ForbiddenKey), - } -} - -fn get_call_args(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut args = vec![]; - - ps.skip_inline_ws(); - - loop { - if ps.current_is(')') { - break; - } - - let arg = get_call_arg(ps)?; - args.push(arg); - - ps.skip_inline_ws(); - - if ps.current_is(',') { - ps.next(); - ps.skip_inline_ws(); - continue; - } else { - break; - } - } - - Ok(args) -} - -fn get_arg_val(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if ps.is_number_start() { - return Ok(ast::ArgValue::Number(get_number(ps)?)); - } else if ps.current_is('"') { - return Ok(ast::ArgValue::String(get_string(ps)?)); - } - error!(ErrorKind::MissingValue) -} - -fn get_string(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut val = String::new(); - - ps.expect_char('"')?; - - while let Some(ch) = ps.take_char(|x| x != '"' && x != '\n') { - val.push(ch); - } - - if ps.current_is('\n') { - return error!(ErrorKind::UnterminatedStringExpression); - } - - ps.next(); - - Ok(val) -} - -fn get_literal(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if let Some(ch) = ps.current() { - let exp = match ch { - '0'...'9' => ast::Expression::NumberExpression { - value: get_number(ps)?, - }, - '-' => { - if let Some('0'...'9') = ps.peek() { - ps.reset_peek(None); - ast::Expression::NumberExpression { - value: get_number(ps)?, - } - } else { - ps.reset_peek(None); - ast::Expression::MessageReference { - id: get_entry_identifier(ps)?, - } - } - } - '"' => ast::Expression::StringExpression { - value: get_string(ps)?, - }, - '$' => { - ps.next(); - ast::Expression::ExternalArgument { - id: get_identifier(ps, false)?, - } - } - _ => ast::Expression::MessageReference { - id: get_entry_identifier(ps)?, - }, - }; - Ok(exp) - } else { - return error!(ErrorKind::MissingLiteral); - } -} - -fn get_junk_entry(ps: &mut ParserStream, source: &str, entry_start: usize) -> ast::Entry -where - I: Iterator, -{ - ps.skip_to_next_entry_start(); - - let slice = get_error_slice(source, entry_start, ps.get_index()); - - ast::Entry::Junk { - content: String::from(slice), - } -} diff --git a/fluent-syntax/src/parser/stream.rs b/fluent-syntax/src/parser/stream.rs deleted file mode 100644 index 816413c8..00000000 --- a/fluent-syntax/src/parser/stream.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::iter::Fuse; - -#[derive(Clone, Debug)] -pub struct ParserStream -where - I: Iterator, -{ - iter: Fuse, - pub buf: Vec, - peek_index: usize, - index: usize, - - pub ch: Option, - - iter_end: bool, - peek_end: bool, -} - -impl> ParserStream { - pub fn new(iterable: I) -> ParserStream { - let mut iter = iterable.into_iter().fuse(); - let ch = iter.next(); - - ParserStream { - iter, - buf: vec![], - peek_index: 0, - index: 0, - ch, - - iter_end: false, - peek_end: false, - } - } - - pub fn current(&mut self) -> Option { - self.ch - } - - pub fn current_is(&mut self, ch: char) -> bool { - self.ch == Some(ch) - } - - pub fn current_peek(&self) -> Option { - if self.peek_end { - return None; - } - - let diff = self.peek_index - self.index; - - if diff == 0 { - self.ch - } else { - Some(self.buf[diff - 1]) - } - } - - pub fn current_peek_is(&mut self, ch: char) -> bool { - self.current_peek() == Some(ch) - } - - pub fn peek(&mut self) -> Option { - if self.peek_end { - return None; - } - - self.peek_index += 1; - - let diff = self.peek_index - self.index; - - if diff > self.buf.len() { - match self.iter.next() { - Some(c) => { - self.buf.push(c); - } - None => { - self.peek_end = true; - return None; - } - } - } - - Some(self.buf[diff - 1]) - } - - pub fn get_index(&self) -> usize { - self.index - } - - pub fn get_peek_index(&self) -> usize { - self.peek_index - } - - pub fn peek_char_is(&mut self, ch: char) -> bool { - if self.peek_end { - return false; - } - - let ret = self.peek() == Some(ch); - - self.peek_index -= 1; - ret - } - - pub fn reset_peek(&mut self, pos: Option) { - match pos { - Some(pos) => { - if pos < self.peek_index { - self.peek_end = false - } - self.peek_index = pos - } - None => { - self.peek_index = self.index; - self.peek_end = self.iter_end; - } - } - } - - pub fn skip_to_peek(&mut self) { - let diff = self.peek_index - self.index; - - for _ in 0..diff { - self.ch = Some(self.buf.remove(0)); - } - - self.index = self.peek_index; - } -} - -impl Iterator for ParserStream -where - I: Iterator, -{ - type Item = char; - - fn next(&mut self) -> Option { - if self.iter_end { - return None; - } - - self.ch = if self.buf.is_empty() { - self.iter.next() - } else { - Some(self.buf.remove(0)) - }; - - self.index += 1; - - if self.ch.is_none() { - self.iter_end = true; - self.peek_end = true; - } - - self.peek_index = self.index; - - self.ch - } -} diff --git a/fluent-syntax/tests/ast/helper.rs b/fluent-syntax/tests/ast/helper.rs new file mode 100644 index 00000000..243a33d8 --- /dev/null +++ b/fluent-syntax/tests/ast/helper.rs @@ -0,0 +1,67 @@ +use std::char; +use std::collections::VecDeque; + +fn encode_unicode(s: &str, l: usize) -> char { + if s.len() != l { + return '�'; + } + let u = match u32::from_str_radix(s, 16) { + Ok(u) => u, + Err(_) => return '�', + }; + char::from_u32(u).unwrap_or('�') +} + +pub fn unescape_unicode(s: &str) -> String { + let mut queue: VecDeque<_> = String::from(s).chars().collect(); + let mut result = String::new(); + + while let Some(c) = queue.pop_front() { + if c != '\\' { + result.push(c); + continue; + } + + match queue.pop_front() { + Some('u') => { + let mut buffer = String::new(); + for _ in 0..4 { + if let Some(c) = queue.pop_front() { + match c { + '0'...'9' | 'a'...'f' | 'A'...'F' => { + buffer.push(c); + } + _ => break, + } + } else { + break; + } + } + let new_char = encode_unicode(&buffer, 4); + result.push(new_char); + } + Some('U') => { + let mut buffer = String::new(); + for _ in 0..6 { + if let Some(c) = queue.pop_front() { + match c { + '0'...'9' | 'a'...'f' | 'A'...'F' => { + buffer.push(c); + } + _ => break, + } + } else { + break; + } + } + let new_char = encode_unicode(&buffer, 6); + result.push(new_char); + } + Some(c) => { + result.push(c); + } + None => break, + } + } + result +} diff --git a/fluent-syntax/tests/ast/mod.rs b/fluent-syntax/tests/ast/mod.rs new file mode 100644 index 00000000..807b850b --- /dev/null +++ b/fluent-syntax/tests/ast/mod.rs @@ -0,0 +1,446 @@ +mod helper; + +use fluent_syntax::ast; +use serde::ser::SerializeMap; +use serde::ser::SerializeSeq; +use serde::{Serialize, Serializer}; +use serde_derive::Serialize; +use std::error::Error; + +pub fn serialize<'s>(res: &'s ast::Resource) -> Result> { + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); + Ok(serde_json::to_string(&Helper(res)).unwrap()) +} + +pub fn _serialize_to_pretty_json<'s>(res: &'s ast::Resource) -> Result> { + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); + + let buf = Vec::new(); + let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); + let mut ser = serde_json::Serializer::with_formatter(buf, formatter); + Helper(res).serialize(&mut ser).unwrap(); + Ok(String::from_utf8(ser.into_inner()).unwrap()) +} + +#[derive(Serialize)] +#[serde(remote = "ast::Resource")] +#[serde(tag = "type")] +#[serde(rename = "Resource")] +pub struct ResourceDef<'ast> { + #[serde(serialize_with = "serialize_resource_entry_vec")] + pub body: Vec>, +} + +fn serialize_resource_entry_vec<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + #[serde(tag = "type")] + enum EntryHelper<'ast> { + Junk { + annotations: Vec<&'ast str>, + content: &'ast str, + }, + #[serde(with = "MessageDef")] + Message(&'ast ast::Message<'ast>), + #[serde(with = "TermDef")] + Term(&'ast ast::Term<'ast>), + Comment { + content: String, + }, + GroupComment { + content: String, + }, + ResourceComment { + content: String, + }, + } + + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + let entry = match *e { + ast::ResourceEntry::Entry(ref entry) => match entry { + ast::Entry::Message(ref msg) => EntryHelper::Message(msg), + ast::Entry::Term(ref term) => EntryHelper::Term(term), + ast::Entry::Comment(ast::Comment::Comment { ref content }) => { + EntryHelper::Comment { + content: content.join("\n"), + } + } + ast::Entry::Comment(ast::Comment::GroupComment { ref content }) => { + EntryHelper::GroupComment { + content: content.join("\n"), + } + } + ast::Entry::Comment(ast::Comment::ResourceComment { ref content }) => { + EntryHelper::ResourceComment { + content: content.join("\n"), + } + } + }, + ast::ResourceEntry::Junk(ref junk) => EntryHelper::Junk { + content: junk, + annotations: vec![], + }, + }; + seq.serialize_element(&entry)?; + } + seq.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::Message")] +pub struct MessageDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(serialize_with = "serialize_pattern_option")] + pub value: Option>, + #[serde(serialize_with = "serialize_attribute_vec")] + pub attributes: Vec>, + #[serde(serialize_with = "serialize_comment_option")] + pub comment: Option>, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Identifier")] +#[serde(tag = "type")] +#[serde(rename = "Identifier")] +pub struct IdentifierDef<'ast> { + pub name: &'ast str, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Variant")] +#[serde(tag = "type")] +#[serde(rename = "Variant")] +pub struct VariantDef<'ast> { + #[serde(with = "VariantKeyDef")] + pub key: ast::VariantKey<'ast>, + #[serde(with = "PatternDef")] + pub value: ast::Pattern<'ast>, + pub default: bool, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Term")] +pub struct TermDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(with = "ValueDef")] + pub value: ast::Value<'ast>, + #[serde(serialize_with = "serialize_attribute_vec")] + pub attributes: Vec>, + #[serde(serialize_with = "serialize_comment_option")] + pub comment: Option>, +} + +fn serialize_pattern_option<'se, S>( + v: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "PatternDef")] &'ast ast::Pattern<'ast>); + v.as_ref().map(Helper).serialize(serializer) +} + +fn serialize_attribute_vec<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "AttributeDef")] &'ast ast::Attribute<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_comment_option<'se, S>( + v: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "CommentDef")] &'ast ast::Comment<'ast>); + v.as_ref().map(Helper).serialize(serializer) +} + +#[derive(Serialize)] +#[serde(remote = "ast::Value")] +#[serde(tag = "type")] +pub enum ValueDef<'ast> { + #[serde(with = "PatternDef")] + Pattern(ast::Pattern<'ast>), + VariantList { + #[serde(serialize_with = "serialize_variants")] + variants: Vec>, + }, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Pattern")] +#[serde(tag = "type")] +#[serde(rename = "Pattern")] +pub struct PatternDef<'ast> { + #[serde(serialize_with = "serialize_pattern_elements")] + pub elements: Vec>, +} + +fn serialize_pattern_elements<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "PatternElementDef")] &'ast ast::PatternElement<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + let mut buffer = String::new(); + for e in v { + match e { + ast::PatternElement::TextElement(e) => { + buffer.push_str(e); + } + _ => { + if !buffer.is_empty() { + seq.serialize_element(&Helper(&ast::PatternElement::TextElement(&buffer)))?; + buffer = String::new(); + } + + seq.serialize_element(&Helper(e))?; + } + } + } + if !buffer.is_empty() { + seq.serialize_element(&Helper(&ast::PatternElement::TextElement(&buffer)))?; + } + seq.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::PatternElement")] +#[serde(untagged)] +pub enum PatternElementDef<'ast> { + #[serde(serialize_with = "serialize_text_element")] + TextElement(&'ast str), + #[serde(serialize_with = "serialize_placeable")] + Placeable(ast::Expression<'ast>), +} + +fn serialize_text_element<'se, S>(s: &'se str, serializer: S) -> Result +where + S: Serializer, +{ + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "TextElement")?; + map.serialize_entry("value", s)?; + map.end() +} + +fn serialize_placeable<'se, S>(exp: &ast::Expression<'se>, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "ExpressionDef")] &'ast ast::Expression<'ast>); + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "Placeable")?; + map.serialize_entry("expression", &Helper(exp))?; + map.end() +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::VariantKey")] +#[serde(tag = "type")] +pub enum VariantKeyDef<'ast> { + Identifier { name: &'ast str }, + NumberLiteral { value: &'ast str }, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Comment")] +#[serde(tag = "type")] +pub enum CommentDef<'ast> { + Comment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, + GroupComment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, + ResourceComment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, +} + +fn serialize_comment_content<'se, S>(v: &Vec<&'se str>, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&v.join("\n")) +} + +#[derive(Serialize)] +#[serde(remote = "ast::InlineExpression")] +#[serde(tag = "type")] +pub enum InlineExpressionDef<'ast> { + #[serde(serialize_with = "serialize_string_literal")] + StringLiteral { + raw: &'ast str, + }, + NumberLiteral { + value: &'ast str, + }, + VariableReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + CallExpression { + #[serde(with = "InlineExpressionDef")] + callee: ast::InlineExpression<'ast>, + #[serde(serialize_with = "serialize_inline_expressions")] + positional: Vec>, + #[serde(serialize_with = "serialize_named_arguments")] + named: Vec>, + }, + AttributeExpression { + #[serde(with = "InlineExpressionDef")] + #[serde(rename = "ref")] + reference: ast::InlineExpression<'ast>, + #[serde(with = "IdentifierDef")] + name: ast::Identifier<'ast>, + }, + VariantExpression { + #[serde(with = "InlineExpressionDef")] + #[serde(rename = "ref")] + reference: ast::InlineExpression<'ast>, + #[serde(with = "VariantKeyDef")] + key: ast::VariantKey<'ast>, + }, + MessageReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + TermReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + FunctionReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + Placeable { + #[serde(with = "ExpressionDef")] + expression: ast::Expression<'ast>, + }, +} + +fn serialize_string_literal<'se, S>(raw: &'se str, serializer: S) -> Result +where + S: Serializer, +{ + let mut map = serializer.serialize_map(Some(3))?; + map.serialize_entry("type", "StringLiteral")?; + map.serialize_entry("raw", raw)?; + map.serialize_entry("value", &helper::unescape_unicode(&raw))?; + map.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::Attribute")] +#[serde(tag = "type")] +#[serde(rename = "Attribute")] +pub struct AttributeDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(with = "PatternDef")] + pub value: ast::Pattern<'ast>, +} + +#[derive(Serialize)] +#[serde(remote = "ast::NamedArgument")] +#[serde(tag = "type")] +#[serde(rename = "NamedArgument")] +pub struct NamedArgumentDef<'ast> { + #[serde(with = "IdentifierDef")] + pub name: ast::Identifier<'ast>, + #[serde(with = "InlineExpressionDef")] + pub value: ast::InlineExpression<'ast>, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Expression")] +#[serde(tag = "type")] +pub enum ExpressionDef<'ast> { + #[serde(with = "InlineExpressionDef")] + InlineExpression(ast::InlineExpression<'ast>), + SelectExpression { + #[serde(with = "InlineExpressionDef")] + selector: ast::InlineExpression<'ast>, + #[serde(serialize_with = "serialize_variants")] + variants: Vec>, + }, +} + +fn serialize_variants<'se, S>(v: &Vec>, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "VariantDef")] &'ast ast::Variant<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_inline_expressions<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "InlineExpressionDef")] &'ast ast::InlineExpression<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_named_arguments<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "NamedArgumentDef")] &'ast ast::NamedArgument<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} diff --git a/fluent-syntax/tests/errors.rs b/fluent-syntax/tests/errors.rs deleted file mode 100644 index 76cf5f64..00000000 --- a/fluent-syntax/tests/errors.rs +++ /dev/null @@ -1,291 +0,0 @@ -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::errors::display::annotate_error; -use self::fluent_syntax::parser::errors::ErrorInfo; -use self::fluent_syntax::parser::errors::ErrorKind; -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -#[test] -fn empty_errors() { - let path = "./tests/fixtures/parser/ftl/errors/01-empty.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedEntry, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: " key = value".to_owned(), - line: 0, - pos: 0, - },), - error1.info - ); - } - } -} - -#[test] -fn bad_id_start_errors() { - let path = "./tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedEntry, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "2".to_owned(), - line: 0, - pos: 0, - },), - error1.info - ); - } - } -} - -#[test] -fn just_id_errors() { - let path = "./tests/fixtures/parser/ftl/errors/03-just-id.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key".to_owned(), - line: 0, - pos: 3, - },), - error1.info - ); - } - } -} - -#[test] -fn no_equal_sign_errors() { - let path = "./tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '=' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key Value".to_owned(), - line: 0, - pos: 4, - },), - error1.info - ); - } - } -} - -#[test] -fn wrong_char_in_id_errors() { - let path = "./tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "'a'...'z' | 'A'...'Z'".to_owned(), - }, - error1.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key = Value\n .# = Foo".to_owned(), - line: 0, - pos: 14, - },), - error1.info - ); - } - } -} - -#[test] -fn missing_trait_value_errors() { - let path = "./tests/fixtures/parser/ftl/errors/06-trait-value.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key = Value\n .foo".to_owned(), - line: 0, - pos: 17, - },), - error1.info - ); - } - } -} - -#[test] -fn message_missing_fields_errors() { - let path = "./tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key".to_owned(), - line: 0, - pos: 3, - },), - error1.info - ); - } - } -} - -#[test] -fn private_errors() { - let path = "./tests/fixtures/parser/ftl/errors/08-private.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(4, errors.len()); - - let error1 = &errors[0]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "0...9".to_owned(), - }, - error1.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key =\n { $foo ->\n [one] Foo\n *[-other] Foo 2\n }" - .to_owned(), - line: 1, - pos: 48, - },), - error1.info - ); - - let error2 = &errors[1]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "'a'...'z' | 'A'...'Z'".to_owned(), - }, - error2.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key2 = { $-foo }".to_owned(), - line: 7, - pos: 10, - },), - error2.info - ); - - let error3 = &errors[2]; - - assert_eq!(ErrorKind::TermAttributeAsSelector, error3.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key3 = { -brand.gender }".to_owned(), - line: 9, - pos: 23, - },), - error3.info - ); - - let error4 = &errors[3]; - - assert_eq!(ErrorKind::ForbiddenCallee, error4.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key4 = { -brand() }".to_owned(), - line: 11, - pos: 15, - },), - error4.info - ); - } - } -} - -#[test] -fn test_annotate_errors() { - let input = "key Value"; - - let res = parse(input); - - match res { - Ok(_) => panic!("Should have return an error!"), - Err((_, errors)) => { - assert_eq!(errors.len(), 1); - let err = annotate_error(&errors[0], &None, false); - assert_eq!( - err, - "error[E0003]: expected token `=`\n |\n0 | key Value\n | ^\n |" - ); - } - } -} diff --git a/fluent-syntax/tests/fixtures.rs b/fluent-syntax/tests/fixtures.rs deleted file mode 100644 index b6ceab51..00000000 --- a/fluent-syntax/tests/fixtures.rs +++ /dev/null @@ -1,67 +0,0 @@ -extern crate fluent_syntax; -extern crate glob; - -use self::glob::glob; -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -fn attempt_parse(source: &str) -> Result<(), ()> { - match parse(source) { - Ok(_) => Ok(()), - Err(_) => Err(()), - } -} - -#[test] -fn parse_ftl() { - for entry in glob("./tests/fixtures/parser/ftl/*.ftl").expect("Failed to read glob pattern") { - let p = entry.expect("Error while getting an entry"); - let path = p.to_str().expect("Can't print path"); - - if path.contains("errors") { - continue; - } - - println!("Attempting to parse file: {}", path); - - let string = read_file(path).expect("Failed to read"); - - attempt_parse(&string).expect("Failed to parse"); - } -} - -#[test] -fn error_ftl() { - for entry in glob("./tests/fixtures/parser/ftl/*.ftl").expect("Failed to read glob pattern") { - let p = entry.expect("Error while getting an entry"); - let path = p.to_str().expect("Can't print path"); - - if !path.contains("errors") { - continue; - } - - println!("Attempting to parse error file: {}", path); - - let string = read_file(path).expect("Failed to read"); - - let chunks = string.split("\n\n"); - - for chunk in chunks { - println!("Testing chunk: {:?}", chunk); - match attempt_parse(chunk) { - Ok(_) => panic!("Test didn't fail"), - Err(_) => continue, - } - } - } -} diff --git a/fluent-syntax/tests/fixtures/any_char.ftl b/fluent-syntax/tests/fixtures/any_char.ftl new file mode 100644 index 00000000..6966a0da --- /dev/null +++ b/fluent-syntax/tests/fixtures/any_char.ftl @@ -0,0 +1,8 @@ +# ↓ BEL, U+0007 +control0 = abcdef + +# ↓ DEL, U+007F +delete = abcdef + +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/fluent-syntax/tests/fixtures/any_char.json b/fluent-syntax/tests/fixtures/any_char.json new file mode 100644 index 00000000..07e7dc4b --- /dev/null +++ b/fluent-syntax/tests/fixtures/any_char.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc\u0007def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BEL, U+0007" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "delete" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcdef" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ DEL, U+007F" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc‚def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BPM, U+0082" + } + } + ] +} diff --git a/fluent-syntax/tests/fixtures/astral.ftl b/fluent-syntax/tests/fixtures/astral.ftl new file mode 100644 index 00000000..b77e32e3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/astral.ftl @@ -0,0 +1,20 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 + +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = {"\uD83D\uDE02"} +surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} + +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = {"A face 😂 with tears of joy."} + +# ERROR Invalid identifier +err-😂 = Value + +# ERROR Invalid expression +err-invalid-expression = { 😂 } + +# ERROR Invalid variant key +err-invalid-variant-key = { $sel -> + *[😂] Value +} diff --git a/fluent-syntax/tests/fixtures/astral.json b/fluent-syntax/tests/fixtures/astral.json new file mode 100644 index 00000000..b69743ca --- /dev/null +++ b/fluent-syntax/tests/fixtures/astral.json @@ -0,0 +1,178 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "face-with-tears-of-joy" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "😂" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "tetragram-for-centre" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "𝌆" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\uD83D\\uDE02" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uD83D\\uDE02", + "value": "��" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-adjacent-strings" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uD83D", + "value": "�" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uDE02", + "value": "�" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A face 😂 with tears of joy." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "A face 😂 with tears of joy.", + "value": "A face 😂 with tears of joy." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Invalid identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-😂 = Value\n\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid expression" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-expression = { 😂 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid variant key" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/call_expressions.ftl b/fluent-syntax/tests/fixtures/call_expressions.ftl new file mode 100644 index 00000000..19e76b7e --- /dev/null +++ b/fluent-syntax/tests/fixtures/call_expressions.ftl @@ -0,0 +1,104 @@ +## Arguments + +positional-args = {FUN(1, "a", msg)} +named-args = {FUN(x: 1, y: "Y")} +dense-named-args = {FUN(x:1, y:"Y")} +mixed-args = {FUN(1, "a", msg, x: 1, y: "Y")} + +# ERROR Positional arg must not follow keyword args +shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} + +# ERROR Named arguments must be unique +duplicate-named-args = {FUN(x: 1, x: "X")} + + +## Whitespace around arguments + +sparse-inline-call = {FUN( "a" , msg, x: 1 )} +empty-inline-call = {FUN( )} +multiline-call = {FUN( + "a", + msg, + x: 1 + )} +sparse-multiline-call = {FUN( + + "a" , + msg + , x: 1 + )} +empty-multiline-call = {FUN( + + )} + + +unindented-arg-number = {FUN( +1)} + +unindented-arg-string = {FUN( +"a")} + +unindented-arg-msg-ref = {FUN( +msg)} + +unindented-arg-term-ref = {FUN( +-msg)} + +unindented-arg-var-ref = {FUN( +$var)} + +unindented-arg-call = {FUN( +OTHER())} + +unindented-named-arg = {FUN( +x:1)} + +unindented-closing-paren = {FUN( + x +)} + + + +## Optional trailing comma + +one-argument = {FUN(1,)} +many-arguments = {FUN(1, 2, 3,)} +inline-sparse-args = {FUN( 1, 2, 3, )} +mulitline-args = {FUN( + 1, + 2, + )} +mulitline-sparse-args = {FUN( + + 1 + , + 2 + , + )} + + +## Syntax errors for trailing comma + +one-argument = {FUN(1,,)} +missing-arg = {FUN(,)} +missing-sparse-arg = {FUN( , )} + + +## Whitespace in named arguments + +sparse-named-arg = {FUN( + x : 1, + y : 2, + z + : + 3 + )} + + +unindented-colon = {FUN( + x +:1)} + +unindented-value = {FUN( + x: +1)} diff --git a/fluent-syntax/tests/fixtures/call_expressions.json b/fluent-syntax/tests/fixtures/call_expressions.json new file mode 100644 index 00000000..4dd73483 --- /dev/null +++ b/fluent-syntax/tests/fixtures/call_expressions.json @@ -0,0 +1,1139 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "positional-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "named-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "dense-named-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mixed-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Positional arg must not follow keyword args" + }, + { + "type": "Junk", + "annotations": [], + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n\n" + }, + { + "type": "Comment", + "content": "ERROR Named arguments must be unique" + }, + { + "type": "Junk", + "annotations": [], + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Whitespace around arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-inline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-inline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-number" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-msg-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-term-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-var-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "OTHER" + } + }, + "positional": [], + "named": [] + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-named-arg" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-closing-paren" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "x" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Optional trailing comma" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "one-argument" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "many-arguments" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "inline-sparse-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mulitline-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mulitline-sparse-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Syntax errors for trailing comma" + }, + { + "type": "Junk", + "annotations": [], + "content": "one-argument = {FUN(1,,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-arg = {FUN(,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-sparse-arg = {FUN( , )}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Whitespace in named arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-named-arg" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "z" + }, + "value": { + "type": "NumberLiteral", + "value": "3" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-colon" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-value" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/callee_expressions.ftl b/fluent-syntax/tests/fixtures/callee_expressions.ftl new file mode 100644 index 00000000..637a2e4d --- /dev/null +++ b/fluent-syntax/tests/fixtures/callee_expressions.ftl @@ -0,0 +1,46 @@ +## Callees in placeables. + +function-callee-placeable = {FUNCTION()} +term-callee-placeable = {-term()} + +# ERROR Messages cannot be parameterized. +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = {FUNCTION() -> + *[key] Value +} +term-attr-callee-selector = {-term.attr() -> + *[key] Value +} + +# ERROR Messages cannot be parameterized. +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/callee_expressions.json b/fluent-syntax/tests/fixtures/callee_expressions.json new file mode 100644 index 00000000..50cdaeb4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/callee_expressions.json @@ -0,0 +1,270 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Callees in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-placeable = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-placeable = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-placeable = {message.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used in Placeables." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attr-callee-placeable = {-term.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-placeable = {$variable()}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Callees in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attr-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-selector = {message() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-selector = {Function() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-selector = {message.attr() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee-selector = {-term() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-selector = {$variable() ->\n *[key] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/comments.ftl b/fluent-syntax/tests/fixtures/comments.ftl new file mode 100644 index 00000000..cc3246ea --- /dev/null +++ b/fluent-syntax/tests/fixtures/comments.ftl @@ -0,0 +1,15 @@ +# Standalone Comment + +# Message Comment +foo = Foo + +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent +## Group Comment +### Resource Comment diff --git a/fluent-syntax/tests/fixtures/comments.json b/fluent-syntax/tests/fixtures/comments.json new file mode 100644 index 00000000..c28115ac --- /dev/null +++ b/fluent-syntax/tests/fixtures/comments.json @@ -0,0 +1,63 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Foo" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Term" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Term Comment\nwith a blank last line.\n" + } + }, + { + "type": "Comment", + "content": "Another standalone\n\n with indent" + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/cr.ftl b/fluent-syntax/tests/fixtures/cr.ftl new file mode 100644 index 00000000..549c662a --- /dev/null +++ b/fluent-syntax/tests/fixtures/cr.ftl @@ -0,0 +1 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/cr.json b/fluent-syntax/tests/fixtures/cr.json new file mode 100644 index 00000000..44eab75f --- /dev/null +++ b/fluent-syntax/tests/fixtures/cr.json @@ -0,0 +1,9 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/crlf.ftl b/fluent-syntax/tests/fixtures/crlf.ftl new file mode 100644 index 00000000..133b6c2c --- /dev/null +++ b/fluent-syntax/tests/fixtures/crlf.ftl @@ -0,0 +1,14 @@ + +key01 = Value 01 +key02 = + + Value 02 + Continued + + .title = Title + +# ERROR Unclosed StringLiteral +err03 = { "str + +# ERROR Missing newline after ->. +err04 = { $sel -> } diff --git a/fluent-syntax/tests/fixtures/crlf.json b/fluent-syntax/tests/fixtures/crlf.json new file mode 100644 index 00000000..58d26a77 --- /dev/null +++ b/fluent-syntax/tests/fixtures/crlf.json @@ -0,0 +1,76 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 01" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 02\nContinued" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "title" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Title" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unclosed StringLiteral" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = { \"str\r\n\r\n" + }, + { + "type": "Comment", + "content": "ERROR Missing newline after ->." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = { $sel -> }\r\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/eof_comment.ftl b/fluent-syntax/tests/fixtures/eof_comment.ftl new file mode 100644 index 00000000..cdeafd90 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_comment.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +# No EOL \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_comment.json b/fluent-syntax/tests/fixtures/eof_comment.json new file mode 100644 index 00000000..9483a1eb --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_comment.json @@ -0,0 +1,13 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Comment", + "content": "No EOL" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/eof_empty.ftl b/fluent-syntax/tests/fixtures/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/eof_empty.json b/fluent-syntax/tests/fixtures/eof_empty.json new file mode 100644 index 00000000..b1992785 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_empty.json @@ -0,0 +1,4 @@ +{ + "type": "Resource", + "body": [] +} diff --git a/fluent-syntax/tests/fixtures/eof_id.ftl b/fluent-syntax/tests/fixtures/eof_id.ftl new file mode 100644 index 00000000..63fa86d6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id.json b/fluent-syntax/tests/fixtures/eof_id.json new file mode 100644 index 00000000..93693f94 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/eof_id_equals.ftl new file mode 100644 index 00000000..7d0d953a --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id_equals.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id = \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id_equals.json b/fluent-syntax/tests/fixtures/eof_id_equals.json new file mode 100644 index 00000000..eefd3285 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id_equals.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id =" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/eof_junk.ftl b/fluent-syntax/tests/fixtures/eof_junk.ftl new file mode 100644 index 00000000..dbafd3a3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_junk.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +000 \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_junk.json b/fluent-syntax/tests/fixtures/eof_junk.json new file mode 100644 index 00000000..d7d12824 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_junk.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "000" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/eof_value.ftl b/fluent-syntax/tests/fixtures/eof_value.ftl new file mode 100644 index 00000000..0d255c57 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_value.json b/fluent-syntax/tests/fixtures/eof_value.json new file mode 100644 index 00000000..502435fc --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_value.json @@ -0,0 +1,27 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "no-eol" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "No EOL" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/escaped_characters.ftl b/fluent-syntax/tests/fixtures/escaped_characters.ftl new file mode 100644 index 00000000..ec862320 --- /dev/null +++ b/fluent-syntax/tests/fixtures/escaped_characters.ftl @@ -0,0 +1,34 @@ +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals +quote-in-string = {"\""} +backslash-in-string = {"\\"} +# ERROR Mismatched quote +mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-4digits = {"\u0041"} +escape-unicode-4digits = {"\\u0041"} +string-unicode-6digits = {"\U01F602"} +escape-unicode-6digits = {"\\U01F602"} + +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = {"\u004100"} +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = {"\U01F60200"} + +# ERROR Too few hex digits after \u. +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. +string-too-few-6digits = {"\U1F602"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/tests/fixtures/escaped_characters.json b/fluent-syntax/tests/fixtures/escaped_characters.json new file mode 100644 index 00000000..a3220996 --- /dev/null +++ b/fluent-syntax/tests/fixtures/escaped_characters.json @@ -0,0 +1,399 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Literal text" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\ a backslash" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\\\ two backslashes" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-brace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\" + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-backslash-u" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "String literals" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "quote-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\"", + "value": "\"" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "backslash-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\", + "value": "\\" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Mismatched quote" + }, + { + "type": "Junk", + "annotations": [], + "content": "mismatched-quote = {\"\\\\\"\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Unknown escape" + }, + { + "type": "Junk", + "annotations": [], + "content": "unknown-escape = {\"\\x\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Unicode escapes" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u0041", + "value": "A" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\u0041", + "value": "\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F602", + "value": "😂" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\U01F602", + "value": "\\U01F602" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u004100", + "value": "A00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F60200", + "value": "😂00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\u." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-4digits = {\"\\u41\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\U." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-6digits = {\"\\U1F602\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Literal braces" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "{", + "value": "{" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "}", + "value": "}" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/junk.ftl b/fluent-syntax/tests/fixtures/junk.ftl new file mode 100644 index 00000000..b0b0c5f3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/junk.ftl @@ -0,0 +1,21 @@ +## Two adjacent Junks. +err01 = {1x} +err02 = {2x} + +# A single Junk. +err03 = {1x +2 + +# A single Junk. +ą=Invalid identifier +ć=Another one + +# The COMMENT ends this junk. +err04 = { +# COMMENT + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. +err04 = { +# COMMENT +} diff --git a/fluent-syntax/tests/fixtures/junk.json b/fluent-syntax/tests/fixtures/junk.json new file mode 100644 index 00000000..15e62a4d --- /dev/null +++ b/fluent-syntax/tests/fixtures/junk.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Two adjacent Junks." + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {1x}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {2x}\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {1x\n2\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Junk", + "annotations": [], + "content": "}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/leading_dots.ftl b/fluent-syntax/tests/fixtures/leading_dots.ftl new file mode 100644 index 00000000..0b9d6693 --- /dev/null +++ b/fluent-syntax/tests/fixtures/leading_dots.ftl @@ -0,0 +1,76 @@ +key01 = .Value +key02 = …Value +key03 = {"."}Value +key04 = + {"."}Value + +key05 = Value + {"."}Continued + +key06 = .Value + {"."}Continued + +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value + .Continued + +# JUNK (attr .Value must have a value) +key08 = + .Value + +# JUNK (attr .Value must have a value) +key09 = + .Value + Continued + +key10 = + .Value = which is an attribute + Continued + +key11 = + {"."}Value = which looks like an attribute + Continued + +key12 = + .accesskey = + A + +key13 = + .attribute = .Value + +key14 = + .attribute = + {"."}Value + +key15 = + { 1 -> + [one] .Value + *[other] + {"."}Value + } + +# JUNK (variant must have a value) +key16 = + { 1 -> + *[one] + .Value + } + +# JUNK (unclosed placeable) +key17 = + { 1 -> + *[one] Value + .Continued + } + +# JUNK (attr .Value must have a value) +key18 = +.Value + +key19 = +.attribute = Value + Continued + +key20 = +{"."}Value diff --git a/fluent-syntax/tests/fixtures/leading_dots.json b/fluent-syntax/tests/fixtures/leading_dots.json new file mode 100644 index 00000000..a185d489 --- /dev/null +++ b/fluent-syntax/tests/fixtures/leading_dots.json @@ -0,0 +1,487 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "…Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key07" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "MESSAGE (value = \"Value\", attributes = [])\nJUNK (attr .Continued\" must have a value)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .Continued\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08 =\n .Value\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key09 =\n .Value\n Continued\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "Value" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "which is an attribute\nContinued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value = which looks like an attribute\nContinued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "accesskey" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key14" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key15" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK (variant must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n\n" + }, + { + "type": "Comment", + "content": "JUNK (unclosed placeable)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key18 =\n.Value\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key19" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key20" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/literal_expressions.ftl b/fluent-syntax/tests/fixtures/literal_expressions.ftl new file mode 100644 index 00000000..937b8a17 --- /dev/null +++ b/fluent-syntax/tests/fixtures/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = {"abc"} +number-expression = {123} +number-expression = {-3.14} diff --git a/fluent-syntax/tests/fixtures/literal_expressions.json b/fluent-syntax/tests/fixtures/literal_expressions.json new file mode 100644 index 00000000..da73e131 --- /dev/null +++ b/fluent-syntax/tests/fixtures/literal_expressions.json @@ -0,0 +1,69 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "abc", + "value": "abc" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "number-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "123" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "number-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "-3.14" + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/member_expressions.ftl b/fluent-syntax/tests/fixtures/member_expressions.ftl new file mode 100644 index 00000000..b4a93922 --- /dev/null +++ b/fluent-syntax/tests/fixtures/member_expressions.ftl @@ -0,0 +1,28 @@ +## Member expressions in placeables. + +message-attribute-expression-placeable = {msg.attr} +term-variant-expression-placeable = {-term[case]} + +# ERROR Message values cannot be VariantLists +message-variant-expression-placeable = {msg[case]} +# ERROR Term attributes may not be used for interpolation. +term-attribute-expression-placeable = {-term.attr} + +## Member expressions in selectors. + +term-attribute-expression-selector = {-term.attr -> + *[key] Value +} + +# ERROR Message attributes may not be used as selector. +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} +# ERROR Term values may not be used as selector. +term-variant-expression-selector = {-term[case] -> + *[key] Value +} +# ERROR Message values cannot be VariantLists +message-variant-expression-selector = {msg[case] -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/member_expressions.json b/fluent-syntax/tests/fixtures/member_expressions.json new file mode 100644 index 00000000..f6890e56 --- /dev/null +++ b/fluent-syntax/tests/fixtures/member_expressions.json @@ -0,0 +1,173 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Member expressions in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "message-attribute-expression-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "AttributeExpression", + "ref": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-variant-expression-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariantExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "key": { + "type": "Identifier", + "name": "case" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-placeable = {msg[case]}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used for interpolation." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attribute-expression-placeable = {-term.attr}\n\n" + }, + { + "type": "GroupComment", + "content": "Member expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attribute-expression-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message attributes may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attribute-expression-selector = {msg.attr ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-variant-expression-selector = {-term[case] ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-selector = {msg[case] ->\n *[key] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/messages.ftl b/fluent-syntax/tests/fixtures/messages.ftl new file mode 100644 index 00000000..00b4ab46 --- /dev/null +++ b/fluent-syntax/tests/fixtures/messages.ftl @@ -0,0 +1,29 @@ +key01 = Value + +key02 = Value + .attr = Attribute + +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +key03 = + .attr = Attribute + +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +# < whitespace > +key05 = + .attr1 = Attribute 1 + +key06 = {""} + +# JUNK Missing value +key07 = + +# JUNK Missing = +key08 + +KEY09 = Value 09 diff --git a/fluent-syntax/tests/fixtures/messages.json b/fluent-syntax/tests/fixtures/messages.json new file mode 100644 index 00000000..cdbe5c93 --- /dev/null +++ b/fluent-syntax/tests/fixtures/messages.json @@ -0,0 +1,267 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr2" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr2" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + } + ], + "comment": { + "type": "Comment", + "content": " < whitespace >" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "key07 =\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "KEY09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 09" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/mixed_entries.ftl b/fluent-syntax/tests/fixtures/mixed_entries.ftl new file mode 100644 index 00000000..99cc023d --- /dev/null +++ b/fluent-syntax/tests/fixtures/mixed_entries.ftl @@ -0,0 +1,24 @@ +# License Comment + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute + +ą=Invalid identifier +ć=Another one + +# Message Comment +key02 = Value + +# Standalone Comment + .attr = Dangling attribute + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 + +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/mixed_entries.json b/fluent-syntax/tests/fixtures/mixed_entries.json new file mode 100644 index 00000000..a9dc501f --- /dev/null +++ b/fluent-syntax/tests/fixtures/mixed_entries.json @@ -0,0 +1,136 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "License Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "brand-name" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Aurora" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr = Dangling attribute\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 03" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are 5 spaces on the line between key03 and key04." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 04" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/multiline_values.ftl b/fluent-syntax/tests/fixtures/multiline_values.ftl new file mode 100644 index 00000000..e3739bb5 --- /dev/null +++ b/fluent-syntax/tests/fixtures/multiline_values.ftl @@ -0,0 +1,60 @@ +key01 = A multiline value + continued on the next line + + and also down here. + +key02 = + A multiline value starting + on a new line. + +key03 = + .attr = A multiline attribute value + continued on the next line + + and also down here. + +key04 = + .attr = + A multiline attribute value + staring on a new line + +key05 = + + A multiline value with non-standard + + indentation. + +key06 = + A multiline value with {"placeables"} + {"at"} the beginning and the end + {"of lines"}{"."} + +key07 = + {"A multiline value"} starting and ending {"with a placeable"} + +key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/fluent-syntax/tests/fixtures/multiline_values.json b/fluent-syntax/tests/fixtures/multiline_values.json new file mode 100644 index 00000000..4d3dd033 --- /dev/null +++ b/fluent-syntax/tests/fixtures/multiline_values.json @@ -0,0 +1,329 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value\ncontinued on the next line\n\nand also down here." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value starting\non a new line." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\ncontinued on the next line\n\nand also down here." + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\nstaring on a new line" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with non-standard\n\n indentation." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "placeables", + "value": "placeables" + } + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "at", + "value": "at" + } + }, + { + "type": "TextElement", + "value": " the beginning and the end\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "of lines", + "value": "of lines" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key07" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "A multiline value", + "value": "A multiline value" + } + }, + { + "type": "TextElement", + "value": " starting and ending " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "with a placeable", + "value": "with a placeable" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key08" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Leading and trailing whitespace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl deleted file mode 100644 index 87aebc9b..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl +++ /dev/null @@ -1,23 +0,0 @@ - key2 = Value 2 - -= Value - - = Value - -= - - = - - = = - -id - = value - -id -= value - -1 = 1 - --1 = Foo - -19 = Foo2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl deleted file mode 100644 index f7082aec..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl +++ /dev/null @@ -1 +0,0 @@ -key1 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl deleted file mode 100644 index c517b5ff..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl +++ /dev/null @@ -1,10 +0,0 @@ -key1 = Value 1 -key2 =Value 2 - -key3 = Value 3 - -key4=Value4 -key5= Value 5 -key6= Value 6 -key7 = Value 7 -key8 = Value 8 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl deleted file mode 100644 index a1e8ae0b..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl +++ /dev/null @@ -1,9 +0,0 @@ -key1 = "Value 1" -key2="Value 2" -key3 = " Value 3 " - -key4 = "" - -key5 = " " - -key6 = " Foo \" Foo2 " diff --git a/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl deleted file mode 100644 index f7f32e63..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl +++ /dev/null @@ -1,14 +0,0 @@ -key1 = - This is a new line - -key4 = -

- So - Many - Lines -

- -key5 = -

- Foo -

diff --git a/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl b/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl deleted file mode 100644 index de4baa2f..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl +++ /dev/null @@ -1,22 +0,0 @@ -### File comment - -# Standalone comment - -# Another standalone comment - -# Comment with a leading space - -## Multi -## Line Section -## -## Comment - -# Comment for entity key1 -key1 = New entity - -# Comment for entity key2 -key2= - | Multi line message - -## Group comment -key3 = Message diff --git a/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl deleted file mode 100644 index 170eeb2a..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl +++ /dev/null @@ -1,25 +0,0 @@ -[section]] - -section]] - -[[section - -[section - -[[section] - -[[section]]dd - -[[section -]] - - -[[ ]] - -[[]] - -#doo -[[ - -#foo -[ diff --git a/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl deleted file mode 100644 index caba0259..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl +++ /dev/null @@ -1,40 +0,0 @@ -key1 = { - [:key] Value - *[ok] Valid -} - -key3 = { - [] Value - *[ok] Valid -} - -key4 = { - **[f] Foo - *[ok] Valid -} - -key5 = { - *fw] Foo - *[ok] Valid -} - -key6 = { - [ a] A - *[ok] Valid -} - -key7 = { - [ x/a] XA - *[ok] Valid -} - -key8 = { - [x y/a] XYA - *[ok] Valid -} - -key10 = { - [x/a ] XA - [x/a b ] XAB - *[ok] Valid -} diff --git a/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl b/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl deleted file mode 100644 index 3b2ea9c2..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl +++ /dev/null @@ -1,37 +0,0 @@ -key1 = Value - .gender = male - -key2 = - .gender = male - -key3 = { - *[masculine] Variant for masculine - [feminine] Variant for feminine - } - -key4 = - .aria-label = Test - -key5 = - .aria-label = Test - .loop = Foo - -key5 = - .m = Foo - -## section - - -key6 = { - *[one] One - [two] Two - [three] Three - } - -key7 = { - *[a b] A - } - -key8 = { - *[a b] A - } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl deleted file mode 100644 index 984a5bfa..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl +++ /dev/null @@ -1,87 +0,0 @@ -key1 = { -} - -key2 = { -} - - -key3 = { - $user1 -} - - -key4 = { - - -key6 = {,} - -key7 = {,$user} - -key8 = {$user,,$user2} - -key9 = {$user, ,$user2} - -key10 = {$user, -} - -key11 = { $user - - -key12 = { $user -> - -key13 = { $user -> -} - -key14 = { $user -> $user2 } - -key15 = { $user -> - $user2 -} - -key11 = Foo {} Foo2 - -key12 = Foo { } Foo 2 - -key13 = {$user,} - -key14 = {$user, -} - -key15 = {_,} - -key16 = { foo($user1=) } - -key17 = { foo(1=) } - -key18 = { foo(len()=)} - -key19 = { foo(bar/baz=1)} - -key20 = { len(bar=user) } - -key21 = { len(bar=bar/baz) } - -key22 = { len(bar=foo[key]) } - -key23 = { len(bar=bar/baz[foo:faa]) } - -key24 = { len(bar = baz) } - -key25 = { len( bar = baz ) } - -key27 = { $len -> [foo] Value } - -key28 = { $len -> *[foo] Value } - -key29 = { menu/open } - -key30 = { LEN($u1, $u2, open/brand-name, type:"short") } - -key31 = { menu/brand-name[accusative] } - -key32 = { len(bar/baz) } - -key33 = { len(bar/baz[foo]) } - -key34 = { len(bar/baz[foo/fab]) } - -keyLast = { diff --git a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl deleted file mode 100644 index 38ec12fa..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl +++ /dev/null @@ -1,44 +0,0 @@ -key1 = AA { $num } BB - -key2 = { brand-name } - -key4 = { $num -> - *[one] One - [two] Two - } - -key5 = { LEN($num) -> - *[one] One - [two] Two - } - -key6 = { LEN(NEL($num)) -> - *[one] One - [two] Two - } - -key7 = { LIST($user1, $user2) } - -key9 = { LEN(2, 2.5, -3.12, -1.00) } - -key11 = { LEN() } - -key12 = { LEN(1) } - -key13 = { LEN(-1) } - -key14 = { LEN($foo) } - -key15 = { LEN(foo) } - -key19 = { LEN(bar: 1) } - -key20 = { LEN(bar: -1) } - -key21 = { LEN(bar: "user") } - -key22 = { brand-name[masculine] } - -key23 = { NUMBER(style: "percent") } - -key24 = { NUMBER_SPECIAL($num, style: "percent", foo: "bar") } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl b/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl deleted file mode 100644 index 84c89463..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl +++ /dev/null @@ -1,11 +0,0 @@ --brand-short-name = Firefox - .gender = masculine - -key = Test { -brand-short-name } - -key2 = Test { -brand-short-name.gender -> - [masculine] Foo - *[feminine] Foo 2 - } - -key3 = Test { -brand[one] } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl deleted file mode 100644 index ad8473dd..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl +++ /dev/null @@ -1 +0,0 @@ - key = value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl deleted file mode 100644 index 0cfbf088..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl deleted file mode 100644 index 06bfde49..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl +++ /dev/null @@ -1 +0,0 @@ -key diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl deleted file mode 100644 index e4612356..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl +++ /dev/null @@ -1 +0,0 @@ -key Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl deleted file mode 100644 index 708ef4c5..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl +++ /dev/null @@ -1,2 +0,0 @@ -key = Value - .# = Foo diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl deleted file mode 100644 index 154f0a4d..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl +++ /dev/null @@ -1,2 +0,0 @@ -key = Value - .foo diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl deleted file mode 100644 index 1fc4ee77..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl +++ /dev/null @@ -1,3 +0,0 @@ -key - -key2 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl deleted file mode 100644 index 2e6e9340..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl +++ /dev/null @@ -1,12 +0,0 @@ - -key = - { $foo -> - [one] Foo - *[-other] Foo 2 - } - -key2 = { $-foo } - -key3 = { -brand.gender } - -key4 = { -brand() } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl deleted file mode 100644 index 4c2f1f72..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl +++ /dev/null @@ -1,12 +0,0 @@ -key1 = Value -key2 = Value 2 -key3 Value -key4 = Value - -key5 = Value - -key6 Value - -key7 = Value - -key8 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl deleted file mode 100644 index 54952568..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl +++ /dev/null @@ -1,2 +0,0 @@ -0 = key -key2 = Value 2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl deleted file mode 100644 index c8015f87..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl +++ /dev/null @@ -1,5 +0,0 @@ -key1 = Value - -key2 = Value - -key diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl deleted file mode 100644 index ee701588..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl +++ /dev/null @@ -1,11 +0,0 @@ -key1 = Value - -key2 = Value - .one = { OS -> - *[masculine] { LEN($num) -> - *[1] Something - [] Two - } - [feminine] Faa - } -key3 = Value 2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl deleted file mode 100644 index 3fe5a742..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl +++ /dev/null @@ -1,7 +0,0 @@ -key = Value - -key - -# This is a comment - -key2 = Value 2 diff --git a/fluent-syntax/tests/fixtures/placeables.ftl b/fluent-syntax/tests/fixtures/placeables.ftl new file mode 100644 index 00000000..7a1b280f --- /dev/null +++ b/fluent-syntax/tests/fixtures/placeables.ftl @@ -0,0 +1,15 @@ +nested-placeable = {{{1}}} +padded-placeable = { 1 } +sparse-placeable = { { 1 } } + +# ERROR Unmatched opening brace +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/tests/fixtures/placeables.json b/fluent-syntax/tests/fixtures/placeables.json new file mode 100644 index 00000000..7d67d940 --- /dev/null +++ b/fluent-syntax/tests/fixtures/placeables.json @@ -0,0 +1,113 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "nested-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "padded-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open1 = { 1\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open2 = {{ 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close1 = 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close2 = { 1 }}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/reference_expressions.ftl b/fluent-syntax/tests/fixtures/reference_expressions.ftl new file mode 100644 index 00000000..9c2e9c54 --- /dev/null +++ b/fluent-syntax/tests/fixtures/reference_expressions.ftl @@ -0,0 +1,28 @@ +## Reference expressions in placeables. + +message-reference-placeable = {msg} +term-reference-placeable = {-term} +variable-reference-placeable = {$var} + +# ERROR Function references are invalid outside of call expressions. +function-reference-placeable = {FUN} + + +## Reference expressions in selectors. + +variable-reference-selector = {$var -> + *[key] Value +} + +# ERROR Message values may not be used as selectors. +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions. +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/reference_expressions.json b/fluent-syntax/tests/fixtures/reference_expressions.json new file mode 100644 index 00000000..65c9d4cc --- /dev/null +++ b/fluent-syntax/tests/fixtures/reference_expressions.json @@ -0,0 +1,185 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Reference expressions in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "message-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + } + }, + { + "type": "GroupComment", + "content": "Reference expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-reference-selector = {msg ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-reference-selector = {-term ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + }, + { + "type": "Junk", + "annotations": [], + "content": "function-expression-selector = {FUN ->\n *[key] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/select_expressions.ftl b/fluent-syntax/tests/fixtures/select_expressions.ftl new file mode 100644 index 00000000..7a1fb820 --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_expressions.ftl @@ -0,0 +1,53 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] {""}Other + } + +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR +invalid-selector-term-variant = + { -term[case] -> + *[key] value + } + +# ERROR +invalid-selector-term-call = + { -term(case: "nominative") -> + *[key] value + } + +empty-variant = + { 1 -> + *[one] {""} + } + +nested-select = + { 1 -> + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists cannot be Variant values. +nested-variant-list = + { 1 -> + *[one] { + *[two] Value + } + } + +# ERROR Missing line end after variant list +missing-line-end = + { 1 -> + *[one] One} diff --git a/fluent-syntax/tests/fixtures/select_expressions.json b/fluent-syntax/tests/fixtures/select_expressions.json new file mode 100644 index 00000000..a3dc5730 --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_expressions.json @@ -0,0 +1,294 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "new-messages" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "BUILTIN" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Zero" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + }, + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "valid-selector-term-attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "case" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-variant" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "nested-select" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Missing line end after variant list" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-line-end =\n { 1 ->\n *[one] One}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/select_indent.ftl b/fluent-syntax/tests/fixtures/select_indent.ftl new file mode 100644 index 00000000..6c13076b --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_indent.ftl @@ -0,0 +1,95 @@ +select-1tbs-inline = { $selector -> + *[key] Value +} + +select-1tbs-newline = { +$selector -> + *[key] Value +} + +select-1tbs-indent = { + $selector -> + *[key] Value +} + +select-allman-inline = +{ $selector -> + *[key] Value +} + +select-allman-newline = +{ +$selector -> + *[key] Value +} + +select-allman-indent = +{ + $selector -> + *[key] Value +} + +select-gnu-inline = + { $selector -> + *[key] Value + } + +select-gnu-newline = + { +$selector -> + *[key] Value + } + +select-gnu-indent = + { + $selector -> + *[key] Value + } + +select-no-indent = +{ +$selector -> +*[key] Value +[other] Other +} + +select-no-indent-multiline = +{ +$selector -> +*[key] Value + Continued +[other] + Other + Multiline +} + +# ERROR (Multiline text must be indented) +select-no-indent-multiline = { $selector -> + *[key] Value +Continued without indent. +} + +select-flat = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} + +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} diff --git a/fluent-syntax/tests/fixtures/select_indent.json b/fluent-syntax/tests/fixtures/select_indent.json new file mode 100644 index 00000000..d8c0fa88 --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_indent.json @@ -0,0 +1,688 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-no-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-no-indent-multiline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other\nMultiline" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR (Multiline text must be indented)" + }, + { + "type": "Junk", + "annotations": [], + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "Continued without indent.\n}\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-flat" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-flat-with-trailing-spaces" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Each line ends with 5 spaces." + } + } + ] +} diff --git a/fluent-syntax/tests/fixtures/sparse_entries.ftl b/fluent-syntax/tests/fixtures/sparse_entries.ftl new file mode 100644 index 00000000..67920b2c --- /dev/null +++ b/fluent-syntax/tests/fixtures/sparse_entries.ftl @@ -0,0 +1,39 @@ +key01 = + + + Value + +key02 = + + + .attr = Attribute + + +key03 = + Value + Continued + + + Over multiple + Lines + + + + .attr = Attribute + + +key05 = Value + +key06 = { 1 -> + + + [one] One + + + + + *[two] Two + + + + } diff --git a/fluent-syntax/tests/fixtures/sparse_entries.json b/fluent-syntax/tests/fixtures/sparse_entries.json new file mode 100644 index 00000000..0c4a3686 --- /dev/null +++ b/fluent-syntax/tests/fixtures/sparse_entries.json @@ -0,0 +1,163 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued\n\n\nOver multiple\nLines" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "One" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Two" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/tab.ftl b/fluent-syntax/tests/fixtures/tab.ftl new file mode 100644 index 00000000..4b23ad87 --- /dev/null +++ b/fluent-syntax/tests/fixtures/tab.ftl @@ -0,0 +1,14 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) +key02 = Value 02 + +# Error (tab is not a valid indent) +key03 = + This line isn't properly indented. + +# Partial Error (tab is not a valid indent) +key04 = + This line is indented by 4 spaces, + whereas this line by 1 tab. diff --git a/fluent-syntax/tests/fixtures/tab.json b/fluent-syntax/tests/fixtures/tab.json new file mode 100644 index 00000000..714eb947 --- /dev/null +++ b/fluent-syntax/tests/fixtures/tab.json @@ -0,0 +1,70 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\tValue 01" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK (tab after = is part of the value)" + } + }, + { + "type": "Comment", + "content": "Error (tab before =)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key02\t= Value 02\n\n" + }, + { + "type": "Comment", + "content": "Error (tab is not a valid indent)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key03 =\n\tThis line isn't properly indented.\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "This line is indented by 4 spaces," + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Partial Error (tab is not a valid indent)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": "\twhereas this line by 1 tab.\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/term_parameters.ftl b/fluent-syntax/tests/fixtures/term_parameters.ftl new file mode 100644 index 00000000..61442361 --- /dev/null +++ b/fluent-syntax/tests/fixtures/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = { $arg -> + *[key] Value +} + +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/term_parameters.json b/fluent-syntax/tests/fixtures/term_parameters.json new file mode 100644 index 00000000..f9f09613 --- /dev/null +++ b/fluent-syntax/tests/fixtures/term_parameters.json @@ -0,0 +1,203 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "arg" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "arg" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "positional", + "value": "positional" + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg1" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg2" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/terms.ftl b/fluent-syntax/tests/fixtures/terms.ftl new file mode 100644 index 00000000..b8791fcf --- /dev/null +++ b/fluent-syntax/tests/fixtures/terms.ftl @@ -0,0 +1,23 @@ +-term01 = Value + .attr = Attribute + +-term02 = {""} + +# JUNK Missing value +-term03 = + .attr = Attribute + +# JUNK Missing value +# < whitespace > +-term04 = + .attr1 = Attribute 1 + +# JUNK Missing value +-term05 = + +# JUNK Missing value +# < whitespace > +-term06 = + +# JUNK Missing = +-term07 diff --git a/fluent-syntax/tests/fixtures/terms.json b/fluent-syntax/tests/fixtures/terms.json new file mode 100644 index 00000000..2321283f --- /dev/null +++ b/fluent-syntax/tests/fixtures/terms.json @@ -0,0 +1,107 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term03 =\n .attr = Attribute\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term04 = \n .attr1 = Attribute 1\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term05 =\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term06 = \n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term07\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/variables.ftl b/fluent-syntax/tests/fixtures/variables.ftl new file mode 100644 index 00000000..6c343692 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variables.ftl @@ -0,0 +1,17 @@ +key01 = {$var} +key02 = { $var } +key03 = { + $var +} +key04 = { +$var} + + +## Errors + +# ERROR Missing variable identifier +err01 = {$} +# ERROR Double $$ +err02 = {$$var} +# ERROR Invalid first char of the identifier +err03 = {$-var} diff --git a/fluent-syntax/tests/fixtures/variables.json b/fluent-syntax/tests/fixtures/variables.json new file mode 100644 index 00000000..58682e5b --- /dev/null +++ b/fluent-syntax/tests/fixtures/variables.json @@ -0,0 +1,132 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Errors" + }, + { + "type": "Comment", + "content": "ERROR Missing variable identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {$}\n" + }, + { + "type": "Comment", + "content": "ERROR Double $$" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {$$var}\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid first char of the identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {$-var}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/variant_keys.ftl b/fluent-syntax/tests/fixtures/variant_keys.ftl new file mode 100644 index 00000000..7586d524 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_keys.ftl @@ -0,0 +1,37 @@ +-simple-identifier = + { + *[key] value + } + +-identifier-surrounded-by-whitespace = + { + *[ key ] value + } + +-int-number = + { + *[1] value + } + +-float-number = + { + *[3.14] value + } + +# ERROR +-invalid-identifier = + { + *[two words] value + } + +# ERROR +-invalid-int = + { + *[1 apple] value + } + +# ERROR +-invalid-int = + { + *[3.14 apples] value + } diff --git a/fluent-syntax/tests/fixtures/variant_keys.json b/fluent-syntax/tests/fixtures/variant_keys.json new file mode 100644 index 00000000..cf752e43 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_keys.json @@ -0,0 +1,156 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "simple-identifier" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "identifier-surrounded-by-whitespace" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "int-number" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "float-number" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "3.14" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[3.14 apples] value\n }\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/variant_lists.ftl b/fluent-syntax/tests/fixtures/variant_lists.ftl new file mode 100644 index 00000000..e5c61dd8 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_lists.ftl @@ -0,0 +1,55 @@ +-variant-list-in-term = + { + *[key] Value + } + +# ERROR Attributes of Terms must be Patterns. +-variant-list-in-term-attr = Value + .attr = + { + *[key] Value + } + +# ERROR Message values must be Patterns. +variant-list-in-message = + { + *[key] Value + } + +# ERROR Attributes of Messages must be Patterns. +variant-list-in-message-attr = Value + .attr = + { + *[key] Value + } + +# ERROR VariantLists cannot be Variant values. +-nested-variant-list-in-term = + { + *[one] { + *[two] Value + } + } + +-nested-select = + { + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists cannot be Variant values. +nested-select-then-variant-list = + { + *[one] { 2 -> + *[two] { + *[three] Value + } + } + } + +# ERROR VariantLists are value types and may not appear in Placeables +variant-list-in-placeable = + A prefix here { + *[key] Value + } and a postfix here make this a Pattern. diff --git a/fluent-syntax/tests/fixtures/variant_lists.json b/fluent-syntax/tests/fixtures/variant_lists.json new file mode 100644 index 00000000..83f16662 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_lists.json @@ -0,0 +1,180 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variant-list-in-term" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variant-list-in-term-attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Terms must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Message values must be Patterns." + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variant-list-in-message-attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Messages must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "-nested-variant-list-in-term =\n {\n *[one] {\n *[two] Value\n }\n }\n\n" + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "nested-select" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR VariantLists are value types and may not appear in Placeables" + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-placeable =\n A prefix here {\n *[key] Value\n } and a postfix here make this a Pattern.\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/variants_indent.ftl b/fluent-syntax/tests/fixtures/variants_indent.ftl new file mode 100644 index 00000000..38f5a62e --- /dev/null +++ b/fluent-syntax/tests/fixtures/variants_indent.ftl @@ -0,0 +1,19 @@ +-variants-1tbs = { + *[key] Value +} + +-variants-allman = +{ + *[key] Value +} + +-variants-gnu = + { + *[key] Value + } + +-variants-no-indent = +{ +*[key] Value +[other] Other +} diff --git a/fluent-syntax/tests/fixtures/variants_indent.json b/fluent-syntax/tests/fixtures/variants_indent.json new file mode 100644 index 00000000..7b987d48 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variants_indent.json @@ -0,0 +1,146 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-1tbs" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-allman" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-gnu" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-no-indent" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/whitespace_in_value.ftl new file mode 100644 index 00000000..2fba5535 --- /dev/null +++ b/fluent-syntax/tests/fixtures/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/whitespace_in_value.json b/fluent-syntax/tests/fixtures/whitespace_in_value.json new file mode 100644 index 00000000..077e6867 --- /dev/null +++ b/fluent-syntax/tests/fixtures/whitespace_in_value.json @@ -0,0 +1,26 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "first line\n\n\n\n\n\n\nlast line" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Caution, lines 6 and 7 contain white-space-only lines" + } + } + ] +} diff --git a/fluent-syntax/tests/junk.rs b/fluent-syntax/tests/junk.rs deleted file mode 100644 index c025af6f..00000000 --- a/fluent-syntax/tests/junk.rs +++ /dev/null @@ -1,79 +0,0 @@ -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -#[test] -fn basic_junk() { - let path = "./tests/fixtures/parser/ftl/junk/01-basic.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(2, errors.len()); - assert_eq!(8, res.body.len()); - } - } -} - -#[test] -fn start_junk() { - let path = "./tests/fixtures/parser/ftl/junk/02-start.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(2, res.body.len()); - } - } -} - -#[test] -fn end_junk() { - let path = "./tests/fixtures/parser/ftl/junk/03-end.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(3, res.body.len()); - } - } -} - -#[test] -fn multiline_junk() { - let path = "./tests/fixtures/parser/ftl/junk/04-multiline.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(3, res.body.len()); - } - } -} - -#[test] -fn recover_at_comment() { - let path = "./tests/fixtures/parser/ftl/junk/05-comment.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(4, res.body.len()); - } - } -} diff --git a/fluent-syntax/tests/parser_fixtures.rs b/fluent-syntax/tests/parser_fixtures.rs new file mode 100644 index 00000000..1eee7949 --- /dev/null +++ b/fluent-syntax/tests/parser_fixtures.rs @@ -0,0 +1,65 @@ +mod ast; + +use assert_json_diff::assert_json_include; +use glob::glob; +use serde_json::Value; +use std::fs::File; +use std::io; +use std::io::prelude::*; + +use fluent_syntax::parser::parse; + +fn compare_jsons(value: &str, reference: &str) { + let a: Value = serde_json::from_str(value).unwrap(); + + let b: Value = serde_json::from_str(reference).unwrap(); + + assert_json_include!(actual: a, expected: b); +} + +fn read_file(path: &str, trim: bool) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + if trim { + Ok(s.trim().to_string()) + } else { + Ok(s) + } +} + +#[test] +fn parse_fixtures_compare() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let p = entry.expect("Error while getting an entry"); + let path = p.to_str().expect("Can't print path"); + + let reference_path = path.replace(".ftl", ".json"); + let reference_file = read_file(&reference_path, true).unwrap(); + let ftl_file = read_file(&path, false).unwrap(); + + println!("Parsing: {:#?}", path); + let target_ast = match parse(&ftl_file) { + Ok(res) => res, + Err((res, _errors)) => res, + }; + + let target_json = ast::serialize(&target_ast).unwrap(); + + compare_jsons(&target_json, &reference_file); + } +} + +#[test] +fn parse_fixtures() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let p = entry.expect("Error while getting an entry"); + let path = p.to_str().expect("Can't print path"); + + println!("Attempting to parse file: {}", path); + + let string = read_file(path, false).expect("Failed to read"); + + let _ = parse(&string); + } +} diff --git a/fluent-syntax/tests/stream.rs b/fluent-syntax/tests/stream.rs deleted file mode 100644 index 9eaca418..00000000 --- a/fluent-syntax/tests/stream.rs +++ /dev/null @@ -1,185 +0,0 @@ -extern crate fluent_syntax; - -use self::fluent_syntax::parser::stream::ParserStream; - -#[test] -fn next() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('a'), ps.current()); - assert_eq!(0, ps.get_index()); - - assert_eq!(Some('b'), ps.next()); - assert_eq!(Some('b'), ps.current()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.next()); - assert_eq!(Some('c'), ps.current()); - assert_eq!(2, ps.get_index()); - - assert_eq!(Some('d'), ps.next()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(3, ps.get_index()); - - assert_eq!(None, ps.next()); - assert_eq!(None, ps.current()); - assert_eq!(4, ps.get_index()); -} - -#[test] -fn peek() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('a'), ps.current_peek()); - assert_eq!(0, ps.get_peek_index()); - - assert_eq!(Some('b'), ps.peek()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - - assert_eq!(None, ps.peek()); - assert_eq!(None, ps.current_peek()); - assert_eq!(4, ps.get_peek_index()); -} - -#[test] -fn peek_and_next() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('b'), ps.peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(0, ps.get_index()); - - assert_eq!(Some('b'), ps.next()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.next()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - assert_eq!(Some('d'), ps.next()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - - assert_eq!(None, ps.peek()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(None, ps.current_peek()); - - assert_eq!(None, ps.peek()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - - assert_eq!(None, ps.next()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(4, ps.get_index()); -} - -#[test] -fn skip_to_peek() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.peek(); - ps.peek(); - - ps.skip_to_peek(); - - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - ps.peek(); - - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - ps.next(); - - assert_eq!(Some('d'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); -} - -#[test] -fn reset_peek() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.next(); - ps.peek(); - ps.peek(); - ps.reset_peek(None); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - ps.peek(); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - ps.peek(); - ps.peek(); - ps.peek(); - ps.reset_peek(None); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(None, ps.peek()); -} - -#[test] -fn peek_char_is() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.next(); - ps.peek(); - - assert_eq!(ps.peek_char_is('d'), true); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - - ps.skip_to_peek(); - - assert_eq!(Some('c'), ps.current()); -} diff --git a/fluent/examples/hello.rs b/fluent/examples/hello.rs deleted file mode 100644 index b5d2d9b2..00000000 --- a/fluent/examples/hello.rs +++ /dev/null @@ -1,10 +0,0 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; - -fn main() { - let mut bundle = FluentBundle::new(&["en-US"]); - bundle.add_messages("hello-world = Hello, world!").unwrap(); - let (value, _) = bundle.format("hello-world", None).unwrap(); - assert_eq!(&value, "Hello, world!"); -} diff --git a/fluent/src/bin/parser.rs b/fluent/src/bin/parser.rs deleted file mode 100644 index afbab0e8..00000000 --- a/fluent/src/bin/parser.rs +++ /dev/null @@ -1,64 +0,0 @@ -extern crate clap; -extern crate fluent; -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::Read; - -use clap::App; - -use fluent_syntax::ast::Resource; -use fluent_syntax::parser::errors::display::annotate_error; -use fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -fn print_entries_resource(res: &Resource) { - println!("{:#?}", res); -} - -fn main() { - let matches = App::new("Fluent Parser") - .version("1.0") - .about("Parses FTL file into an AST") - .args_from_usage( - "-s, --silence 'disable output' - 'Sets the input file to use'", - ) - .get_matches(); - - let input = matches.value_of("INPUT").unwrap(); - - let source = read_file(&input).expect("Read file failed"); - - let res = parse(&source); - - if matches.is_present("silence") { - return; - }; - - match res { - Ok(res) => print_entries_resource(&res), - Err((res, errors)) => { - print_entries_resource(&res); - println!("==============================\n"); - if errors.len() == 1 { - println!("Parser encountered one error:"); - } else { - println!("Parser encountered {} errors:", errors.len()); - } - println!("-----------------------------"); - for err in errors { - let f = annotate_error(&err, &Some(input.to_string()), true); - println!("{}", f); - println!("-----------------------------"); - } - } - }; -} diff --git a/fluent/src/bundle.rs b/fluent/src/bundle.rs deleted file mode 100644 index 1eff32c7..00000000 --- a/fluent/src/bundle.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! `FluentBundle` is a collection of localization messages in Fluent. -//! -//! It stores a list of messages in a single locale which can reference one another, use the same -//! internationalization formatters, functions, environmental variables and are expected to be used -//! together. - -use std::cell::RefCell; -use std::collections::hash_map::{Entry as HashEntry, HashMap}; - -use super::entry::{Entry, GetEntry}; -pub use super::errors::FluentError; -use super::resolve::{Env, ResolveValue}; -use super::resource::FluentResource; -use super::types::FluentValue; -use fluent_locale::{negotiate_languages, NegotiationStrategy}; -use fluent_syntax::ast; -use intl_pluralrules::{IntlPluralRules, PluralRuleType}; - -#[derive(Debug, PartialEq)] -pub struct Message { - pub value: Option, - pub attributes: HashMap, -} - -/// `FluentBundle` is a collection of localization messages which are meant to be used together -/// in a single view, widget or any other UI abstraction. -/// -/// # `FluentBundle` Life-cycle -/// -/// To create a bundle, call `FluentBundle::new` with a locale list that represents the best -/// possible fallback chain for a given locale. The simplest case is a one-locale list. -/// -/// Next, call `add_messages` one or more times, supplying translations in the FTL syntax. The -/// `FluentBundle` instance is now ready to be used for localization. -/// -/// To format a translation, call `get_message` to retrieve a `fluent::bundle::Message` structure -/// and then `format` it within the bundle. -/// -/// The result is an Option wrapping a single string that should be displayed in the UI. It is -/// recommended to treat the result as opaque from the perspective of the program and use it only -/// to display localized messages. Do not examine it or alter in any way before displaying. This -/// is a general good practice as far as all internationalization operations are concerned. -/// -/// -/// # Locale Fallback Chain -/// -/// `FluentBundle` stores messages in a single locale, but keeps a locale fallback chain for the -/// purpose of language negotiation with i18n formatters. For instance, if date and time formatting -/// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain -/// to negotiate a sensible fallback for date and time formatting. -#[allow(dead_code)] -pub struct FluentBundle<'bundle> { - pub locales: Vec, - pub entries: HashMap>, - pub plural_rules: IntlPluralRules, -} - -impl<'bundle> FluentBundle<'bundle> { - pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { - let locales = locales - .into_iter() - .map(|s| s.to_string()) - .collect::>(); - let pr_locale = negotiate_languages( - &locales, - IntlPluralRules::get_locales(PluralRuleType::CARDINAL), - Some("en"), - &NegotiationStrategy::Lookup, - )[0] - .to_owned(); - - let pr = IntlPluralRules::create(&pr_locale, PluralRuleType::CARDINAL).unwrap(); - FluentBundle { - locales, - entries: HashMap::new(), - plural_rules: pr, - } - } - - pub fn has_message(&self, id: &str) -> bool { - self.entries.get_message(id).is_some() - } - - pub fn add_function(&mut self, id: &str, func: F) -> Result<(), FluentError> - where - F: 'bundle - + Fn(&[Option], &HashMap) -> Option - + Sync - + Send, - { - match self.entries.entry(id.to_owned()) { - HashEntry::Vacant(entry) => { - entry.insert(Entry::Function(Box::new(func))); - Ok(()) - } - HashEntry::Occupied(_) => Err(FluentError::Overriding { - kind: "function", - id: id.to_owned(), - }), - } - } - - pub fn add_messages(&mut self, source: &str) -> Result<(), Vec> { - match FluentResource::from_string(source) { - Ok(res) => self.add_resource(res), - Err((res, err)) => { - let mut errors: Vec = - err.into_iter().map(FluentError::ParserError).collect(); - - self.add_resource(res).map_err(|err| { - for e in err { - errors.push(e); - } - errors - }) - } - } - } - - pub fn add_resource(&mut self, res: FluentResource) -> Result<(), Vec> { - let mut errors = vec![]; - - for entry in res.ast.body { - let id = match entry { - ast::Entry::Message(ast::Message { ref id, .. }) => id.name.clone(), - ast::Entry::Term(ast::Term { ref id, .. }) => id.name.clone(), - _ => continue, - }; - - let (entry, kind) = match entry { - ast::Entry::Message(message) => (Entry::Message(message), "message"), - ast::Entry::Term(term) => (Entry::Term(term), "term"), - _ => continue, - }; - - match self.entries.entry(id.clone()) { - HashEntry::Vacant(empty) => { - empty.insert(entry); - } - HashEntry::Occupied(_) => { - errors.push(FluentError::Overriding { kind, id }); - } - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(errors) - } - } - - pub fn format( - &self, - path: &str, - args: Option<&HashMap<&str, FluentValue>>, - ) -> Option<(String, Vec)> { - let env = Env { - bundle: self, - args, - travelled: RefCell::new(Vec::new()), - }; - - let mut errors = vec![]; - - if let Some(ptr_pos) = path.find('.') { - let message_id = &path[..ptr_pos]; - let message = self.entries.get_message(message_id)?; - let attr_name = &path[(ptr_pos + 1)..]; - if let Some(ref attributes) = message.attributes { - for attribute in attributes { - if attribute.id.name == attr_name { - match attribute.to_value(&env) { - Ok(val) => { - return Some((val.format(self), errors)); - } - Err(err) => { - errors.push(FluentError::ResolverError(err)); - // XXX: In the future we'll want to get the partial - // value out of resolver and return it here. - // We also expect to get a Vec or errors out of resolver. - return Some((path.to_string(), errors)); - } - } - } - } - } - } else { - let message_id = path; - let message = self.entries.get_message(message_id)?; - match message.to_value(&env) { - Ok(val) => { - let s = val.format(self); - return Some((s, errors)); - } - Err(err) => { - errors.push(FluentError::ResolverError(err)); - return Some((message_id.to_string(), errors)); - } - } - } - - None - } - - pub fn format_message( - &self, - message_id: &str, - args: Option<&HashMap<&str, FluentValue>>, - ) -> Option<(Message, Vec)> { - let mut errors = vec![]; - - let env = Env { - bundle: self, - args, - travelled: RefCell::new(Vec::new()), - }; - let message = self.entries.get_message(message_id)?; - - let value = match message.to_value(&env) { - Ok(value) => Some(value.format(self)), - Err(err) => { - errors.push(FluentError::ResolverError(err)); - None - } - }; - - let mut attributes = HashMap::new(); - - if let Some(ref attrs) = message.attributes { - for attr in attrs { - match attr.to_value(&env) { - Ok(value) => { - let val = value.format(self); - attributes.insert(attr.id.name.to_owned(), val); - } - Err(err) => { - errors.push(FluentError::ResolverError(err)); - } - } - } - } - - Some((Message { value, attributes }, errors)) - } -} diff --git a/fluent/src/resource.rs b/fluent/src/resource.rs deleted file mode 100644 index 9521b486..00000000 --- a/fluent/src/resource.rs +++ /dev/null @@ -1,16 +0,0 @@ -use fluent_syntax::ast; -use fluent_syntax::parser::errors::ParserError; -use fluent_syntax::parser::parse; - -pub struct FluentResource { - pub ast: ast::Resource, -} - -impl FluentResource { - pub fn from_string(source: &str) -> Result)> { - match parse(source) { - Ok(ast) => Ok(FluentResource { ast }), - Err((ast, errors)) => Err((FluentResource { ast }, errors)), - } - } -} diff --git a/fluent/tests/helpers/mod.rs b/fluent/tests/helpers/mod.rs deleted file mode 100644 index 6733bb9e..00000000 --- a/fluent/tests/helpers/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -use fluent::bundle::FluentError; -use fluent::bundle::Message; - -#[allow(dead_code)] -pub fn assert_format_none(result: Option<(String, Vec)>) { - assert!(result.is_none()); -} - -#[allow(dead_code)] -pub fn assert_format_no_errors(result: Option<(String, Vec)>, expected: &str) { - assert!(result.is_some()); - assert_eq!(result, Some((expected.to_string(), vec![]))); -} - -#[allow(dead_code)] -pub fn assert_format_message_no_errors( - result: Option<(Message, Vec)>, - expected: Message, -) { - assert_eq!(result, Some((expected, vec![]))); -} - -pub fn assert_add_messages_no_errors(result: Result<(), Vec>) { - assert!(result.is_ok()); -} diff --git a/fluent/tests/resolve_value.rs b/fluent/tests/resolve_value.rs deleted file mode 100644 index 19cd074c..00000000 --- a/fluent/tests/resolve_value.rs +++ /dev/null @@ -1,31 +0,0 @@ -extern crate fluent; - -mod helpers; - -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; - -#[test] -fn format_message() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( - " -foo = Foo -", - )); - - assert_format_no_errors(bundle.format("foo", None), "Foo"); -} - -#[test] -fn format_attribute() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( - " -foo = Foo - .attr = Foo Attr -", - )); - - assert_format_no_errors(bundle.format("foo.attr", None), "Foo Attr"); -}