From 16128cdb139fc547c92bde7b1251ffc18c7b4b3f Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Mon, 2 Dec 2024 07:28:19 -0800 Subject: [PATCH] Add more tests --- .../Services/TokenizerTests.cs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/NUnitEngine/nunit.engine.tests/Services/TokenizerTests.cs b/src/NUnitEngine/nunit.engine.tests/Services/TokenizerTests.cs index 4d5ad1818..babc0dd5f 100644 --- a/src/NUnitEngine/nunit.engine.tests/Services/TokenizerTests.cs +++ b/src/NUnitEngine/nunit.engine.tests/Services/TokenizerTests.cs @@ -42,6 +42,40 @@ public void WordsInUnicode() Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Eof))); } + [Test] + public void WordsWithSpecialCharacters() + { + var tokenizer = new Tokenizer("word_with_underscores word-with-dashes word.with.dots"); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word_with_underscores"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word-with-dashes"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word.with.dots"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Eof))); + } + + private const string WORD_BREAK_CHARS = "=!()&| \t,"; + [Test] + public void WordBreakCharacters() + { + var tokenizer = new Tokenizer("word1==word2!=word3 func(arg1, arg2) this&&that||both"); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word1"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, "=="))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word2"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, "!="))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "word3"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "func"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, "("))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "arg1"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, ","))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "arg2"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, ")"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "this"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, "&&"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "that"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Symbol, "||"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Word, "both"))); + Assert.That(tokenizer.NextToken(), Is.EqualTo(new Token(TokenKind.Eof))); + } + [Test] public void StringWithDoubleQuotes() {