From d12f5ef40f9e73f0fb6b0465c8995c6f34128405 Mon Sep 17 00:00:00 2001 From: ike709 Date: Sun, 26 Jan 2025 23:40:49 -0600 Subject: [PATCH] Support `@(XYZ)` raw strings (#2110) Co-authored-by: ike709 Co-authored-by: wixoa --- .../DMProject/Tests/Expression/String/raw2.dm | 7 ++ .../DMProject/Tests/Expression/String/raw3.dm | 7 ++ .../DMPreprocessor/DMPreprocessorLexer.cs | 72 ++++++++++++++++++- 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 Content.Tests/DMProject/Tests/Expression/String/raw2.dm create mode 100644 Content.Tests/DMProject/Tests/Expression/String/raw3.dm diff --git a/Content.Tests/DMProject/Tests/Expression/String/raw2.dm b/Content.Tests/DMProject/Tests/Expression/String/raw2.dm new file mode 100644 index 0000000000..01f9e97343 --- /dev/null +++ b/Content.Tests/DMProject/Tests/Expression/String/raw2.dm @@ -0,0 +1,7 @@ + +//# issue 380 + +/proc/RunTest() + var/a = @(ZZZ) +asdfZZZ + ASSERT(a == "asdf") \ No newline at end of file diff --git a/Content.Tests/DMProject/Tests/Expression/String/raw3.dm b/Content.Tests/DMProject/Tests/Expression/String/raw3.dm new file mode 100644 index 0000000000..b750f3610b --- /dev/null +++ b/Content.Tests/DMProject/Tests/Expression/String/raw3.dm @@ -0,0 +1,7 @@ + +//# issue 380 + +/proc/RunTest() + var/a = @(ZZQ) +asdfZZQ + ASSERT(a == "asdf") \ No newline at end of file diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index 83a3a3a403..659cc3c03b 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -301,12 +301,31 @@ public Token NextToken(bool ignoreWhitespace = false) { } case '@': { //Raw string char delimiter = Advance(); + var startLoc = CurrentLocation(); + + // @(XYZ) where XYZ is the delimiter + string complexDelimiter = string.Empty; + if (delimiter == '(') { + Advance(); + while (GetCurrent() != ')') { + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + break; + } + + complexDelimiter += GetCurrent(); + Advance(); + } + } TokenTextBuilder.Clear(); TokenTextBuilder.Append('@'); TokenTextBuilder.Append(delimiter); + bool isComplex = complexDelimiter != string.Empty; bool isLong = false; + c = Advance(); if (delimiter == '{') { TokenTextBuilder.Append(c); @@ -314,7 +333,33 @@ public Token NextToken(bool ignoreWhitespace = false) { if (c == '"') isLong = true; } - if (isLong) { + if (isComplex) { + TokenTextBuilder.Append(complexDelimiter); + TokenTextBuilder.Append(')'); + + // Ignore a newline immediately after @(complexDelimiter) + if (c == '\r') c = Advance(); + if (c == '\n') c = Advance(); + + var delimIdx = 0; + do { + TokenTextBuilder.Append(c); + + if (GetCurrent() == complexDelimiter[delimIdx]) delimIdx++; + else delimIdx = 0; + + if (delimIdx == complexDelimiter.Length && c == complexDelimiter[^1]) { // latter check ensures a 1-char delimiter actually matches + break; + } + + c = Advance(); + } while (!AtEndOfSource()); + + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + } + } else if (isLong) { bool nextCharCanTerm = false; Advance(); @@ -335,6 +380,11 @@ public Token NextToken(bool ignoreWhitespace = false) { if (c == '"') nextCharCanTerm = true; } while (!AtEndOfSource()); + + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + } } else { while (c != delimiter && !AtLineEnd() && !AtEndOfSource()) { TokenTextBuilder.Append(c); @@ -342,19 +392,31 @@ public Token NextToken(bool ignoreWhitespace = false) { } } - TokenTextBuilder.Append(c); + if (!isComplex) TokenTextBuilder.Append(c); + if (!HandleLineEnd()) Advance(); string text = TokenTextBuilder.ToString(); string value; - if (isLong) { + if (isComplex) { + // Complex strings need to strip @(complexDelimiter) and a potential final newline. Newline after @(complexDelimiter) is already handled + var trimEnd = complexDelimiter.Length; + if (TokenTextBuilder[^(complexDelimiter.Length + 1)] == '\n') trimEnd += 1; + if (TokenTextBuilder[^(complexDelimiter.Length + 2)] == '\r') trimEnd += 1; + var trimStart = 3 + complexDelimiter.Length; // 3 is from these chars: @() + value = TokenTextBuilder.ToString(trimStart, TokenTextBuilder.Length - (trimStart + trimEnd)); + } else if (isLong) { // Long strings ignore a newline immediately after the @{" and before the "} + if (TokenTextBuilder[3] == '\r') + TokenTextBuilder.Remove(3, 1); if (TokenTextBuilder[3] == '\n') TokenTextBuilder.Remove(3, 1); if (TokenTextBuilder[^3] == '\n') TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); + if (TokenTextBuilder[^3] == '\r') + TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); value = TokenTextBuilder.ToString(3, TokenTextBuilder.Length - 5); } else { @@ -639,6 +701,10 @@ private char GetCurrent() { return _current; } + private Location CurrentLocation() { + return new Location(File, _previousLine, _previousColumn, _isDMStandard); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private char Advance() { int value = _source.Read();