Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support @(XYZ) raw strings #2110

Merged
merged 11 commits into from
Jan 27, 2025
Merged

Support @(XYZ) raw strings #2110

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ba72e61
Support `@(XYZ)` raw strings
Nov 27, 2024
4f5027b
more robust checking
Nov 27, 2024
61defaa
cope with \r in long strings
Nov 27, 2024
16c8fbf
I don't think this advance() was needed?
ike709 Nov 27, 2024
7b358e7
zoo wee mama
ike709 Nov 27, 2024
d013861
add another comment
ike709 Nov 27, 2024
7725623
yep that advance() was needed
Nov 27, 2024
0c224e9
Merge remote-tracking branch 'ike709/longstringfix' into longstringfix
Nov 27, 2024
a746d10
Update DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs
ike709 Jan 27, 2025
730862b
Merge branch 'master' into longstringfix
ike709 Jan 27, 2025
55431e4
remove pointless stringbuilder operations
Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 ...ct/Broken Tests/Expression/String/raw2.dm → ...DMProject/Tests/Expression/String/raw2.dm
View file
Open in desktop
File renamed without changes.
74 changes: 72 additions & 2 deletions DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,20 +301,68 @@ public Token NextToken(bool ignoreWhitespace = false) {
}
case '@': { //Raw string
char delimiter = Advance();
var startLoc = CurrentLocation();

// @(XYZ) where XYZ is the delimiter
string complexDelimiter = string.Empty;
if (delimiter == '(') {
Advance();
while (GetCurrent() != ')') {
if (AtEndOfSource()) {
_compiler.Emit(WarningCode.BadExpression, startLoc,
"Unterminated string delimiter");
break;
}

complexDelimiter += GetCurrent();
Advance();
}
}

TokenTextBuilder.Clear();
TokenTextBuilder.Append('@');
TokenTextBuilder.Append(delimiter);

bool isComplex = complexDelimiter != string.Empty;
bool isLong = false;

c = Advance();
if (delimiter == '{') {
TokenTextBuilder.Append(c);

if (c == '"') isLong = true;
}

if (isLong) {
if (isComplex) {
TokenTextBuilder.Append(complexDelimiter);
TokenTextBuilder.Append(')');

// Ignore a newline immediately after @(complexDelimiter)
if (c == '\r') c = Advance();
if (c == '\n') c = Advance();

var delimIdx = 0;
do {
TokenTextBuilder.Append(c);

if (GetCurrent() == complexDelimiter[delimIdx]) delimIdx++;
else delimIdx = 0;

if (delimIdx + 1 == complexDelimiter.Length && c == complexDelimiter[^1]) { // latter check ensures a 1-char delimiter actually matches
TokenTextBuilder.Remove(TokenTextBuilder.Length - delimIdx, complexDelimiter.Length - 1);
break;
}

c = Advance();
} while (!AtEndOfSource());

if (AtEndOfSource()) {
_compiler.Emit(WarningCode.BadExpression, startLoc,
"Unterminated string delimiter");
}

TokenTextBuilder.Append(complexDelimiter);
} else if (isLong) {
bool nextCharCanTerm = false;

Advance();
Expand All @@ -335,6 +383,11 @@ public Token NextToken(bool ignoreWhitespace = false) {
if (c == '"')
nextCharCanTerm = true;
} while (!AtEndOfSource());

if (AtEndOfSource()) {
_compiler.Emit(WarningCode.BadExpression, startLoc,
"Unterminated string delimiter");
}
} else {
while (c != delimiter && !AtLineEnd() && !AtEndOfSource()) {
TokenTextBuilder.Append(c);
Expand All @@ -343,18 +396,31 @@ public Token NextToken(bool ignoreWhitespace = false) {
}

TokenTextBuilder.Append(c);

if (!HandleLineEnd())
Advance();

string text = TokenTextBuilder.ToString();
string value;

if (isLong) {
if (isComplex) {
// Complex strings need to strip @(complexDelimiter) and a potential final newline. Newline after @(complexDelimiter) is already handled
var trimEnd = complexDelimiter.Length;
if (TokenTextBuilder[^(complexDelimiter.Length + 1)] == '\n') trimEnd += 1;
if (TokenTextBuilder[^(complexDelimiter.Length + 2)] == '\r') trimEnd += 1;
var trimStart = 3 + complexDelimiter.Length;
value = TokenTextBuilder.ToString(trimStart, TokenTextBuilder.Length - (trimStart + trimEnd));
}
else if (isLong) {
// Long strings ignore a newline immediately after the @{" and before the "}
if (TokenTextBuilder[3] == '\r')
TokenTextBuilder.Remove(3, 1);
if (TokenTextBuilder[3] == '\n')
TokenTextBuilder.Remove(3, 1);
if (TokenTextBuilder[^3] == '\n')
TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1);
if (TokenTextBuilder[^3] == '\r')
TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1);

value = TokenTextBuilder.ToString(3, TokenTextBuilder.Length - 5);
} else {
Expand Down Expand Up @@ -639,6 +705,10 @@ private char GetCurrent() {
return _current;
}

private Location CurrentLocation() {
return new Location(File, _previousLine, _previousColumn, _isDMStandard);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private char Advance() {
int value = _source.Read();
Expand Down
Loading