Skip to content
This repository was archived by the owner on Jul 24, 2024. It is now read-only.

Commit 7cdb331

Browse files
ryangjchandlerryangjchandler
committed
lexer+parser: remove opening and closes quotes from strings
1 parent fee6cef commit 7cdb331

File tree

52 files changed

+296
-161
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+296
-161
lines changed

‎schema.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6894,10 +6894,14 @@
68946894
"LiteralString": {
68956895
"type": "object",
68966896
"required": [
6897+
"kind",
68976898
"span",
68986899
"value"
68996900
],
69006901
"properties": {
6902+
"kind": {
6903+
"$ref": "#/definitions/LiteralStringKind"
6904+
},
69016905
"span": {
69026906
"$ref": "#/definitions/Span"
69036907
},
@@ -6906,6 +6910,13 @@
69066910
}
69076911
}
69086912
},
6913+
"LiteralStringKind": {
6914+
"type": "string",
6915+
"enum": [
6916+
"SingleQuoted",
6917+
"DoubleQuoted"
6918+
]
6919+
},
69096920
"LiteralStringPart": {
69106921
"type": "object",
69116922
"required": [

‎src/lexer/mod.rs

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -293,20 +293,20 @@ impl Lexer {
293293
}
294294
// Single quoted string.
295295
[b'\'', ..] => {
296-
let opening = state.source.read_and_skip(1);
297-
self.tokenize_single_quote_string(state, opening)?
296+
state.source.skip(1);
297+
self.tokenize_single_quote_string(state)?
298298
}
299299
[b'b' | b'B', b'\'', ..] => {
300-
let opening = state.source.read_and_skip(2);
301-
self.tokenize_single_quote_string(state, opening)?
300+
state.source.skip(2);
301+
self.tokenize_single_quote_string(state)?
302302
}
303303
[b'"', ..] => {
304-
let opening = state.source.read_and_skip(1);
305-
self.tokenize_double_quote_string(state, opening)?
304+
state.source.skip(1);
305+
self.tokenize_double_quote_string(state)?
306306
}
307307
[b'b' | b'B', b'"', ..] => {
308-
let opening = state.source.read_and_skip(2);
309-
self.tokenize_double_quote_string(state, opening)?
308+
state.source.skip(2);
309+
self.tokenize_double_quote_string(state)?
310310
}
311311
[b'$', ident_start!(), ..] => self.tokenize_variable(state),
312312
[b'$', ..] => {
@@ -1470,14 +1470,12 @@ impl Lexer {
14701470
fn tokenize_single_quote_string(
14711471
&self,
14721472
state: &mut State,
1473-
opening: &[u8],
14741473
) -> SyntaxResult<(TokenKind, ByteString)> {
1475-
let mut buffer = opening.to_vec();
1474+
let mut buffer = vec![];
14761475

14771476
loop {
14781477
match state.source.read(2) {
14791478
[b'\'', ..] => {
1480-
buffer.push(b'\'');
14811479
state.source.next();
14821480
break;
14831481
}
@@ -1493,20 +1491,18 @@ impl Lexer {
14931491
}
14941492
}
14951493

1496-
Ok((TokenKind::LiteralString, buffer.into()))
1494+
Ok((TokenKind::LiteralSingleQuotedString, buffer.into()))
14971495
}
14981496

14991497
fn tokenize_double_quote_string(
15001498
&self,
15011499
state: &mut State,
1502-
opening: &[u8],
15031500
) -> SyntaxResult<(TokenKind, ByteString)> {
1504-
let mut buffer = opening.to_vec();
1501+
let mut buffer = vec![];
15051502

15061503
let constant = loop {
15071504
match state.source.read(3) {
15081505
[b'"', ..] => {
1509-
buffer.push(b'"');
15101506
state.source.next();
15111507
break true;
15121508
}
@@ -1614,7 +1610,7 @@ impl Lexer {
16141610
};
16151611

16161612
Ok(if constant {
1617-
(TokenKind::LiteralString, buffer.into())
1613+
(TokenKind::LiteralDoubleQuotedString, buffer.into())
16181614
} else {
16191615
state.replace(StackFrame::DoubleQuote);
16201616
(TokenKind::StringPart, buffer.into())

‎src/lexer/token.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ pub enum TokenKind {
134134
MultiLineComment,
135135
DocumentComment,
136136
Const,
137-
LiteralString,
137+
LiteralSingleQuotedString,
138+
LiteralDoubleQuotedString,
138139
Continue,
139140
CurlyOpen,
140141
Declare,
@@ -474,7 +475,8 @@ impl Display for TokenKind {
474475
| Self::QualifiedIdentifier
475476
| Self::Identifier
476477
| Self::FullyQualifiedIdentifier
477-
| Self::LiteralString
478+
| Self::LiteralSingleQuotedString
479+
| Self::LiteralDoubleQuotedString
478480
| Self::SingleLineComment
479481
| Self::MultiLineComment
480482
| Self::HashMarkComment

‎src/parser/ast/literals.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ impl Node for Literal {
2929
pub struct LiteralString {
3030
pub value: ByteString,
3131
pub span: Span,
32+
pub kind: LiteralStringKind,
33+
}
34+
35+
#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
36+
pub enum LiteralStringKind {
37+
SingleQuoted,
38+
DoubleQuoted,
3239
}
3340

3441
impl Node for LiteralString {

‎src/parser/error.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,10 @@ pub fn unexpected_token(expected: Vec<String>, found: &Token) -> ParseError {
175175
| TokenKind::QualifiedIdentifier
176176
| TokenKind::FullyQualifiedIdentifier => ("identifier".to_string(), false),
177177
TokenKind::Variable => ("variable".to_string(), false),
178-
TokenKind::LiteralInteger | TokenKind::LiteralFloat | TokenKind::LiteralString => {
179-
("literal".to_string(), false)
180-
}
178+
TokenKind::LiteralInteger
179+
| TokenKind::LiteralFloat
180+
| TokenKind::LiteralSingleQuotedString
181+
| TokenKind::LiteralDoubleQuotedString => ("literal".to_string(), false),
181182
_ => (format!("token `{}`", found.value), false),
182183
},
183184
};

‎src/parser/expressions.rs

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ use crate::parser::internal::utils;
4040
use crate::parser::internal::variables;
4141
use crate::parser::state::State;
4242

43+
use super::ast::literals::LiteralStringKind;
4344
use super::ast::BoolExpression;
4445
use super::ast::CastExpression;
4546
use super::ast::CloneExpression;
@@ -848,17 +849,28 @@ expressions! {
848849
}
849850
})
850851

851-
#[before(string_part), current(TokenKind::LiteralString)]
852+
#[before(string_part), current(TokenKind::LiteralSingleQuotedString | TokenKind::LiteralDoubleQuotedString)]
852853
literal_string({
853854
let current = state.stream.current();
854855

855-
if let TokenKind::LiteralString = &current.kind {
856+
if let TokenKind::LiteralSingleQuotedString = &current.kind {
856857
state.stream.next();
857858

858859
Ok(Expression::Literal(
859860
Literal::String(LiteralString {
860861
span: current.span,
861-
value: current.value.clone()
862+
value: current.value.clone(),
863+
kind: LiteralStringKind::SingleQuoted,
864+
})
865+
))
866+
} else if let TokenKind::LiteralDoubleQuotedString = &current.kind {
867+
state.stream.next();
868+
869+
Ok(Expression::Literal(
870+
Literal::String(LiteralString {
871+
span: current.span,
872+
value: current.value.clone(),
873+
kind: LiteralStringKind::DoubleQuoted,
862874
})
863875
))
864876
} else {

‎src/parser/internal/strings.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use crate::parser::ast::identifiers::Identifier;
77
use crate::parser::ast::literals::Literal;
88
use crate::parser::ast::literals::LiteralInteger;
99
use crate::parser::ast::literals::LiteralString;
10+
use crate::parser::ast::literals::LiteralStringKind;
1011
use crate::parser::ast::operators::ArithmeticOperationExpression;
1112
use crate::parser::ast::variables::Variable;
1213
use crate::parser::ast::ExpressionStringPart;
@@ -223,7 +224,10 @@ pub fn nowdoc(state: &mut State) -> ParseResult<Expression> {
223224
string_part = bytes.into();
224225
}
225226

226-
Ok(Expression::Nowdoc(NowdocExpression { label, value: string_part }))
227+
Ok(Expression::Nowdoc(NowdocExpression {
228+
label,
229+
value: string_part,
230+
}))
227231
}
228232

229233
fn part(state: &mut State) -> ParseResult<Option<StringPart>> {
@@ -303,6 +307,7 @@ fn part(state: &mut State) -> ParseResult<Option<StringPart>> {
303307
Expression::Literal(Literal::String(LiteralString {
304308
span: current.span,
305309
value: current.value.clone(),
310+
kind: LiteralStringKind::SingleQuoted,
306311
}))
307312
}
308313
TokenKind::Variable => Expression::Variable(Variable::SimpleVariable(

‎src/parser/macros.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,18 @@ macro_rules! expect_literal {
7373
},
7474
)
7575
}
76-
TokenKind::LiteralString => {
76+
TokenKind::LiteralSingleQuotedString | TokenKind::LiteralDoubleQuotedString => {
7777
$state.stream.next();
7878

7979
$crate::parser::ast::literals::Literal::String(
8080
$crate::parser::ast::literals::LiteralString {
8181
span: current.span,
8282
value: current.value.clone(),
83+
kind: if matches!(current.kind, TokenKind::LiteralSingleQuotedString) {
84+
$crate::parser::ast::literals::LiteralStringKind::SingleQuoted
85+
} else {
86+
$crate::parser::ast::literals::LiteralStringKind::DoubleQuoted
87+
},
8388
},
8489
)
8590
}

‎tests/fixtures/0001/ast.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,13 @@
6666
Literal(
6767
String(
6868
LiteralString {
69-
value: """",
69+
value: "",
7070
span: Span {
7171
line: 3,
7272
column: 26,
7373
position: 32,
7474
},
75+
kind: DoubleQuoted,
7576
},
7677
),
7778
),

‎tests/fixtures/0006/ast.txt

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,13 @@
2020
path: Literal(
2121
String(
2222
LiteralString {
23-
value: "'foo.php'",
23+
value: "foo.php",
2424
span: Span {
2525
line: 3,
2626
column: 9,
2727
position: 15,
2828
},
29+
kind: SingleQuoted,
2930
},
3031
),
3132
),
@@ -52,12 +53,13 @@
5253
path: Literal(
5354
String(
5455
LiteralString {
55-
value: "'bar.php'",
56+
value: "bar.php",
5657
span: Span {
5758
line: 5,
5859
column: 14,
5960
position: 40,
6061
},
62+
kind: SingleQuoted,
6163
},
6264
),
6365
),
@@ -84,12 +86,13 @@
8486
path: Literal(
8587
String(
8688
LiteralString {
87-
value: "'baz.php'",
89+
value: "baz.php",
8890
span: Span {
8991
line: 7,
9092
column: 9,
9193
position: 60,
9294
},
95+
kind: SingleQuoted,
9396
},
9497
),
9598
),
@@ -116,12 +119,13 @@
116119
path: Literal(
117120
String(
118121
LiteralString {
119-
value: "'qux.php'",
122+
value: "qux.php",
120123
span: Span {
121124
line: 9,
122125
column: 14,
123126
position: 85,
124127
},
128+
kind: SingleQuoted,
125129
},
126130
),
127131
),

0 commit comments

Comments
 (0)