Skip to content

Commit 86941f1

Browse files
CjiWCjiW
authored
Merge pull request #155 from Pivot-Studio/fix/space
fix: catch err when missing space
2 parents a75c383 + dc10277 commit 86941f1

File tree

15 files changed

+179
-93
lines changed

15 files changed

+179
-93
lines changed

‎src/nomparser/array.rs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use std::fmt::Error;
22

3+
use internal_macro::test_parser;
34
use nom::{
45
branch::alt,
56
bytes::complete::tag,
@@ -18,15 +19,23 @@ use crate::{
1819

1920
use super::*;
2021

22+
#[test_parser("[1,2,3]")]
23+
#[test_parser(
24+
"[
25+
1,
26+
2,
27+
x
28+
]"
29+
)]
2130
pub fn array_init(input: Span) -> IResult<Span, Box<NodeEnum>> {
2231
map_res(
2332
tuple((
24-
tag_token(TokenType::LBRACKET),
33+
tag_token_symbol(TokenType::LBRACKET),
2534
separated_list0(
26-
tag_token(TokenType::COMMA),
35+
tag_token_symbol(TokenType::COMMA),
2736
del_newline_or_space!(logic_exp),
2837
),
29-
tag_token(TokenType::RBRACKET),
38+
tag_token_symbol(TokenType::RBRACKET),
3039
)),
3140
|((_, lb), exps, (_, rb))| {
3241
let range = lb.start.to(rb.end);
@@ -35,15 +44,16 @@ pub fn array_init(input: Span) -> IResult<Span, Box<NodeEnum>> {
3544
)(input)
3645
}
3746

47+
#[test_parser("[123]")]
3848
/// ```ebnf
3949
/// array_element_op = ('[' logic_exp ']') ;
4050
/// ```
4151
pub fn array_element_op(input: Span) -> IResult<Span, (ComplexOp, Vec<Box<NodeEnum>>)> {
4252
delspace(map_res(
4353
tuple((
44-
tag_token(TokenType::LBRACKET),
54+
tag_token_symbol(TokenType::LBRACKET),
4555
opt(logic_exp),
46-
tag_token(TokenType::RBRACKET),
56+
tag_token_symbol(TokenType::RBRACKET),
4757
many0(comment),
4858
)),
4959
|(_, idx, (_, rr), com)| {

‎src/nomparser/comment.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use crate::nomparser::Span;
22
use crate::{ast::node::comment::CommentNode, ast::range::Range};
3+
use internal_macro::{test_parser, test_parser_error};
34
use nom::{
45
branch::alt,
56
bytes::complete::{tag, take_until},
@@ -11,6 +12,9 @@ use nom_locate::LocatedSpan;
1112

1213
use super::*;
1314

15+
#[test_parser("//123")]
16+
#[test_parser("/// 123\n")]
17+
#[test_parser_error("/ / 123\n")]
1418
pub fn comment(input: Span) -> IResult<Span, Box<NodeEnum>> {
1519
map_res(
1620
pair(

‎src/nomparser/constval.rs‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,16 @@ pub fn number(input: Span) -> IResult<Span, Box<NodeEnum>> {
3636
Ok((re, Box::new(node.into())))
3737
}
3838

39-
#[test_parser("true")]
39+
#[test_parser(" true")]
4040
#[test_parser("false")]
4141
#[test_parser_error("tru")]
4242
#[test_parser_error("fales")]
4343
pub fn bool_const(input: Span) -> IResult<Span, Box<NodeEnum>> {
4444
alt((
45-
map_res(tag_token(TokenType::TRUE), |(_, range)| {
45+
map_res(tag_token_word(TokenType::TRUE), |(_, range)| {
4646
res_enum(BoolConstNode { value: true, range }.into())
4747
}),
48-
map_res(tag_token(TokenType::FALSE), |(_, range)| {
48+
map_res(tag_token_word(TokenType::FALSE), |(_, range)| {
4949
res_enum(
5050
BoolConstNode {
5151
value: false,
@@ -57,6 +57,10 @@ pub fn bool_const(input: Span) -> IResult<Span, Box<NodeEnum>> {
5757
))(input)
5858
}
5959

60+
#[test_parser("123")]
61+
#[test_parser("12_3")]
62+
#[test_parser("1_2_3")]
63+
#[test_parser_error("1 23")]
6064
fn decimal(input: Span) -> IResult<Span, Span> {
6165
recognize(many1(terminated(one_of("0123456789"), many0(char('_')))))(input)
6266
}
@@ -70,8 +74,8 @@ fn float(input: Span) -> IResult<Span, Span> {
7074
opt(tuple((
7175
one_of("eE"),
7276
opt(alt((
73-
tag_token(TokenType::PLUS),
74-
tag_token(TokenType::MINUS),
77+
tag_token_symbol(TokenType::PLUS),
78+
tag_token_symbol(TokenType::MINUS),
7579
))),
7680
decimal,
7781
))),
@@ -81,8 +85,8 @@ fn float(input: Span) -> IResult<Span, Span> {
8185
opt(preceded(char('.'), decimal)),
8286
one_of("eE"),
8387
opt(alt((
84-
tag_token(TokenType::PLUS),
85-
tag_token(TokenType::MINUS),
88+
tag_token_symbol(TokenType::PLUS),
89+
tag_token_symbol(TokenType::MINUS),
8690
))),
8791
decimal,
8892
))), // Case three: 42. and 42.42

‎src/nomparser/control.rs‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,25 @@ use super::*;
4545
return;
4646
}"#
4747
)]
48+
#[test_parser_error(
49+
"ifa > 1 {
50+
a = 1;
51+
} else {
52+
a = 2;
53+
}"
54+
)]
4855
/// ```ebnf
4956
/// if_statement = "if" logic_exp statement_block ("else" (if_statement | statement_block))? ;
5057
/// ```
5158
pub fn if_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
5259
map_res(
5360
delspace(tuple((
54-
tag_token(TokenType::IF),
61+
tag_token_word(TokenType::IF),
5562
parse_with_ex(logic_exp, true),
5663
statement_block,
5764
opt(delspace(comment)),
5865
opt(preceded(
59-
tag_token(TokenType::ELSE),
66+
tag_token_word(TokenType::ELSE),
6067
alt((
6168
if_statement,
6269
map_res(statement_block, |n| res_enum(n.into())),
@@ -98,13 +105,19 @@ pub fn if_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
98105
}
99106
"
100107
)]
108+
#[test_parser_error(
109+
"whiletrue {
110+
let a = b;
111+
}
112+
"
113+
)]
101114
/// ```ebnf
102115
/// while_statement = "while" logic_exp statement_block ;
103116
/// ```
104117
pub fn while_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
105118
map_res(
106119
delspace(tuple((
107-
tag_token(TokenType::WHILE),
120+
tag_token_word(TokenType::WHILE),
108121
alt_except(
109122
logic_exp,
110123
"{",
@@ -159,18 +172,22 @@ pub fn while_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
159172
160173
}"
161174
)]
162-
175+
#[test_parser_error(
176+
"forlet i = 0; i < 5; i = i + 1{
177+
178+
}"
179+
)]
163180
/// ```enbf
164181
/// for_statement = "for" (assignment | new_variable) ";" logic_exp ";" assignment statement_block;
165182
/// ```
166183
pub fn for_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
167184
map_res(
168185
delspace(tuple((
169-
tag_token(TokenType::FOR),
186+
tag_token_word(TokenType::FOR),
170187
opt(alt((assignment, new_variable))),
171-
tag_token(TokenType::SEMI),
188+
tag_token_symbol(TokenType::SEMI),
172189
logic_exp,
173-
tag_token(TokenType::SEMI),
190+
tag_token_symbol(TokenType::SEMI),
174191
opt(assignment),
175192
statement_block,
176193
opt(delspace(comment)),
@@ -204,8 +221,8 @@ pub fn for_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
204221
pub fn break_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
205222
map_res(
206223
tuple((
207-
tag_token(TokenType::BREAK),
208-
tag_token(TokenType::SEMI),
224+
tag_token_word(TokenType::BREAK),
225+
tag_token_symbol(TokenType::SEMI),
209226
opt(delspace(comment)),
210227
)),
211228
|(_, _, optcomment)| {
@@ -228,8 +245,8 @@ pub fn break_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
228245
pub fn continue_statement(input: Span) -> IResult<Span, Box<NodeEnum>> {
229246
map_res(
230247
tuple((
231-
tag_token(TokenType::CONTINUE),
232-
tag_token(TokenType::SEMI),
248+
tag_token_word(TokenType::CONTINUE),
249+
tag_token_symbol(TokenType::SEMI),
233250
opt(delspace(comment)),
234251
)),
235252
|(_, _, optcomment)| {

‎src/nomparser/expression.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,10 @@ fn unary_exp(input: Span) -> IResult<Span, Box<NodeEnum>> {
5656
pointer_exp,
5757
map_res(
5858
tuple((
59-
alt((tag_token(TokenType::MINUS), tag_token(TokenType::NOT))),
59+
alt((
60+
tag_token_symbol(TokenType::MINUS),
61+
tag_token_symbol(TokenType::NOT),
62+
)),
6063
pointer_exp,
6164
)),
6265
|((op, op_range), exp)| {
@@ -83,8 +86,8 @@ pub fn pointer_exp(input: Span) -> IResult<Span, Box<NodeEnum>> {
8386
map_res(
8487
delspace(pair(
8588
many0(alt((
86-
tag_token(TokenType::TAKE_PTR),
87-
tag_token(TokenType::TAKE_VAL),
89+
tag_token_symbol(TokenType::TAKE_PTR),
90+
tag_token_symbol(TokenType::TAKE_VAL),
8891
))),
8992
complex_exp,
9093
)),
@@ -221,7 +224,7 @@ fn primary_exp(input: Span) -> IResult<Span, Box<NodeEnum>> {
221224
fn take_exp_op(input: Span) -> IResult<Span, (ComplexOp, Vec<Box<NodeEnum>>)> {
222225
delspace(map_res(
223226
preceded(
224-
tag_token(TokenType::DOT),
227+
tag_token_symbol(TokenType::DOT),
225228
pair(opt(identifier), many0(comment)),
226229
),
227230
|(idx, coms)| Ok::<_, Error>((ComplexOp::FieldOp(idx), coms)),
@@ -236,9 +239,9 @@ fn take_exp_op(input: Span) -> IResult<Span, (ComplexOp, Vec<Box<NodeEnum>>)> {
236239
fn parantheses_exp(input: Span) -> IResult<Span, Box<NodeEnum>> {
237240
map_res(
238241
delimited(
239-
tag_token(TokenType::LPAREN),
242+
tag_token_symbol(TokenType::LPAREN),
240243
parse_with_ex(logic_exp, false),
241-
tag_token(TokenType::RPAREN),
244+
tag_token_symbol(TokenType::RPAREN),
242245
),
243246
|exp| {
244247
res_enum(

‎src/nomparser/function.rs‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use nom::{
1212
use crate::nomparser::Span;
1313
use crate::{ast::node::function::FuncDefNode, ast::tokens::TokenType};
1414

15-
use internal_macro::test_parser;
15+
use internal_macro::{test_parser, test_parser_error};
1616

1717
use super::*;
1818

@@ -44,25 +44,26 @@ use super::*;
4444
"
4545
)]
4646
#[test_parser("fn f( \n) int;")]
47+
#[test_parser_error("fnf( \n) int;")]
4748
pub fn function_def(input: Span) -> IResult<Span, Box<TopLevel>> {
4849
map_res(
4950
tuple((
5051
many0(del_newline_or_space!(comment)),
51-
tag_token(TokenType::FN),
52+
tag_token_word(TokenType::FN),
5253
identifier,
5354
opt(generic_type_def),
54-
tag_token(TokenType::LPAREN),
55+
tag_token_symbol(TokenType::LPAREN),
5556
del_newline_or_space!(separated_list0(
56-
tag_token(TokenType::COMMA),
57+
tag_token_symbol(TokenType::COMMA),
5758
del_newline_or_space!(typed_identifier),
5859
)),
59-
tag_token(TokenType::RPAREN),
60+
tag_token_symbol(TokenType::RPAREN),
6061
type_name,
6162
alt((
6263
map_res(statement_block, |b| {
6364
Ok::<_, Error>((Some(b.clone()), b.range))
6465
}),
65-
map_res(tag_token(TokenType::SEMI), |(_, range)| {
66+
map_res(tag_token_symbol(TokenType::SEMI), |(_, range)| {
6667
Ok::<_, Error>((None, range))
6768
}),
6869
)),
@@ -104,12 +105,12 @@ pub fn call_function_op(input: Span) -> IResult<Span, (ComplexOp, Vec<Box<NodeEn
104105
delspace(map_res(
105106
tuple((
106107
opt(generic_param_def),
107-
tag_token(TokenType::LPAREN),
108+
tag_token_symbol(TokenType::LPAREN),
108109
del_newline_or_space!(separated_list0(
109-
tag_token(TokenType::COMMA),
110+
tag_token_symbol(TokenType::COMMA),
110111
del_newline_or_space!(logic_exp)
111112
)),
112-
tag_token(TokenType::RPAREN),
113+
tag_token_symbol(TokenType::RPAREN),
113114
many0(comment),
114115
)),
115116
|(generic, (_, st), paras, (_, end), com)| {

‎src/nomparser/helper.rs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,40 @@ use std::fmt::Error;
22

33
use crate::nomparser::Span;
44
use crate::{ast::range::Range, ast::tokens::TokenType};
5+
use nom::character::is_alphanumeric;
6+
use nom::sequence::preceded;
57
use nom::{
68
bytes::complete::tag, character::complete::space0, combinator::map_res, error::ParseError,
79
sequence::delimited, AsChar, IResult, InputTake, InputTakeAtPosition, Parser,
810
};
911

1012
use super::*;
1113

12-
pub fn tag_token(token: TokenType) -> impl Fn(Span) -> IResult<Span, (TokenType, Range)> {
14+
pub fn tag_token_symbol(token: TokenType) -> impl Fn(Span) -> IResult<Span, (TokenType, Range)> {
1315
move |input| {
1416
map_res(delspace(tag(token.get_str())), |_out: Span| {
1517
let end = _out.take_split(token.get_str().len()).0;
1618
Ok::<(TokenType, Range), Error>((token, Range::new(_out, end)))
1719
})(input)
1820
}
1921
}
22+
23+
/// 不能直接接 `字母`、`数字` 或 `_`，用于关键字
24+
pub fn tag_token_word(token: TokenType) -> impl Fn(Span) -> IResult<Span, (TokenType, Range)> {
25+
move |input| {
26+
let (s1, s2): (LocatedSpan<&str, bool>, LocatedSpan<&str, bool>) =
27+
preceded(space0, tag(token.get_str()))(input)?;
28+
if s1.starts_with(|c: char| is_alphanumeric(c as u8) || c == '_') {
29+
return Err(nom::Err::Error(nom::error::Error::new(
30+
s2,
31+
nom::error::ErrorKind::Tag,
32+
)));
33+
} else {
34+
let end = s2.take_split(token.get_str().len()).0;
35+
return Ok((s1, (token, Range::new(s2, end))));
36+
}
37+
}
38+
}
2039
pub fn delspace<I, O, E, G>(parser: G) -> impl FnMut(I) -> IResult<I, O, E>
2140
where
2241
G: Parser<I, O, E>,

‎src/nomparser/identifier.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,12 @@ use super::*;
3737
pub fn extern_identifier(input: Span) -> IResult<Span, Box<NodeEnum>> {
3838
delspace(map_res(
3939
tuple((
40-
separated_list1(tag_token(TokenType::DOUBLE_COLON), delspace(identifier)),
41-
opt(tag_token(TokenType::DOUBLE_COLON)), // 容忍未写完的语句
42-
opt(tag_token(TokenType::COLON)), // 容忍未写完的语句
40+
separated_list1(
41+
tag_token_symbol(TokenType::DOUBLE_COLON),
42+
delspace(identifier),
43+
),
44+
opt(tag_token_symbol(TokenType::DOUBLE_COLON)), // 容忍未写完的语句
45+
opt(tag_token_symbol(TokenType::COLON)), // 容忍未写完的语句
4346
)),
4447
|(mut ns, opt, opt2)| {
4548
let id = ns.pop().unwrap();
@@ -88,7 +91,7 @@ pub fn typed_identifier(input: Span) -> IResult<Span, Box<TypedIdentifierNode>>
8891
delspace(map_res(
8992
tuple((
9093
identifier,
91-
tag_token(TokenType::COLON),
94+
tag_token_symbol(TokenType::COLON),
9295
opt(type_name),
9396
opt(comment),
9497
)),

0 commit comments

Comments
 (0)