Skip to content

Commit b57843d

Browse files
parrtparrt
authored
Merge pull request antlr#1546 from sharwell/fix-1545
Fix multiple problems with optional block bypass at end of rule
2 parents 644478a + 24b20e2 commit b57843d

File tree

18 files changed

+103
-52
lines changed

18 files changed

+103
-52
lines changed

‎runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserErrorsDescriptors.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public static class InvalidATNStateRemoval extends BaseParserTestDescriptor {
135135
public static class InvalidEmptyInput extends BaseParserTestDescriptor {
136136
public String input = "";
137137
public String output = null;
138-
public String errors = "line 1:0 missing ID at '<EOF>'\n";
138+
public String errors = "line 1:0 mismatched input '<EOF>' expecting ID\n";
139139
public String startRule = "start";
140140
public String grammarName = "T";
141141

@@ -414,7 +414,7 @@ public static class SingleTokenDeletionBeforeLoop extends BaseParserTestDescript
414414

415415
/**
416416
grammar T;
417-
a : 'a' 'b'* ;
417+
a : 'a' 'b'* EOF ;
418418
*/
419419
@CommentHasStringValue
420420
public String grammar;
@@ -436,7 +436,7 @@ public static class SingleTokenDeletionBeforeLoop2 extends BaseParserTestDescrip
436436

437437
/**
438438
grammar T;
439-
a : 'a' ('b'|'z'{<Pass()>})*;
439+
a : 'a' ('b'|'z'{<Pass()>})* EOF ;
440440
*/
441441
@CommentHasStringValue
442442
public String grammar;

‎runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java

+77
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,83 @@ public static class ReferenceToATN_2 extends BaseParserTestDescriptor {
753753
*/
754754
@CommentHasStringValue
755755
public String grammar;
756+
}
757+
758+
/**
759+
* This is a regression test for antlr/antlr4#1545, case 1.
760+
*/
761+
public static class OpenDeviceStatement_Case1 extends BaseParserTestDescriptor {
762+
public String input = "OPEN DEVICE DEVICE";
763+
public String output = "OPEN DEVICE DEVICE\n";
764+
public String errors = null;
765+
public String startRule = "statement";
766+
public String grammarName = "OpenDeviceStatement";
767+
768+
/**
769+
grammar OpenDeviceStatement;
770+
program : statement+ '.' ;
771+
772+
statement : 'OPEN' ( 'DEVICE' ( OPT1 | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
773+
774+
OPT1 : 'OPT-1';
775+
OPT2 : 'OPT-2';
776+
OPT3 : 'OPT-3';
777+
778+
WS : (' '|'\n')+ -> channel(HIDDEN);
779+
*/
780+
@CommentHasStringValue
781+
public String grammar;
782+
}
783+
784+
/**
785+
* This is a regression test for antlr/antlr4#1545, case 2.
786+
*/
787+
public static class OpenDeviceStatement_Case2 extends BaseParserTestDescriptor {
788+
public String input = "OPEN DEVICE DEVICE";
789+
public String output = "OPEN DEVICE DEVICE\n";
790+
public String errors = null;
791+
public String startRule = "statement";
792+
public String grammarName = "OpenDeviceStatement";
793+
794+
/**
795+
grammar OpenDeviceStatement;
796+
program : statement+ '.' ;
797+
798+
statement : 'OPEN' ( 'DEVICE' ( (OPT1) | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
799+
800+
OPT1 : 'OPT-1';
801+
OPT2 : 'OPT-2';
802+
OPT3 : 'OPT-3';
803+
804+
WS : (' '|'\n')+ -> channel(HIDDEN);
805+
*/
806+
@CommentHasStringValue
807+
public String grammar;
808+
}
756809

810+
/**
811+
* This is a regression test for antlr/antlr4#1545, case 3.
812+
*/
813+
public static class OpenDeviceStatement_Case3 extends BaseParserTestDescriptor {
814+
public String input = "OPEN DEVICE DEVICE.";
815+
public String output = "OPEN DEVICE DEVICE\n";
816+
public String errors = null;
817+
public String startRule = "statement";
818+
public String grammarName = "OpenDeviceStatement";
819+
820+
/**
821+
grammar OpenDeviceStatement;
822+
program : statement+ '.' ;
823+
824+
statement : 'OPEN' ( 'DEVICE' ( (OPT1) | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
825+
826+
OPT1 : 'OPT-1';
827+
OPT2 : 'OPT-2';
828+
OPT3 : 'OPT-3';
829+
830+
WS : (' '|'\n')+ -> channel(HIDDEN);
831+
*/
832+
@CommentHasStringValue
833+
public String grammar;
757834
}
758835
}

‎runtime/CSharp/runtime/CSharp/Antlr4.Runtime/DefaultErrorStrategy.cs

+2-6
Original file line numberDiff line numberDiff line change
@@ -265,12 +265,8 @@ public virtual void Sync(Parser recognizer)
265265
ITokenStream tokens = ((ITokenStream)recognizer.InputStream);
266266
int la = tokens.LA(1);
267267
// try cheaper subset first; might get lucky. seems to shave a wee bit off
268-
if (recognizer.Atn.NextTokens(s).Contains(la) || la == TokenConstants.EOF)
269-
{
270-
return;
271-
}
272-
// Return but don't end recovery. only do that upon valid token match
273-
if (recognizer.IsExpectedToken(la))
268+
var nextTokens = recognizer.Atn.NextTokens(s);
269+
if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la))
274270
{
275271
return;
276272
}

‎runtime/Cpp/runtime/src/DefaultErrorStrategy.cpp

+2-6
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,8 @@ void DefaultErrorStrategy::sync(Parser *recognizer) {
100100
size_t la = tokens->LA(1);
101101

102102
// try cheaper subset first; might get lucky. seems to shave a wee bit off
103-
if (recognizer->getATN().nextTokens(s).contains(la) || la == Token::EOF) {
104-
return;
105-
}
106-
107-
// Return but don't end recovery. only do that upon valid token match
108-
if (recognizer->isExpectedToken((int)la)) {
103+
auto nextTokens = recognizer->getATN().nextTokens(s);
104+
if (nextTokens.contains(Token::EPSILON) || nextTokens.contains(la)) {
109105
return;
110106
}
111107

‎runtime/Go/antlr/error_strategy.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -218,11 +218,8 @@ func (d *DefaultErrorStrategy) Sync(recognizer Parser) {
218218
la := recognizer.GetTokenStream().LA(1)
219219

220220
// try cheaper subset first might get lucky. seems to shave a wee bit off
221-
if la == TokenEOF || recognizer.GetATN().NextTokens(s, nil).contains(la) {
222-
return
223-
}
224-
// Return but don't end recovery. only do that upon valid token Match
225-
if recognizer.IsExpectedToken(la) {
221+
nextTokens := recognizer.GetATN().NextTokens(s, nil)
222+
if nextTokens.contains(TokenEpsilon) || nextTokens.contains(la) {
226223
return
227224
}
228225

‎runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -224,10 +224,8 @@ public void sync(Parser recognizer) throws RecognitionException {
224224
int la = tokens.LA(1);
225225

226226
// try cheaper subset first; might get lucky. seems to shave a wee bit off
227-
if ( recognizer.getATN().nextTokens(s).contains(la) || la==Token.EOF ) return;
228-
229-
// Return but don't end recovery. only do that upon valid token match
230-
if (recognizer.isExpectedToken(la)) {
227+
IntervalSet nextTokens = recognizer.getATN().nextTokens(s);
228+
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
231229
return;
232230
}
233231

‎runtime/JavaScript/src/antlr4/error/ErrorStrategy.js

+2-5
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,8 @@ DefaultErrorStrategy.prototype.sync = function(recognizer) {
223223
var s = recognizer._interp.atn.states[recognizer.state];
224224
var la = recognizer.getTokenStream().LA(1);
225225
// try cheaper subset first; might get lucky. seems to shave a wee bit off
226-
if (la===Token.EOF || recognizer.atn.nextTokens(s).contains(la)) {
227-
return;
228-
}
229-
// Return but don't end recovery. only do that upon valid token match
230-
if(recognizer.isExpectedToken(la)) {
226+
var nextTokens = recognizer.atn.nextTokens(s);
227+
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
231228
return;
232229
}
233230
switch (s.stateType) {

‎runtime/Python2/src/antlr4/error/ErrorStrategy.py

+2-5
Original file line numberDiff line numberDiff line change
@@ -202,11 +202,8 @@ def sync(self, recognizer):
202202
s = recognizer._interp.atn.states[recognizer.state]
203203
la = recognizer.getTokenStream().LA(1)
204204
# try cheaper subset first; might get lucky. seems to shave a wee bit off
205-
if la==Token.EOF or la in recognizer.atn.nextTokens(s):
206-
return
207-
208-
# Return but don't end recovery. only do that upon valid token match
209-
if recognizer.isExpectedToken(la):
205+
nextTokens = recognizer.atn.nextTokens(s)
206+
if Token.EPSILON in nextTokens or la in nextTokens:
210207
return
211208

212209
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

‎runtime/Python3/src/antlr4/error/ErrorStrategy.py

+2-5
Original file line numberDiff line numberDiff line change
@@ -207,11 +207,8 @@ def sync(self, recognizer:Parser):
207207
s = recognizer._interp.atn.states[recognizer.state]
208208
la = recognizer.getTokenStream().LA(1)
209209
# try cheaper subset first; might get lucky. seems to shave a wee bit off
210-
if la==Token.EOF or la in recognizer.atn.nextTokens(s):
211-
return
212-
213-
# Return but don't end recovery. only do that upon valid token match
214-
if recognizer.isExpectedToken(la):
210+
nextTokens = recognizer.atn.nextTokens(s)
211+
if Token.EPSILON in nextTokens or la in nextTokens:
215212
return
216213

217214
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

‎runtime/Swift/Antlr4/org/antlr/v4/runtime/DefaultErrorStrategy.swift

+2-3
Original file line numberDiff line numberDiff line change
@@ -226,12 +226,11 @@ public class DefaultErrorStrategy: ANTLRErrorStrategy {
226226
// try cheaper subset first; might get lucky. seems to shave a wee bit off
227227
//let set : IntervalSet = recognizer.getATN().nextTokens(s)
228228

229-
if try recognizer.getATN().nextTokens(s).contains(la) || la == CommonToken.EOF {
229+
if try recognizer.getATN().nextTokens(s).contains(CommonToken.EPSILON) {
230230
return
231231
}
232232

233-
// Return but don't end recovery. only do that upon valid token match
234-
if try recognizer.isExpectedToken(la) {
233+
if try recognizer.getATN().nextTokens(s).contains(la) {
235234
return
236235
}
237236

‎tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ switch (TokenStream.LA(1)) {
566566
<alt>
567567
break;}; separator="\n">
568568
default:
569-
<error>
569+
break;
570570
}
571571
>>
572572

‎tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ switch (_input->LA(1)) {
644644
\}
645645
}; separator="\n">
646646
default:
647-
<error>
647+
break;
648648
}
649649
>>
650650

‎tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg

-3
Original file line numberDiff line numberDiff line change
@@ -563,9 +563,6 @@ switch p.GetTokenStream().LA(1) {
563563

564564
<endif>
565565
default:
566-
<if(error)>
567-
<error>
568-
<endif>
569566
}
570567
>>
571568

‎tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,7 @@ switch (_input.LA(1)) {
489489
<alt>
490490
break;}; separator="\n">
491491
default:
492-
<error>
492+
break;
493493
}
494494
>>
495495

‎tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ switch (this._input.LA(1)) {
360360
<alt>
361361
break;}; separator="\n">
362362
default:
363-
<error>
363+
break;
364364
}
365365
>>
366366

‎tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ token = self._input.LA(1)
334334
<alt>
335335
pass}; separator="\nel">
336336
else:
337-
<error>
337+
pass
338338
>>
339339

340340
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<

‎tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ token = self._input.LA(1)
342342
<alt>
343343
pass}; separator="\nel">
344344
else:
345-
<error>
345+
pass
346346
>>
347347

348348
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<

‎tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg

+1-1
Original file line numberDiff line numberDiff line change
@@ -510,7 +510,7 @@ switch (<parser.name>.Tokens(rawValue: try _input.LA(1))!) {
510510
<alt>
511511
break}; separator="\n">
512512
default:
513-
<error>
513+
break
514514
}
515515
>>
516516

0 commit comments

Comments
 (0)