Fix lexer to emit correct location with range operator

Author: antonmedv

cmd/exe/main.go

@@ -19,13 +19,14 @@ import (
 )
 
 var (
-	bytecode bool
-	debug    bool
-	run      bool
-	past     bool
-	dot      bool
-	repl     bool
-	opt      bool
+	bytecode  bool
+	debug     bool
+	run       bool
+	past      bool
+	dot       bool
+	repl      bool
+	opt       bool
+	typeCheck bool
 )
 
 func init() {
@@ -36,6 +37,7 @@ func init() {
 	flag.BoolVar(&dot, "dot", false, "dot format")
 	flag.BoolVar(&repl, "repl", false, "start repl")
 	flag.BoolVar(&opt, "opt", true, "do optimization")
+	flag.BoolVar(&typeCheck, "type", true, "do a type check")
 }
 
 func main() {
@@ -85,13 +87,15 @@ func printAst() {
 	tree, err := parser.Parse(input())
 	check(err)
 
-	_, err = checker.Check(tree, &conf.Config{
-		AllowUndefinedVariables: true,
-	})
-	check(err)
+	if typeCheck {
+		_, err = checker.Check(tree, &conf.Config{
+			AllowUndefinedVariables: true,
+		})
+		check(err)
 
-	if opt {
-		optimizer.Optimize(&tree.Node)
+		if opt {
+			optimizer.Optimize(&tree.Node)
+		}
 	}
 
 	if !dot {
@@ -105,11 +109,13 @@ func printDisassemble() {
 	tree, err := parser.Parse(input())
 	check(err)
 
-	_, err = checker.Check(tree, nil)
-	check(err)
+	if typeCheck {
+		_, err = checker.Check(tree, nil)
+		check(err)
 
-	if opt {
-		optimizer.Optimize(&tree.Node)
+		if opt {
+			optimizer.Optimize(&tree.Node)
+		}
 	}
 
 	program, err := compiler.Compile(tree, nil)
@@ -122,11 +128,13 @@ func runProgram() {
 	tree, err := parser.Parse(input())
 	check(err)
 
-	_, err = checker.Check(tree, nil)
-	check(err)
+	if typeCheck {
+		_, err = checker.Check(tree, nil)
+		check(err)
 
-	if opt {
-		optimizer.Optimize(&tree.Node)
+		if opt {
+			optimizer.Optimize(&tree.Node)
+		}
 	}
 
 	program, err := compiler.Compile(tree, nil)

optimizer/optimizer_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/antonmedv/expr/internal/conf"
 	"github.com/antonmedv/expr/optimizer"
 	"github.com/antonmedv/expr/parser"
-	"github.com/sanity-io/litter"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,7 +23,7 @@ func TestOptimize_constant_folding(t *testing.T) {
 		Index: &ast.IntegerNode{Value: 0},
 	}
 
-	assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node))
+	assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node))
 }
 
 func TestOptimize_in_array(t *testing.T) {
@@ -44,7 +43,7 @@ func TestOptimize_in_array(t *testing.T) {
 		Right:    &ast.ConstantNode{Value: optimizer.Map{1: {}, 2: {}, 3: {}}},
 	}
 
-	assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node))
+	assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node))
 }
 
 func TestOptimize_in_range(t *testing.T) {
@@ -74,7 +73,7 @@ func TestOptimize_in_range(t *testing.T) {
 		},
 	}
 
-	assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node))
+	assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node))
 }
 
 func TestOptimize_const_range(t *testing.T) {
@@ -87,5 +86,5 @@ func TestOptimize_const_range(t *testing.T) {
 		Value: []int{-1, 0, 1},
 	}
 
-	assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node))
+	assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node))
 }

parser/lexer/lexer_test.go

@@ -3,6 +3,7 @@ package lexer_test
 import (
 	"fmt"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"strings"
 	"testing"
 
@@ -117,6 +118,21 @@ func TestLex(t *testing.T) {
 	}
 }
 
+func TestLex_location(t *testing.T) {
+	source := file.NewSource("1..2 3..4")
+	tokens, err := Lex(source)
+	require.NoError(t, err)
+	require.Equal(t, []Token{
+		{Location: file.Location{Line: 1, Column: 0}, Kind: Number, Value: "1"},
+		{Location: file.Location{Line: 1, Column: 1}, Kind: Operator, Value: ".."},
+		{Location: file.Location{Line: 1, Column: 3}, Kind: Number, Value: "2"},
+		{Location: file.Location{Line: 1, Column: 5}, Kind: Number, Value: "3"},
+		{Location: file.Location{Line: 1, Column: 6}, Kind: Operator, Value: ".."},
+		{Location: file.Location{Line: 1, Column: 8}, Kind: Number, Value: "4"},
+		{Location: file.Location{Line: 1, Column: 8}, Kind: EOF, Value: ""},
+	}, tokens)
+}
+
 const errorTests = `
 "\xQA"
 invalid char escape (1:5)

parser/lexer/state.go

@@ -67,10 +67,18 @@ func (l *lexer) scanNumber() bool {
 		}
 	}
 	l.acceptRun(digits)
+	end := l.end
+	loc := l.loc
+	prev := l.prev
 	if l.accept(".") {
 		// Lookup for .. operator: if after dot there is another dot (1..2), it maybe a range operator.
 		if l.peek() == '.' {
-			l.backup()
+			// We can't backup() here, as it would require two backups,
+			// and backup() func supports only one for now. So, save and
+			// restore it here.
+			l.end = end
+			l.loc = loc
+			l.prev = prev
 			return true
 		}
 		l.acceptRun(digits)