add token parsing for function calls and add function types

Author: ALescoulie

lib/Mentat/ParseTypes.hs

@@ -83,7 +83,7 @@ data Token
 data Function = Function String [String] Expr deriving (Show, Eq)
 
 -- | Used as an intermediate step in the parsing process to validate parens and sort them into sub trees
-data TokTree = TLeaf Token | TNode Bracket [TokTree] | TFxn Function deriving (Show, Eq)
+data TokTree = TLeaf Token | TNode Bracket [TokTree] | TFxn String [[TokTree]] deriving (Show, Eq)
 
 
 -- | Literal values of float or boolean
@@ -95,7 +95,7 @@ data Expr
   = LitE Literal
   | VarE String
   | BinOpE BinOp Expr Expr
-  | FxnE Function
+  | FxnE String [Expr]
   deriving (Show, Eq)
 
 

lib/Mentat/Tokenizer.hs

@@ -1,5 +1,6 @@
 module Mentat.Tokenizer where
 import Mentat.ParseTypes
+import Data.List.Split ( splitOn )
 
 tokSubTree :: TokTree -> [TokTree]
 tokSubTree (TNode _ ts) = ts
@@ -20,6 +21,15 @@ parseInner (TClose x : xs) (y : ys)
   | x /= y = Left $ MismatchedBracket (y : ys) (TClose x : xs)
   | otherwise = Right (xs, [])
 parseInner (TClose x : xs) [] = Left $ UnclosedBracket [] (TClose x : xs)
+parseInner (TId name : TOpen Paren : cRest) stack = do
+  let maybeFxn = parseFxnCall (TId name : TOpen Paren : cRest)
+  case maybeFxn of
+    Right (restAfterFxn, fxnCall) -> do
+      (restToks, rest) <- parseInner restAfterFxn stack
+      pure (restToks, fxnCall : rest)
+    Left _ -> do    
+      (restTokens, rest) <- parseInner (TOpen Paren : cRest) stack
+      pure (restTokens, TLeaf (TId name) : rest)
 parseInner (TOpen x : xs) stack = do
   -- don't need cases since errors are infective
   (tokAfterInner, inner) <- parseInner xs (x : stack) -- parse inner for bracket x
@@ -29,3 +39,11 @@ parseInner (x : xs) stack = do
   (restTokens, rest) <- parseInner xs stack
   pure (restTokens, TLeaf x : rest)
 
+parseFxnCall :: [Token] -> Either Error ([Token], TokTree)
+parseFxnCall (TId name : TOpen Paren : TClose Paren : rest) = Right (rest, TFxn name [[]])
+parseFxnCall (TId name : TOpen Paren :  call) = do
+  (rest, argToks) <- parseInner call [Paren]
+  let args = splitOn [TLeaf TSep] argToks
+  Right (rest, TFxn name args) 
+parseFxnCall _ = Left UnfinishedTokenStream -- add an error
+

test/TokenizerSpec.hs

@@ -6,7 +6,6 @@ import Mentat.ParseTypes
 import Mentat.Tokenizer
 import Prelude hiding (lex)
 import Test.Hspec
-import Data.Foldable (concat)
 
 concatTokenList :: [Token] -> String
 concatTokenList [] = ""
@@ -66,4 +65,19 @@ spec = do
         case tokTreeResult of
           Right tokTrees -> tokTrees `shouldBe` expected
           Left err -> error $ "unexpected error " ++ show err
+  describe "Parsing function calls into TokTrees" $ do
+    let cases = 
+          [ ("f()", [TFxn "f" [[]]])
+          , ("f(1, 2)", [TFxn "f" [[TLeaf $ TNumber 1], [TLeaf $ TNumber 2]]])
+          , ("f(n - 1)", [TFxn "f" [[TLeaf $ TId "n", TLeaf $ TOp Sub, TLeaf $ TNumber 1]]])
+          , ("f(f(n + 1))", [TFxn "f" [[TFxn "f" [[TLeaf $ TId "n", TLeaf $ TOp Add, TLeaf $ TNumber 1]]]]])
+          , ("f(2 * (n - 1))",[TFxn "f" [[ TLeaf $ TNumber 2, TLeaf $ TOp Mul, TNode Paren [TLeaf $ TId "n", TLeaf $ TOp Sub, TLeaf $ TNumber 1]]]])
+          ]
+
+    forM_ cases $ \(input, expected) ->
+      it ("Parses: " ++ show input) $ do
+        let tokTreeResult = parseTokTree $ lex input
+        case tokTreeResult of
+          Right tokTrees -> tokTrees `shouldBe` expected
+          Left err -> error $ "unexpected error " ++ show err