Merge pull request #95

packages/parser/README.md

@@ -15,19 +15,20 @@ import parse from "the/parser"
 
 ## API
 
-### `parse (path: str) File`
+### `parse (pathOrCode: str) File`
 Parses path into `File` object.
 
 **Parameters**
 
-- `path` - path to parse
+- `pathOrCode` - path or code to parse
 
 **Return value**
 
 Parsed path as `File` object.
 
 **Exceptions**
 
+- `ParserError` - thrown if path doesn't exists
 - `ParserError` - thrown if path is not a file
 
 **Examples**

packages/parser/package.yml

@@ -1,5 +1,12 @@
 name: the/parser
-version: 0.3.3
+version: 0.4.1
 description: The programming language parser
 license: MIT
 main: src/main
+
+packages:
+  the/testing: 0.3.0
+
+scripts:
+  update-parser-tests: the run scripts/update-parser-tests
+  update-tokenizer-tests: the run scripts/update-tokenizer-tests

packages/parser/scripts/update-parser-tests

@@ -0,0 +1,36 @@
+/*!
+ * Copyright (c) 2018 Aaron Delasy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import parse, textify from "../src/parser"
+import parseTestFile from "../test/utils"
+
+main {
+  files := fs_scandirSync("./test/parser")
+  filesLen := files.len
+
+  loop i := 0; i < filesLen; i++ {
+    file := files[i]
+    if file.slice(-4) != ".txt" { continue }
+    print(file as str)
+
+    result := parseTestFile("./test/parser/" + file)
+    mut f := parse(result.in)
+    stdoutCode := textify(ref f)
+
+    fileResult := (result.in.empty ? "" : result.in + os_EOL) + result.delimiter + stdoutCode
+    fs_writeFileSync("./test/parser/" + file, fileResult.toBuffer())
+  }
+}

packages/parser/scripts/update-tokenizer-tests

@@ -0,0 +1,37 @@
+/*!
+ * Copyright (c) 2018 Aaron Delasy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as Reader from "../src/reader"
+import * as Tokenizer from "../src/tokenizer"
+import parseTestFile from "../test/utils"
+
+main {
+  files := fs_scandirSync("./test/tokenizer")
+  filesLen := files.len
+
+  loop i := 0; i < filesLen; i++ {
+    file := files[i]
+    if file.slice(-4) != ".txt" { continue }
+
+    result := parseTestFile("./test/tokenizer/" + file)
+    mut reader := Reader.init(result.in)
+    mut tokenizer := Tokenizer.init(ref reader)
+    stdoutCode := Tokenizer.stringify(ref tokenizer, allowComments: file.contains("comment"))
+
+    fileResult := result.in + os_EOL + result.delimiter + stdoutCode
+    fs_writeFileSync("./test/tokenizer/" + file, fileResult.toBuffer())
+  }
+}

packages/parser/src/errors

@@ -205,7 +205,7 @@ export fn E0136 () str {
 }
 
 export fn E0137 () str {
-  return "E0137 - Expected union sub type after pipe operator"
+  return "E0137 - Expected union subtype after pipe operator"
 }
 
 export fn E0138 (after: str) str {
@@ -220,9 +220,9 @@ export fn E0140 () str {
   return "E0140 - Expected function type parameter type"
 }
 
-export fn E0141 () str {
-  return "E0141 - Expected function type parameter type after parameter name"
-}
+// export fn E0141 () str {
+//   return "E0141 - Expected function type parameter type after parameter name"
+// }
 
 export fn E0142 () str {
   return "E0142 - Expected function type parameter type after colon"
@@ -256,9 +256,9 @@ export fn E0149 (kw: str) str {
   return "E0149 - Expected identifier after `" + kw + "` keyword"
 }
 
-export fn E0150 () str {
-  return "E0150 - Expected closure expression body"
-}
+// export fn E0150 () str {
+//   return "E0150 - Expected closure expression body"
+// }
 
 export fn E0151 () str {
   return "E0151 - Expected if statement consequent statement"
@@ -268,9 +268,9 @@ export fn E0152 () str {
   return "E0152 - Expected if statement alternate statement"
 }
 
-export fn E0153 () str {
-  return "E0153 - Expected main declaration body"
-}
+// export fn E0153 () str {
+//   return "E0153 - Expected main declaration body"
+// }
 
 export fn E0154 () str {
   return "E0154 - Expected equals sign after type alias declaration name"

packages/parser/src/expression

@@ -30,9 +30,6 @@ export fn endsWithObjExpr (self: t.Expression) bool {
   } elif self.isConditional() {
     selfBody := self.asConditional()
     return endsWithObjExpr(selfBody.alternate)
-  } elif self.isObject() {
-    selfBody := self.asObject()
-    return true
   } elif self.isReference() {
     selfBody := self.asReference()
     return endsWithObjExpr(selfBody.expression)
@@ -43,7 +40,7 @@ export fn endsWithObjExpr (self: t.Expression) bool {
     }
   }
 
-  return false
+  return self.isObject()
 }
 
 export fn extractLastObjExpr (self: t.Expression) t.ObjectExpression {
@@ -78,46 +75,58 @@ export fn extractLastObjExpr (self: t.Expression) t.ObjectExpression {
 export fn subtractLastObjExpr (self: t.Expression) t.Expression {
   if self.isAssignment() {
     selfBody := self.asAssignment()
+    right := subtractLastObjExpr(selfBody.right)
+
     return create(t.AssignmentExpression{
       left: selfBody.left,
       operator: selfBody.operator,
-      right: subtractLastObjExpr(selfBody.right)
-    }, self.start, self.end)
+      right: right
+    }, self.start, right.end)
   } elif self.isAwait() {
     selfBody := self.asAwait()
+    expression := subtractLastObjExpr(selfBody.expression)
+
     return create(t.AwaitExpression{
-      expression: subtractLastObjExpr(selfBody.expression)
-    }, self.start, self.end)
+      expression: expression
+    }, self.start, expression.end)
   } elif self.isBinary() {
     selfBody := self.asBinary()
+    right := subtractLastObjExpr(selfBody.right)
+
     return create(t.BinaryExpression{
       left: selfBody.left,
       operator: selfBody.operator,
-      right: subtractLastObjExpr(selfBody.right)
-    }, self.start, self.end)
+      right: right
+    }, self.start, right.end)
   } elif self.isConditional() {
     selfBody := self.asConditional()
+    alternate := subtractLastObjExpr(selfBody.alternate)
+
     return create(t.ConditionalExpression{
       condition: selfBody.condition,
       consequent: selfBody.consequent,
-      alternate: subtractLastObjExpr(selfBody.alternate)
-    }, self.start, self.end)
+      alternate: alternate
+    }, self.start, alternate.end)
   } elif self.isObject() {
     selfBody := self.asObject()
     return typeToExpression(selfBody.id)
   } elif self.isReference() {
     selfBody := self.asReference()
+    expression := subtractLastObjExpr(selfBody.expression)
+
     return create(t.ReferenceExpression{
-      expression: subtractLastObjExpr(selfBody.expression)
-    }, self.start, self.end)
+      expression: expression
+    }, self.start, expression.end)
   } elif self.isUnary() {
     selfBody := self.asUnary()
     if selfBody.prefix {
+      operand := subtractLastObjExpr(selfBody.operand)
+
       return create(t.UnaryExpression{
         operator: selfBody.operator,
-        operand: subtractLastObjExpr(selfBody.operand),
+        operand: operand,
         prefix: selfBody.prefix
-      }, self.start, self.end)
+      }, self.start, operand.end)
     }
   }
 
@@ -131,15 +140,19 @@ export fn toType (self: t.Expression) t.Type {
     return t.Type{
       body: t.IdentifierType{name: selfBody},
       start: self.start,
-      end: self.end
+      end: self.end,
+      leadingComments: self.leadingComments,
+      trailingComments: self.trailingComments
     }
   } elif self.isPropertyAccess() {
     selfBody := self.asPropertyAccess()
 
     return t.Type{
       body: t.MemberType{t: toType(selfBody.expression), name: selfBody.name},
       start: self.start,
-      end: self.end
+      end: self.end,
+      leadingComments: self.leadingComments,
+      trailingComments: self.trailingComments
     }
   }
 
@@ -153,15 +166,19 @@ export fn typeToExpression (self: t.Type) t.Expression {
     return t.Expression{
       body: selfBody.name,
       start: self.start,
-      end: self.end
+      end: self.end,
+      leadingComments: self.leadingComments,
+      trailingComments: self.trailingComments
     }
   } elif self.isMember() {
     selfBody := self.asMember()
 
     return t.Expression{
       body: t.PropertyAccessExpression{expression: typeToExpression(selfBody.t), name: selfBody.name},
       start: self.start,
-      end: self.end
+      end: self.end,
+      leadingComments: self.leadingComments,
+      trailingComments: self.trailingComments
     }
   }
 

packages/parser/src/main

@@ -73,7 +73,7 @@ import
   Type_toText
 from "./text"
 import Program_traverse from "./program"
-import parse, report, stringify from "./parser"
+import parse, report from "./parser"
 
 export ArrayType
 export FunctionType