Update tree-sitter, add error printing for tree sitter

Cargo.toml

@@ -14,7 +14,7 @@ ariadne = {path = "ariadne"} # for nice errors
 num = "*"
 
 # Tree sitter
-tree-sitter = "0.20.10"
+tree-sitter = "~0.22.1"
 tree-sitter-sus = {path = "tree-sitter-sus"}
 static_init = "1.0.3"
 
@@ -37,3 +37,6 @@ serde = {version = "1.0.156", optional = true}
 lsp = ["lsp-server", "lsp-types", "serde_json", "serde"]
 # codegen = ["calyx-ir", "calyx-opt", "calyx-backend"]
 # codegen = ["moore-circt-sys", "moore-circt"]
+
+[profile.release]
+debug = true

resetNormalizer.sus

@@ -1,9 +1,13 @@
-
+a {}
 /*module hello_from_the_other_side : int[3] a -> int result {
 	result = a[2];
 	int x = a[0] + a[1] + a[2];
 }*/
 
 module a {
-	+ 3;
+	a = ;
+	a = ;
+	a = ;
+
+	{
 }

src/flattening/mod.rs

@@ -1,7 +1,7 @@
 
 pub mod name_context;
 
-use std::{ops::Deref, iter::zip};
+use std::{iter::zip, num::NonZeroU16, ops::Deref};
 
 use tree_sitter::{Node, TreeCursor};
 
@@ -695,12 +695,34 @@ impl<'l> FlatteningContext<'l> {
 
     }
 
+    /// If field is found, cursor is now at field position
+    /// 
+    /// If field is not found, cursor remains in place
+    fn goto_field(&mut self, field_id : NonZeroU16) -> bool {
+        let mut shift_count = 0;
+
+        loop {
+            if self.cursor.field_id() == Some(field_id) {
+                return true;
+            }
+            if !self.cursor.goto_next_sibling() {
+                break;
+            }
+            shift_count += 1;
+        }
+        // Recover from error. Shift back until at starting node
+        for _ in 0..shift_count {
+            self.cursor.goto_previous_sibling();
+        }
+        false
+    }
+
     fn flatten_module_tree(&mut self) {
         let module_node = self.cursor.node();
         assert_eq!(module_node.kind_id(), SUS.module_kind, "{}", module_node.kind());
         println!("TREE SITTER module!");
-        let interface = module_node.child_by_field_id(SUS.interface_ports_field);
-        let code = module_node.child_by_field_id(SUS.block_field).unwrap();
+        //let interface = module_node.child_by_field_id(SUS.interface_ports_field);
+        //let code = module_node.child_by_field_id(SUS.block_field).unwrap();
 
         self.flatten_interface_ports_tree();
         //self.flatten_code_tree(code);

src/linker.rs

@@ -1,6 +1,6 @@
 use std::{collections::{HashMap, HashSet}, rc::Rc, cell::RefCell};
 
-use crate::{arena_alloc::{ArenaAllocator, UUIDMarker, UUID}, ast::{LinkInfo, Module}, errors::{error_info, ErrorCollector}, file_position::{FileText, Span}, flattening::{FlatID, FlattenedModule, Instruction, WireInstance, WireSource}, instantiation::InstantiatedModule, parser::{FullParseResult, TokenTreeNode}, tokenizer::TokenTypeIdx, typing::{Type, WrittenType}, util::{const_str_position, const_str_position_in_tuples}, value::Value};
+use crate::{arena_alloc::{ArenaAllocator, UUIDMarker, UUID}, ast::{LinkInfo, Module}, errors::{error_info, ErrorCollector}, file_position::{FileText, Span}, flattening::{FlatID, FlattenedModule, Instruction, WireInstance, WireSource}, instantiation::InstantiatedModule, parser::{FullParseResult, TokenTreeNode, SUS}, tokenizer::TokenTypeIdx, typing::{Type, WrittenType}, util::{const_str_position, const_str_position_in_tuples}, value::Value};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct ModuleUUIDMarker;
@@ -332,15 +332,13 @@ impl Linker {
     pub fn add_reserved_file(&mut self, file : FileUUID, parse_result : FullParseResult) {
         let mut associated_values = Vec::new();
 
-        let sus = crate::parser::SusTreeSitterSingleton::new();
-
         {
             let root_node = parse_result.tree.root_node();
 
             let mut tmp_cursor = root_node.walk();
             for node in root_node.children(&mut tmp_cursor) {
-                if node.kind_id() == sus.module_kind {
-                    let name_child = node.child_by_field_id(sus.name_field).unwrap();
+                if node.kind_id() == SUS.module_kind {
+                    let name_child = node.child_by_field_id(SUS.name_field.into()).unwrap();
                     println!("MODULE DECL: {}", &parse_result.file_text.file_text[name_child.byte_range()])
                 } else {
                     parse_result.ast.errors.error_basic(Span::from(node.byte_range()), "Only module declarations are allowed at the top level of a file!");

src/main.rs

@@ -36,6 +36,8 @@ use codegen_fallback::gen_verilog_code;
 use dev_aid::syntax_highlighting::*;
 use linker::{Linker, ModuleUUID};
 
+use crate::parser::SUS;
+
 fn codegen_to_file(linker : &Linker, id : ModuleUUID, md : &Module) -> Option<()> {
     let Some(inst) = linker.instantiate(id) else {
         println!("Module {} instantiation encountered errors.", md.link_info.name);
@@ -54,30 +56,37 @@ fn codegen_to_file(linker : &Linker, id : ModuleUUID, md : &Module) -> Option<()
     Some(())
 }
 
-fn test_tree_sitter(path : &Path) {
+fn test_tree_sitter(path : &Path, make_dot : bool) {
     let code = read_to_string(path).unwrap();
     let mut parser = tree_sitter::Parser::new();
-    parser.set_language(tree_sitter_sus::language()).expect("Error loading sus grammar");
+    parser.set_language(&SUS.language).expect("Error loading sus grammar");
     let tree = parser.parse(code, None).unwrap();
 
-    let mut dot_cmd = std::process::Command::new("dot");
-    dot_cmd.arg("-Tsvg");
-    dot_cmd.arg("-Gcharset=latin1");
-    dot_cmd.stdin(Stdio::piped());
-    dot_cmd.stdout(Stdio::piped());
-    let dot_proc = dot_cmd.spawn().unwrap();
-    tree.print_dot_graph(dot_proc.stdin.as_ref().unwrap());
-    let out = dot_proc.wait_with_output().unwrap();
-    let mut out_file = File::create(format!("{}.svg", path.file_stem().unwrap().to_str().unwrap())).unwrap();
-    out_file.write(&out.stdout).unwrap();
+    if make_dot {
+        let mut dot_cmd = std::process::Command::new("dot");
+        dot_cmd.arg("-Tsvg");
+        dot_cmd.arg("-Gcharset=latin1");
+        dot_cmd.stdin(Stdio::piped());
+        dot_cmd.stdout(Stdio::piped());
+        let dot_proc = dot_cmd.spawn().unwrap();
+        tree.print_dot_graph(dot_proc.stdin.as_ref().unwrap());
+        let out = dot_proc.wait_with_output().unwrap();
+        let mut out_file = File::create(format!("{}.svg", path.file_stem().unwrap().to_str().unwrap())).unwrap();
+        out_file.write(&out.stdout).unwrap();
+    }
 
-    /*
     let root = tree.root_node();
     let mut cursor = root.walk();
-    for c in root.children(&mut cursor) {
+    cursor.goto_first_child();
+    /*for c in root.children(&mut cursor) {
         println!("{c:?}");
-    }
-    println!("{root:?}");*/
+    }*/
+    //cursor.reset(cursor.node());
+    println!("First goto child {}", cursor.goto_first_child());
+    println!("First goto parent {}", cursor.goto_parent());
+    println!("Second goto parent {}", cursor.goto_parent());
+    println!("Third goto parent {}", cursor.goto_parent());
+    println!("{root:?}");
 }
 
 
@@ -91,6 +100,7 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
     let mut codegen = None;
     let mut codegen_all = false;
     let mut test_sus_sitter = false;
+    let mut make_dot = false;
     let mut settings = SyntaxHighlightSettings{
         show_tokens : false
     };
@@ -115,6 +125,9 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
             "--tree" => {
                 test_sus_sitter = true;
             }
+            "--dot" => {
+                make_dot = true;
+            }
             other => {
                 file_paths.push(PathBuf::from(other));
             }
@@ -135,7 +148,7 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
 
     if test_sus_sitter {
         for path in &file_paths {
-            test_tree_sitter(&path);
+            test_tree_sitter(&path, make_dot);
         }
         return Ok(())
     }

src/parser.rs

@@ -1,10 +1,11 @@
 
 use num::BigInt;
 use static_init::dynamic;
+use tree_sitter::Tree;
 
 use crate::{ast::*, errors::*, file_position::{BracketSpan, FileText, SingleCharSpan, Span}, flattening::FlattenedModule, instantiation::InstantiationList, linker::FileUUID, tokenizer::*, value::Value};
 
-use std::{iter::Peekable, str::FromStr};
+use std::{iter::Peekable, num::NonZeroU16, str::FromStr};
 use core::slice::Iter;
 
 pub enum TokenTreeNode {
@@ -732,17 +733,64 @@ pub fn perform_full_semantic_parse(file_text : String, file : FileUUID) -> FullP
     let ast = parse(&token_hierarchy, &file_text, errors);
 
     let mut parser = tree_sitter::Parser::new();
-    parser.set_language(SUS.language).unwrap();
+    parser.set_language(&SUS.language).unwrap();
+
+    let tree = parser.parse(&file_text.file_text, None).unwrap();
+
+    report_all_tree_errors(&tree, &ast.errors);
 
     FullParseResult{
-        tree : parser.parse(&file_text.file_text, None).unwrap(),
+        tree,
         file_text,
         tokens,
         token_hierarchy,
         ast,
     }
 }
 
+
+
+
+fn report_all_tree_errors(tree : &Tree, errors : &ErrorCollector) {
+    let mut cursor = tree.walk();
+    loop {
+        let depth_str = "  ".repeat(cursor.depth() as usize);
+        let cursor_node = cursor.node().kind();
+        let cursor_span = Span::from(cursor.node().byte_range());
+        println!("{depth_str}{cursor_node}: {cursor_span}");
+        let n = cursor.node();
+        if n.is_error() || n.is_missing() {
+            let node_name = n.kind();
+            let span = Span::from(n.byte_range());
+            let field_name = cursor.field_name();
+
+            let of_name = if let Some(field) = field_name {
+                format!("in the field '{field}' of type '{node_name}'")
+            } else {
+                format!("in a node of type '{node_name}'")
+            };
+            let (error_type, parent_node) = if n.is_missing() {
+                ("missing field", n.parent().unwrap().parent().unwrap()) // Weird workaround because MISSING nodes can't properly parent?
+            } else {
+                ("syntax error", n.parent().unwrap())
+            };
+            let parent_node_name = parent_node.kind();
+            let parent_info = error_info(Span::from(parent_node.byte_range()), errors.file, format!("Parent node '{parent_node_name}'"));
+            errors.error_with_info(span, format!("While parsing '{parent_node_name}', parser found a {error_type} {of_name}"), vec![parent_info]);
+        } else {
+            if cursor.goto_first_child() {
+                continue;
+            }
+        }
+        while !cursor.goto_next_sibling() {
+            if !cursor.goto_parent() {
+                return;
+            }
+        }
+    }
+}
+
+
 pub struct SusTreeSitterSingleton {
     pub language : tree_sitter::Language,
 
@@ -766,35 +814,35 @@ pub struct SusTreeSitterSingleton {
     pub if_statement_kind : u16,
     pub for_statement_kind : u16,
 
-    pub name_field : u16,
-    pub module_inputs_field : u16,
-    pub module_outputs_field : u16,
-    pub block_field : u16,
-    pub interface_ports_field : u16,
-    pub array_element_type_field : u16,
-    pub array_size_field : u16,
-    pub type_field : u16,
-    pub latency_spec_field : u16,
-    pub declaration_modifiers_field : u16,
-    pub left_field : u16,
-    pub right_field : u16,
-    pub operator_field : u16,
-    pub arr_field : u16,
-    pub arr_idx_field : u16,
-    pub func_arguments_field : u16,
-    pub from_field : u16,
-    pub to_field : u16,
-    pub assign_to_field : u16,
-    pub assign_value_field : u16,
-    pub condition_field : u16,
-    pub then_block_field : u16,
-    pub else_block_field : u16,
-    pub for_decl_field : u16,
-    pub for_range_field : u16
+    pub name_field : NonZeroU16,
+    pub module_inputs_field : NonZeroU16,
+    pub module_outputs_field : NonZeroU16,
+    pub block_field : NonZeroU16,
+    pub interface_ports_field : NonZeroU16,
+    pub array_element_type_field : NonZeroU16,
+    pub array_size_field : NonZeroU16,
+    pub type_field : NonZeroU16,
+    pub latency_spec_field : NonZeroU16,
+    pub declaration_modifiers_field : NonZeroU16,
+    pub left_field : NonZeroU16,
+    pub right_field : NonZeroU16,
+    pub operator_field : NonZeroU16,
+    pub arr_field : NonZeroU16,
+    pub arr_idx_field : NonZeroU16,
+    pub func_arguments_field : NonZeroU16,
+    pub from_field : NonZeroU16,
+    pub to_field : NonZeroU16,
+    pub assign_to_field : NonZeroU16,
+    pub assign_value_field : NonZeroU16,
+    pub condition_field : NonZeroU16,
+    pub then_block_field : NonZeroU16,
+    pub else_block_field : NonZeroU16,
+    pub for_decl_field : NonZeroU16,
+    pub for_range_field : NonZeroU16
 }
 
 impl SusTreeSitterSingleton {
-    pub fn new() -> Self {
+    fn new() -> Self {
         let language = tree_sitter_sus::language();
         SusTreeSitterSingleton {
             error_kind : language.id_for_node_kind("ERROR", false),