Skip to content

Commit

Permalink
Add proc_macro to access tree-sitter-sus kinds/fields at compile-time
Browse files Browse the repository at this point in the history
  • Loading branch information
@VonTum
VonTum committed Apr 10, 2024
1 parent 3a3a5c9 commit 330a4d0
Show file tree
Hide file tree
Showing 8 changed files with 209 additions and 315 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/target
**/target
*.lock
.vscode
*.svg
Expand Down
3 changes: 2 additions & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ repository = "https://github.com/pc2/sus-compiler"
ariadne = {path = "ariadne"} # for nice errors
num = "*"

sus_proc_macro = {path = "sus_proc_macro"}

# Tree sitter
tree-sitter = "~0.22.2"
tree-sitter-sus = {path = "tree-sitter-sus"}
static_init = "1.0.3"

# calyx-ir = {version = "0.6.1", optional = true}
# calyx-opt = {version = "0.6.1", optional = true}
Expand Down
183 changes: 92 additions & 91 deletions src/flattening/mod.rs
View file

Large diffs are not rendered by default.

10 changes: 6 additions & 4 deletions src/linker.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
use std::{collections::{HashMap, HashSet}, rc::Rc, cell::RefCell};

use sus_proc_macro::{field, kind};

use crate::{
arena_alloc::{ArenaAllocator, UUIDMarker, UUID},
errors::{error_info, ErrorCollector},
file_position::{FileText, Span},
flattening::{FlatID, FlattenedModule, Instruction, Module, WireInstance, WireSource},
instantiation::{InstantiatedModule, InstantiationList},
parser::{Cursor, Documentation, FullParseResult, SUS},
parser::{Cursor, Documentation, FullParseResult},
typing::{Type, WrittenType},
util::{const_str_position, const_str_position_in_tuples},
value::Value
Expand Down Expand Up @@ -361,10 +363,10 @@ impl Linker {

{
let mut walker = Cursor::new_at_root(&parse_result.tree, &parse_result.file_text);
walker.list(SUS.source_file_kind, |cursor| {
walker.list(kind!("source_file"), |cursor| {
let (kind, span) = cursor.kind_span();
assert!(kind == SUS.module_kind);
let name_span = cursor.go_down_no_check(|cursor| cursor.field_span(SUS.name_field, SUS.identifier_kind));
assert!(kind == kind!("module"));
let name_span = cursor.go_down_no_check(|cursor| cursor.field_span(field!("name"), kind!("identifier")));
let md = Module{
link_info: LinkInfo {
documentation: cursor.extract_gathered_comments(),
Expand Down
58 changes: 3 additions & 55 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,16 @@ mod typing;
mod dev_aid;
mod linker;

use std::process::Stdio;
use std::io::Write;
use std::{env, ops::Deref};
use std::error::Error;
use std::fs::{read_to_string, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::fs::File;
use std::path::PathBuf;
use flattening::Module;
use codegen_fallback::gen_verilog_code;
use dev_aid::syntax_highlighting::*;
use linker::{Linker, ModuleUUID};

use crate::parser::SUS;

fn codegen_to_file(linker : &Linker, id : ModuleUUID, md : &Module) -> Option<()> {
let Some(inst) = linker.instantiate(id) else {
println!("Module {} instantiation encountered errors.", md.link_info.name);
Expand All @@ -55,40 +52,6 @@ fn codegen_to_file(linker : &Linker, id : ModuleUUID, md : &Module) -> Option<()
Some(())
}

fn test_tree_sitter(path : &Path, make_dot : bool) {
let code = read_to_string(path).unwrap();
let mut parser = tree_sitter::Parser::new();
parser.set_language(&SUS.language).expect("Error loading sus grammar");
let tree = parser.parse(code, None).unwrap();

if make_dot {
let mut dot_cmd = std::process::Command::new("dot");
dot_cmd.arg("-Tsvg");
dot_cmd.arg("-Gcharset=latin1");
dot_cmd.stdin(Stdio::piped());
dot_cmd.stdout(Stdio::piped());
let dot_proc = dot_cmd.spawn().unwrap();
tree.print_dot_graph(dot_proc.stdin.as_ref().unwrap());
let out = dot_proc.wait_with_output().unwrap();
let mut out_file = File::create(format!("{}.svg", path.file_stem().unwrap().to_str().unwrap())).unwrap();
out_file.write(&out.stdout).unwrap();
}

let root = tree.root_node();
let mut cursor = root.walk();
cursor.goto_first_child();
/*for c in root.children(&mut cursor) {
println!("{c:?}");
}*/
//cursor.reset(cursor.node());
println!("First goto child {}", cursor.goto_first_child());
println!("First goto parent {}", cursor.goto_parent());
println!("Second goto parent {}", cursor.goto_parent());
println!("Third goto parent {}", cursor.goto_parent());
println!("{root:?}");
}


fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
let mut args = env::args();

Expand All @@ -99,8 +62,6 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
let mut lsp_port = 25000;
let mut codegen = None;
let mut codegen_all = false;
let mut test_sus_sitter = false;
let mut make_dot = false;
let mut settings = SyntaxHighlightSettings{
show_tokens : false
};
Expand Down Expand Up @@ -131,12 +92,6 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
"--tokens" => {
settings.show_tokens = true;
}
"--tree" => {
test_sus_sitter = true;
}
"--dot" => {
make_dot = true;
}
other => {
panic!("Unknown option {other}");
}
Expand All @@ -163,13 +118,6 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
//codegen = Some("first_bit_idx_6".to_owned());
}

if test_sus_sitter {
for path in &file_paths {
test_tree_sitter(&path, make_dot);
}
return Ok(())
}

let (linker, mut paths_arena) = compile_all(file_paths);
print_all_errors(&linker, &mut paths_arena);

Expand Down
174 changes: 11 additions & 163 deletions src/parser.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

use static_init::dynamic;
use sus_proc_macro::{field, kind};
use tree_sitter::{Tree, TreeCursor};

use crate::{errors::*, file_position::{FileText, Span}, linker::FileUUID};
Expand All @@ -17,7 +17,7 @@ pub fn perform_full_semantic_parse(file_text : String, file : FileUUID) -> FullP
let file_text = FileText::new(file_text);

let mut parser = tree_sitter::Parser::new();
parser.set_language(&SUS.language).unwrap();
parser.set_language(&tree_sitter_sus::language()).unwrap();

let tree = parser.parse(&file_text.file_text, None).unwrap();

Expand All @@ -37,7 +37,7 @@ fn print_current_node_indented(file_text : &FileText, cursor : &TreeCursor) -> S
let indent = " ".repeat(cursor.depth() as usize);
let n = cursor.node();
let cursor_span = Span::from(n.byte_range());
let node_name = if n.kind_id() == SUS.identifier_kind {format!("\"{}\"", &file_text[cursor_span])} else {n.kind().to_owned()};
let node_name = if n.kind_id() == kind!("identifier") {format!("\"{}\"", &file_text[cursor_span])} else {n.kind().to_owned()};
if let Some(field_name) = cursor.field_name() {
println!("{indent} {field_name}: {node_name} [{cursor_span}]");
} else {
Expand Down Expand Up @@ -80,158 +80,6 @@ fn report_all_tree_errors(file_text : &FileText, tree : &Tree, errors : &ErrorCo
}
}


pub struct SusTreeSitterSingleton {
pub language : tree_sitter::Language,

pub source_file_kind : u16,
pub module_kind : u16,
pub interface_ports_kind : u16,
pub identifier_kind : u16,
pub number_kind : u16,
pub global_identifier_kind : u16,
pub array_type_kind : u16,
pub declaration_kind : u16,
pub declaration_list_kind : u16,
pub latency_specifier_kind : u16,
pub unary_op_kind : u16,
pub binary_op_kind : u16,
pub array_op_kind : u16,
pub func_call_kind : u16,
pub parenthesis_expression_kind : u16,
pub parenthesis_expression_list_kind : u16,
pub array_bracket_expression_kind : u16,
pub block_kind : u16,
pub decl_assign_statement_kind : u16,
pub assign_left_side_kind : u16,
pub assign_to_kind : u16,
pub write_modifiers_kind : u16,
pub if_statement_kind : u16,
pub for_statement_kind : u16,

pub single_line_comment_kind : u16,
pub multi_line_comment_kind : u16,

pub gen_kw : u16,
pub state_kw : u16,
pub reg_kw : u16,
pub initial_kw : u16,

pub name_field : NonZeroU16,
pub inputs_field : NonZeroU16,
pub outputs_field : NonZeroU16,
pub block_field : NonZeroU16,
pub interface_ports_field : NonZeroU16,
pub type_field : NonZeroU16,
pub latency_specifier_field : NonZeroU16,
pub declaration_modifiers_field : NonZeroU16,
pub left_field : NonZeroU16,
pub right_field : NonZeroU16,
pub operator_field : NonZeroU16,
pub arr_field : NonZeroU16,
pub arr_idx_field : NonZeroU16,
pub arguments_field : NonZeroU16,
pub from_field : NonZeroU16,
pub write_modifiers_field : NonZeroU16,
pub to_field : NonZeroU16,
pub expr_or_decl_field : NonZeroU16,
pub assign_left_field : NonZeroU16,
pub assign_value_field : NonZeroU16,
pub condition_field : NonZeroU16,
pub then_block_field : NonZeroU16,
pub else_block_field : NonZeroU16,
pub for_decl_field : NonZeroU16,

pub content_field : NonZeroU16,
pub item_field : NonZeroU16
}

impl SusTreeSitterSingleton {
fn new() -> Self {
let language = tree_sitter_sus::language();
let node_kind = |name : &str| -> u16 {
let v = language.id_for_node_kind(name, true);
assert!(v != 0, "{name}");
v
};
let keyword_kind = |name : &str| -> u16 {
let v = language.id_for_node_kind(name, false);
assert!(v != 0, "{name}");
v
};
let field = |name : &str| -> NonZeroU16 {
language.field_id_for_name(name).expect(name)
};
SusTreeSitterSingleton {
source_file_kind : node_kind("source_file"),
module_kind : node_kind("module"),
interface_ports_kind : node_kind("interface_ports"),
identifier_kind : node_kind("identifier"),
number_kind : node_kind("number"),
global_identifier_kind : node_kind("global_identifier"),
array_type_kind : node_kind("array_type"),
declaration_kind : node_kind("declaration"),
declaration_list_kind : node_kind("declaration_list"),
latency_specifier_kind : node_kind("latency_specifier"),
unary_op_kind : node_kind("unary_op"),
binary_op_kind : node_kind("binary_op"),
array_op_kind : node_kind("array_op"),
func_call_kind : node_kind("func_call"),
parenthesis_expression_kind : node_kind("parenthesis_expression"),
parenthesis_expression_list_kind : node_kind("parenthesis_expression_list"),
array_bracket_expression_kind : node_kind("array_bracket_expression"),
block_kind : node_kind("block"),
decl_assign_statement_kind : node_kind("decl_assign_statement"),
assign_left_side_kind : node_kind("assign_left_side"),
assign_to_kind : node_kind("assign_to"),
write_modifiers_kind : node_kind("write_modifiers"),
if_statement_kind : node_kind("if_statement"),
for_statement_kind : node_kind("for_statement"),

single_line_comment_kind : node_kind("single_line_comment"),
multi_line_comment_kind : node_kind("multi_line_comment"),

gen_kw : keyword_kind("gen"),
state_kw : keyword_kind("state"),
reg_kw : keyword_kind("reg"),
initial_kw : keyword_kind("initial"),

name_field : field("name"),
inputs_field : field("inputs"),
outputs_field : field("outputs"),
block_field : field("block"),
interface_ports_field : field("interface_ports"),
type_field : field("type"),
latency_specifier_field : field("latency_specifier"),
declaration_modifiers_field : field("declaration_modifiers"),
left_field : field("left"),
right_field : field("right"),
operator_field : field("operator"),
arr_field : field("arr"),
arr_idx_field : field("arr_idx"),
arguments_field : field("arguments"),
from_field : field("from"),
to_field : field("to"),
write_modifiers_field : field("write_modifiers"),
expr_or_decl_field : field("expr_or_decl"),
assign_left_field : field("assign_left"),
assign_value_field : field("assign_value"),
condition_field : field("condition"),
then_block_field : field("then_block"),
else_block_field : field("else_block"),
for_decl_field : field("for_decl"),

content_field : field("content"),
item_field : field("item"),

language,
}
}
}

#[dynamic]
pub static SUS : SusTreeSitterSingleton = SusTreeSitterSingleton::new();

#[derive(Debug, Clone)]
pub struct Documentation {
gathered : Box<[Span]>
Expand Down Expand Up @@ -357,7 +205,7 @@ impl<'t> Cursor<'t> {
pub fn field(&mut self, field_id : NonZeroU16) {
if !self.optional_field(field_id) {
self.print_stack();
panic!("Did not find required field '{}'", SUS.language.field_name_for_id(field_id.into()).unwrap());
panic!("Did not find required field '{}'", tree_sitter_sus::language().field_name_for_id(field_id.into()).unwrap());
}
}

Expand All @@ -368,7 +216,7 @@ impl<'t> Cursor<'t> {
let span = node.byte_range().into();
if kind != expected_kind {
self.print_stack();
panic!("Expected {}, Was {} instead", SUS.language.node_kind_for_id(expected_kind).unwrap(), node.kind());
panic!("Expected {}, Was {} instead", tree_sitter_sus::language().node_kind_for_id(expected_kind).unwrap(), node.kind());
}
span
}
Expand All @@ -379,7 +227,7 @@ impl<'t> Cursor<'t> {
let kind = node.kind_id();
if kind != expected_kind {
self.print_stack();
panic!("Expected {}, Was {} instead", SUS.language.node_kind_for_id(expected_kind).unwrap(), node.kind());
panic!("Expected {}, Was {} instead", tree_sitter_sus::language().node_kind_for_id(expected_kind).unwrap(), node.kind());
}
}

Expand Down Expand Up @@ -429,11 +277,11 @@ impl<'t> Cursor<'t> {
self.go_down(parent_kind, |cursor| {
loop {
if let Some(found) = cursor.cursor.field_id() {
if found == SUS.item_field {
if found == field!("item") {
func(cursor);
} else {
cursor.print_stack();
panic!("List did not only contain 'item' fields, found field '{}' instead!", SUS.language.field_name_for_id(found.into()).unwrap());
panic!("List did not only contain 'item' fields, found field '{}' instead!", tree_sitter_sus::language().field_name_for_id(found.into()).unwrap());
}
} else {
cursor.maybe_add_comment();
Expand Down Expand Up @@ -464,7 +312,7 @@ impl<'t> Cursor<'t> {
#[track_caller]
pub fn go_down_content<OT, F : FnOnce(&mut Self) -> OT>(&mut self, parent_kind : u16, func : F) -> OT {
self.go_down(parent_kind, |self2| {
self2.field(SUS.content_field);
self2.field(field!("content"));
func(self2)
})
}
Expand All @@ -475,10 +323,10 @@ impl<'t> Cursor<'t> {
let node = self.cursor.node();
let kind = node.kind_id();

if kind == SUS.single_line_comment_kind || kind == SUS.multi_line_comment_kind {
if kind == kind!("single_line_comment") || kind == kind!("multi_line_comment") {
let mut range = node.byte_range();
range.start += 2; // skip '/*' or '//'
if kind == SUS.multi_line_comment_kind {
if kind == kind!("multi_line_comment") {
range.end -= 2; // skip '*/'
}
self.gathered_comments.push(Span::from(range));
Expand Down
Loading

0 comments on commit 330a4d0

Please sign in to comment.