From 939dc0ee5c20e058ff06e72d358a808126a90e95 Mon Sep 17 00:00:00 2001 From: Daniel Kroening Date: Sun, 24 Sep 2023 11:41:21 -0700 Subject: [PATCH] Verilog preprocessor: implement define parameters This implements Verilog 2001 preprocessor defines with parameters. --- regression/verilog/preprocessor/define1.desc | 19 ++ regression/verilog/preprocessor/define1.v | 9 + src/verilog/verilog_preprocessor.cpp | 271 +++++++++++++------ src/verilog/verilog_preprocessor.h | 92 +++++-- 4 files changed, 292 insertions(+), 99 deletions(-) create mode 100644 regression/verilog/preprocessor/define1.desc create mode 100644 regression/verilog/preprocessor/define1.v diff --git a/regression/verilog/preprocessor/define1.desc b/regression/verilog/preprocessor/define1.desc new file mode 100644 index 000000000..eab4b5cf1 --- /dev/null +++ b/regression/verilog/preprocessor/define1.desc @@ -0,0 +1,19 @@ +CORE +define1.v +--preprocess +// Enable multi-line checking +activate-multi-line-match +`line 1 "define1.v" 0 + + + +value + +value + +x-y-z +x-y-value +^EXIT=0$ +^SIGNAL=0$ +-- +^PREPROCESSING FAILED$ diff --git a/regression/verilog/preprocessor/define1.v b/regression/verilog/preprocessor/define1.v new file mode 100644 index 000000000..44bc74141 --- /dev/null +++ b/regression/verilog/preprocessor/define1.v @@ -0,0 +1,9 @@ +`define basic +`basic +`define with_value value +`with_value +`define uses_previous `with_value +`uses_previous +`define with_parameter(a, b, c) a-b-c +`with_parameter(x, y, z) +`with_parameter(x, y, `with_value) diff --git a/src/verilog/verilog_preprocessor.cpp b/src/verilog/verilog_preprocessor.cpp index 6f756a34d..6bdc2681d 100644 --- a/src/verilog/verilog_preprocessor.cpp +++ b/src/verilog/verilog_preprocessor.cpp @@ -18,7 +18,7 @@ Author: Daniel Kroening, kroening@kroening.com /*******************************************************************\ -Function: verilog_preprocessort::filet::make_source_location +Function: verilog_preprocessort::contextt::make_source_location Inputs: @@ -28,18 +28,66 @@ Function: verilog_preprocessort::filet::make_source_location \*******************************************************************/ -source_locationt verilog_preprocessort::filet::make_source_location() const +source_locationt verilog_preprocessort::contextt::make_source_location() const { source_locationt result; result.set_file(filename); - result.set_line(tokenizer.line_no()); + result.set_line(tokenizer->line_no()); return result; } /*******************************************************************\ +Function: verilog_preprocessort::as_string + + Inputs: + + Outputs: + + Purpose: + +\*******************************************************************/ + +std::string verilog_preprocessort::as_string(const std::vector &tokens) +{ + std::string result; + + for(auto &t : tokens) + result.append(t.text); + + return result; +} + +/*******************************************************************\ + +Function: verilog_preprocessort::vector_token_sourcet::get_token_from_stream + + Inputs: + + Outputs: + + Purpose: + +\*******************************************************************/ + +void verilog_preprocessort::vector_token_sourcet::get_token_from_stream() +{ + if(pos == tokens.end()) + { + token.text.clear(); + token.kind = tokent::END_OF_FILE; + } + else + { + token = *pos; + pos++; + } +} + +/*******************************************************************\ + Function: verilog_preprocessort::include Inputs: @@ -62,8 +110,8 @@ void verilog_preprocessort::include(const std::string &filename) if(*in) { - files.emplace_back(true, in, filename); - files.back().print_line_directive(out, 1); // 'enter' + context_stack.emplace_back(true, in, filename); + context().print_line_directive(out, 1); // 'enter' return; // done } else @@ -83,8 +131,8 @@ void verilog_preprocessort::include(const std::string &filename) if(*in) { - files.emplace_back(true, in, filename); - files.back().print_line_directive(out, 1); // 'enter' + context_stack.emplace_back(true, in, filename); + context().print_line_directive(out, 1); // 'enter' return; // done } @@ -111,11 +159,12 @@ void verilog_preprocessort::preprocessor() { try { - files.emplace_back(false, &in, filename); + // the first context is the input file + context_stack.emplace_back(false, &in, filename); - files.back().print_line_directive(out, 0); // 'neither' + context().print_line_directive(out, 0); // 'neither' - while(!files.empty()) + while(!context_stack.empty()) { while(!tokenizer().eof()) { @@ -124,20 +173,30 @@ void verilog_preprocessort::preprocessor() directive(); else if(condition) { - out << token; + auto a_it = context().define_arguments.find(token.text); + if(a_it == context().define_arguments.end()) + out << token; + else + { + // Create a new context for the define argument. + // We then continue in that context. + context_stack.emplace_back(a_it->second); + } } } - files.pop_back(); + const bool is_file = context().is_file(); + context_stack.pop_back(); - if(!files.empty()) - files.back().print_line_directive(out, 2); // 'exit' + // print the line directive when we exit an include file + if(!context_stack.empty() && is_file) + context().print_line_directive(out, 2); // 'exit' } } catch(const verilog_preprocessor_errort &e) { - if(!files.empty()) - error().source_location = files.back().make_source_location(); + if(!context_stack.empty()) + error().source_location = context().make_source_location(); error() << e.what() << eom; throw 0; } @@ -145,7 +204,7 @@ void verilog_preprocessort::preprocessor() /*******************************************************************\ -Function: verilog_preprocessort::replace_macros +Function: verilog_preprocessort::parse_define_parameters Inputs: @@ -155,43 +214,96 @@ Function: verilog_preprocessort::replace_macros \*******************************************************************/ -void verilog_preprocessort::replace_macros(std::string &s) +auto verilog_preprocessort::parse_define_parameters() -> definet::parameterst { - std::string dest; + tokenizer().next_token(); // eat ( - for(unsigned i=0; isecond; - } + if(token == ')') + break; // done + else if(token == ',') + continue; // keep going + else if(token == '=') // SystemVerilog 2009 + throw verilog_preprocessor_errort() + << "default parameters are not supported yet"; else + throw verilog_preprocessor_errort() << "expecting a define parameter"; + } + + return result; +} + +/*******************************************************************\ + +Function: verilog_preprocessort::parse_define_arguments + + Inputs: + + Outputs: + + Purpose: + +\*******************************************************************/ + +auto verilog_preprocessort::parse_define_arguments(const definet &define) + -> std::map> +{ + if(define.parameters.empty()) + return {}; + + if(tokenizer().next_token() != '(') + throw verilog_preprocessor_errort() << "expecting define arguments"; + + // We start with a vector of size 1, + // which contains one empty vector of argument tokens. + std::vector> arguments = {{}}; + + while(true) + { + if(tokenizer().eof()) + throw verilog_preprocessor_errort() + << "eof inside a define argument list"; + + auto token = tokenizer().next_token(); + if(token == ',') { - dest+=s[i]; - i++; + arguments.push_back({}); // next argument + tokenizer().skip_ws(); } + else if(token == ')') + break; // done + else + arguments.back().push_back(std::move(token)); } - dest.swap(s); + // does the number of arguments match the number of parameters? + if(arguments.size() != define.parameters.size()) + throw verilog_preprocessor_errort() + << "expected " << define.parameters.size() << " arguments, but got " + << arguments.size(); + + // sort into the map + std::map> result; + + for(std::size_t i = 0; i < define.parameters.size(); i++) + result[define.parameters[i]] = std::move(arguments[i]); + + return result; } /*******************************************************************\ @@ -209,12 +321,13 @@ Function: verilog_preprocessort::directive void verilog_preprocessort::directive() { // we expect an identifier after the backtick - auto directive_token = tokenizer().next_token(); + const auto directive_token = tokenizer().next_token(); + if(!directive_token.is_identifier()) throw verilog_preprocessor_errort() << "expecting an identifier after backtick"; - const auto &text = directive_token.text; + auto &text = directive_token.text; if(text=="define") { @@ -229,47 +342,41 @@ void verilog_preprocessort::directive() tokenizer().skip_ws(); // we expect an identifier after `define - auto identifier_token = tokenizer().next_token(); + const auto identifier_token = tokenizer().next_token(); + if(!identifier_token.is_identifier()) throw verilog_preprocessor_errort() << "expecting an identifier after `define"; auto &identifier = identifier_token.text; + auto &define = defines[identifier]; + + // skip whitespace + tokenizer().skip_ws(); // Is there a parameter list? // These have been introduced in Verilog 2001. if(tokenizer().peek() == '(') - { - throw verilog_preprocessor_errort() - << "`define with parameters not yet supported"; - } + define.parameters = parse_define_parameters(); // skip whitespace tokenizer().skip_ws(); - // read any tokens until end of line - std::string value; - while(!tokenizer().eof()) + // Read any tokens until end of line. + // Note that any defines in this sequence + // are not expanded at this point. + while(!tokenizer().eof() && tokenizer().peek() != '\n') { auto token = tokenizer().next_token(); - if(token.is_identifier()) - { - value += token.text; - } - else if(token == '\n') - break; - else - { - value += token.text; - } + define.tokens.push_back(std::move(token)); } #ifdef DEBUG - std::cout << "DEFINE: >" << identifier - << "< = >" << value << "<" << std::endl; - #endif - - defines[identifier]=value; + std::cout << "DEFINE: >" << identifier << "< = >"; + for(auto &t : define.tokens) + std::cout << t; + std::cout << '<' << std::endl; +#endif } else if(text=="undef") { @@ -284,7 +391,8 @@ void verilog_preprocessort::directive() tokenizer().skip_ws(); // we expect an identifier after `undef - auto identifier_token = tokenizer().next_token(); + const auto identifier_token = tokenizer().next_token(); + if(!identifier_token.is_identifier()) throw verilog_preprocessor_errort() << "expecting an identifier after `undef"; @@ -309,7 +417,8 @@ void verilog_preprocessort::directive() tokenizer().skip_ws(); // we expect an identifier - auto identifier_token = tokenizer().next_token(); + const auto identifier_token = tokenizer().next_token(); + if(!identifier_token.is_identifier()) throw verilog_preprocessor_errort() << "expecting an identifier after ifdef"; @@ -370,7 +479,7 @@ void verilog_preprocessort::directive() tokenizer().skip_ws(); // we expect a string literal - auto file_token = tokenizer().next_token(); + const auto file_token = tokenizer().next_token(); if(!file_token.is_string_literal()) throw verilog_preprocessor_errort() << "expecting a string literal after `include"; @@ -403,19 +512,23 @@ void verilog_preprocessort::directive() else { // check defines + if(!condition) + return; // ignore - if(condition) + definest::const_iterator it = defines.find(text); + + if(it == defines.end()) { - definest::const_iterator it=defines.find(text); + throw verilog_preprocessor_errort() + << "unknown preprocessor directive \"" << text << "\""; + } - if(it==defines.end()) - { - throw verilog_preprocessor_errort() - << "unknown preprocessor directive \"" << text << "\""; - } + // Found it! + // Parse the arguments, if any. + auto arguments = parse_define_arguments(it->second); - // found it! replace it! - out << it->second; - } + // Create a new context. We then continue in that context. + context_stack.emplace_back(it->second.tokens); + context().define_arguments = std::move(arguments); } } diff --git a/src/verilog/verilog_preprocessor.h b/src/verilog/verilog_preprocessor.h index 0a416bea2..ed55645f1 100644 --- a/src/verilog/verilog_preprocessor.h +++ b/src/verilog/verilog_preprocessor.h @@ -9,6 +9,7 @@ #include "verilog_preprocessor_tokenizer.h" #include +#include class verilog_preprocessort:public preprocessort { @@ -28,14 +29,26 @@ class verilog_preprocessort:public preprocessort virtual ~verilog_preprocessort() { } protected: - typedef std::unordered_map - definest; - + using tokent = verilog_preprocessor_token_sourcet::tokent; + + struct definet + { + using parameterst = std::vector; + parameterst parameters; + std::vector tokens; + }; + + static std::string as_string(const std::vector &); + + using definest = std::unordered_map; definest defines; - - virtual void directive(); - virtual void replace_macros(std::string &s); - virtual void include(const std::string &filename); + + void directive(); + void include(const std::string &filename); + definet::parameterst parse_define_parameters(); + + using define_argumentst = std::map>; + define_argumentst parse_define_arguments(const definet &); // for ifdef, else, endif @@ -60,45 +73,84 @@ class verilog_preprocessort:public preprocessort std::list conditionals; - // for include + class vector_token_sourcet : public verilog_preprocessor_token_sourcet + { + public: + vector_token_sourcet(const std::vector &_tokens) + : tokens(_tokens), pos(tokens.begin()) + { + } + + protected: + const std::vector &tokens; + std::vector::const_iterator pos; + void get_token_from_stream() override; + }; - class filet + // for include and for `define + class contextt { protected: - bool close; + bool deallocate_in; std::istream *in; std::string filename; public: - verilog_preprocessor_tokenizert tokenizer; + verilog_preprocessor_token_sourcet *tokenizer; + + // for `define with parameters + define_argumentst define_arguments; + + contextt(bool _deallocate_in, std::istream *_in, std::string _filename) + : deallocate_in(_deallocate_in), + in(_in), + filename(std::move(_filename)), + tokenizer(new verilog_preprocessor_tokenizert(*in)) + { + } - filet(bool _close, std::istream *_in, std::string _filename) - : close(_close), in(_in), filename(std::move(_filename)), tokenizer(*in) + explicit contextt(const std::vector &tokens) + : deallocate_in(false), + in(nullptr), + filename(), + tokenizer(new vector_token_sourcet(tokens)) { } - ~filet() + ~contextt() { - if(close) + delete tokenizer; + if(deallocate_in) delete in; } void print_line_directive(std::ostream &out, unsigned level) const { - out << "`line " << tokenizer.line_no() << " \"" << filename << "\" " + out << "`line " << tokenizer->line_no() << " \"" << filename << "\" " << level << '\n'; } source_locationt make_source_location() const; + + bool is_file() const + { + return !filename.empty(); + } }; - std::list files; + std::list context_stack; + + // the topmost context + contextt &context() + { + PRECONDITION(!context_stack.empty()); + return context_stack.back(); + } // the topmost tokenizer - verilog_preprocessor_tokenizert &tokenizer() + verilog_preprocessor_token_sourcet &tokenizer() { - PRECONDITION(!files.empty()); - return files.back().tokenizer; + return *(context().tokenizer); } };