Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

Author: bors

Diff for: Cargo.lock

@@ -3504,6 +3504,7 @@ dependencies = [
  "rustc_lint",
  "rustc_metadata",
  "rustc_mir",
+ "rustc_parse",
  "rustc_plugin",
  "rustc_plugin_impl",
  "rustc_resolve",
@@ -3572,6 +3573,7 @@ dependencies = [
  "rustc_lint",
  "rustc_metadata",
  "rustc_mir",
+ "rustc_parse",
  "rustc_passes",
  "rustc_plugin_impl",
  "rustc_privacy",
@@ -3649,6 +3651,7 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_index",
+ "rustc_parse",
  "rustc_target",
  "serialize",
  "smallvec 1.0.0",
@@ -3692,6 +3695,21 @@ dependencies = [
  "core",
 ]
 
+[[package]]
+name = "rustc_parse"
+version = "0.0.0"
+dependencies = [
+ "bitflags",
+ "log",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_lexer",
+ "rustc_target",
+ "smallvec 1.0.0",
+ "syntax",
+ "syntax_pos",
+]
+
 [[package]]
 name = "rustc_passes"
 version = "0.0.0"
@@ -3701,6 +3719,7 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_index",
+ "rustc_parse",
  "rustc_target",
  "syntax",
  "syntax_pos",
@@ -3763,6 +3782,7 @@ dependencies = [
  "rustc",
  "rustc_codegen_utils",
  "rustc_data_structures",
+ "rustc_parse",
  "serde_json",
  "syntax",
  "syntax_pos",
@@ -4372,14 +4392,11 @@ dependencies = [
 name = "syntax_expand"
 version = "0.0.0"
 dependencies = [
- "bitflags",
- "lazy_static 1.3.0",
  "log",
  "rustc_data_structures",
  "rustc_errors",
- "rustc_index",
  "rustc_lexer",
- "scoped-tls",
+ "rustc_parse",
  "serialize",
  "smallvec 1.0.0",
  "syntax",
@@ -4394,6 +4411,7 @@ dependencies = [
  "log",
  "rustc_data_structures",
  "rustc_errors",
+ "rustc_parse",
  "rustc_target",
  "smallvec 1.0.0",
  "syntax",

Diff for: src/librustc/session/mod.rs

@@ -26,7 +26,7 @@ use syntax::expand::allocator::AllocatorKind;
 use syntax::feature_gate::{self, AttributeType};
 use syntax::json::JsonEmitter;
 use syntax::source_map;
-use syntax::sess::ParseSess;
+use syntax::sess::{ParseSess, ProcessCfgMod};
 use syntax::symbol::Symbol;
 use syntax_pos::{MultiSpan, Span};
 use crate::util::profiling::{SelfProfiler, SelfProfilerRef};
@@ -934,6 +934,7 @@ pub fn build_session(
     sopts: config::Options,
     local_crate_source_file: Option<PathBuf>,
     registry: errors::registry::Registry,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     let file_path_mapping = sopts.file_path_mapping();
 
@@ -944,6 +945,7 @@ pub fn build_session(
         Lrc::new(source_map::SourceMap::new(file_path_mapping)),
         DiagnosticOutput::Default,
         Default::default(),
+        process_cfg_mod,
     )
 }
 
@@ -1022,6 +1024,7 @@ pub fn build_session_with_source_map(
     source_map: Lrc<source_map::SourceMap>,
     diagnostics_output: DiagnosticOutput,
     lint_caps: FxHashMap<lint::LintId, lint::Level>,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     // FIXME: This is not general enough to make the warning lint completely override
     // normal diagnostic warnings, since the warning lint can also be denied and changed
@@ -1062,7 +1065,14 @@ pub fn build_session_with_source_map(
         },
     );
 
-    build_session_(sopts, local_crate_source_file, diagnostic_handler, source_map, lint_caps)
+    build_session_(
+        sopts,
+        local_crate_source_file,
+        diagnostic_handler,
+        source_map,
+        lint_caps,
+        process_cfg_mod,
+    )
 }
 
 fn build_session_(
@@ -1071,6 +1081,7 @@ fn build_session_(
     span_diagnostic: errors::Handler,
     source_map: Lrc<source_map::SourceMap>,
     driver_lint_caps: FxHashMap<lint::LintId, lint::Level>,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     let self_profiler =
         if let SwitchWithOptPath::Enabled(ref d) = sopts.debugging_opts.self_profile {
@@ -1109,6 +1120,7 @@ fn build_session_(
     let parse_sess = ParseSess::with_span_handler(
         span_diagnostic,
         source_map,
+        process_cfg_mod,
     );
     let sysroot = match &sopts.maybe_sysroot {
         Some(sysroot) => sysroot.clone(),

Diff for: src/librustc_driver/Cargo.toml

@@ -21,6 +21,7 @@ rustc_data_structures = { path = "../librustc_data_structures" }
 errors = { path = "../librustc_errors", package = "rustc_errors" }
 rustc_metadata = { path = "../librustc_metadata" }
 rustc_mir = { path = "../librustc_mir" }
+rustc_parse = { path = "../librustc_parse" }
 rustc_plugin = { path = "../librustc_plugin/deprecated" } # To get this in the sysroot
 rustc_plugin_impl = { path = "../librustc_plugin" }
 rustc_save_analysis = { path = "../librustc_save_analysis" }

Diff for: src/librustc_driver/lib.rs

@@ -63,7 +63,6 @@ use std::time::Instant;
 use syntax::ast;
 use syntax::source_map::FileLoader;
 use syntax::feature_gate::{GatedCfg, UnstableFeatures};
-use syntax::parse;
 use syntax::symbol::sym;
 use syntax_pos::{DUMMY_SP, FileName};
 
@@ -1062,14 +1061,16 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
 }
 
 fn parse_crate_attrs<'a>(sess: &'a Session, input: &Input) -> PResult<'a, Vec<ast::Attribute>> {
-    match *input {
-        Input::File(ref ifile) => {
-            parse::parse_crate_attrs_from_file(ifile, &sess.parse_sess)
+    match input {
+        Input::File(ifile) => {
+            rustc_parse::parse_crate_attrs_from_file(ifile, &sess.parse_sess)
         }
-        Input::Str { ref name, ref input } => {
-            parse::parse_crate_attrs_from_source_str(name.clone(),
-                                                     input.clone(),
-                                                     &sess.parse_sess)
+        Input::Str { name, input } => {
+            rustc_parse::parse_crate_attrs_from_source_str(
+                name.clone(),
+                input.clone(),
+                &sess.parse_sess,
+            )
         }
     }
 }

Diff for: src/librustc_interface/Cargo.toml

@@ -16,6 +16,7 @@ smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
 syntax_ext = { path = "../libsyntax_ext" }
 syntax_expand = { path = "../libsyntax_expand" }
+rustc_parse = { path = "../librustc_parse" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 rustc = { path = "../librustc" }

Diff for: src/librustc_interface/interface.rs

@@ -11,14 +11,15 @@ use rustc_codegen_utils::codegen_backend::CodegenBackend;
 use rustc_data_structures::OnDrop;
 use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::fx::{FxHashSet, FxHashMap};
+use rustc_parse::new_parser_from_source_str;
 use std::path::PathBuf;
 use std::result;
 use std::sync::{Arc, Mutex};
-use syntax::{self, parse};
 use syntax::ast::{self, MetaItemKind};
 use syntax::token;
 use syntax::source_map::{FileName, FileLoader, SourceMap};
 use syntax::sess::ParseSess;
+use syntax_expand::config::process_configure_mod;
 use syntax_pos::edition;
 
 pub type Result<T> = result::Result<T, ErrorReported>;
@@ -64,9 +65,9 @@ impl Compiler {
 pub fn parse_cfgspecs(cfgspecs: Vec<String>) -> FxHashSet<(String, Option<String>)> {
     syntax::with_default_globals(move || {
         let cfg = cfgspecs.into_iter().map(|s| {
-            let sess = ParseSess::with_silent_emitter();
+            let sess = ParseSess::with_silent_emitter(process_configure_mod);
             let filename = FileName::cfg_spec_source_code(&s);
-            let mut parser = parse::new_parser_from_source_str(&sess, filename, s.to_string());
+            let mut parser = new_parser_from_source_str(&sess, filename, s.to_string());
 
             macro_rules! error {($reason: expr) => {
                 early_error(ErrorOutputType::default(),

Diff for: src/librustc_interface/passes.rs

@@ -27,6 +27,7 @@ use rustc_errors::PResult;
 use rustc_incremental;
 use rustc_metadata::cstore;
 use rustc_mir as mir;
+use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str};
 use rustc_passes::{self, ast_validation, hir_stats, layout_test};
 use rustc_plugin as plugin;
 use rustc_plugin::registry::Registry;
@@ -38,7 +39,6 @@ use syntax::{self, ast, visit};
 use syntax::early_buffered_lints::BufferedEarlyLint;
 use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt};
 use syntax::mut_visit::MutVisitor;
-use syntax::parse;
 use syntax::util::node_count::NodeCounter;
 use syntax::symbol::Symbol;
 use syntax_pos::FileName;
@@ -61,12 +61,11 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
     let krate = time(sess, "parsing", || {
         let _prof_timer = sess.prof.generic_activity("parse_crate");
 
-        match *input {
-            Input::File(ref file) => parse::parse_crate_from_file(file, &sess.parse_sess),
-            Input::Str {
-                ref input,
-                ref name,
-            } => parse::parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess),
+        match input {
+            Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
+            Input::Str { input, name } => {
+                parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+            }
         }
     })?;
 
@@ -182,7 +181,7 @@ pub fn register_plugins<'a>(
         )
     });
 
-    let (krate, features) = syntax::config::features(
+    let (krate, features) = syntax_expand::config::features(
         krate,
         &sess.parse_sess,
         sess.edition(),
@@ -489,7 +488,7 @@ pub fn lower_to_hir(
 ) -> Result<hir::map::Forest> {
     // Lower AST to HIR.
     let hir_forest = time(sess, "lowering AST -> HIR", || {
-        let nt_to_tokenstream = syntax::parse::nt_to_tokenstream;
+        let nt_to_tokenstream = rustc_parse::nt_to_tokenstream;
         let hir_crate = lower_crate(sess, &dep_graph, &krate, resolver, nt_to_tokenstream);
 
         if sess.opts.debugging_opts.hir_stats {

Diff for: src/librustc_interface/tests.rs

@@ -8,7 +8,7 @@ use rustc::session::config::{build_configuration, build_session_options, to_crat
 use rustc::session::config::{LtoCli, LinkerPluginLto, SwitchWithOptPath, ExternEntry};
 use rustc::session::config::{Externs, OutputType, OutputTypes, SymbolManglingVersion};
 use rustc::session::config::{rustc_optgroups, Options, ErrorOutputType, Passes};
-use rustc::session::build_session;
+use rustc::session::{build_session, Session};
 use rustc::session::search_paths::SearchPath;
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::FromIterator;
@@ -17,16 +17,23 @@ use rustc_target::spec::{MergeFunctions, PanicStrategy, RelroLevel};
 use syntax::symbol::sym;
 use syntax::edition::{Edition, DEFAULT_EDITION};
 use syntax;
+use syntax_expand::config::process_configure_mod;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{ColorConfig, emitter::HumanReadableErrorType, registry};
 
-pub fn build_session_options_and_crate_config(
-    matches: &getopts::Matches,
-) -> (Options, FxHashSet<(String, Option<String>)>) {
-    (
-        build_session_options(matches),
-        parse_cfgspecs(matches.opt_strs("cfg")),
-    )
+type CfgSpecs = FxHashSet<(String, Option<String>)>;
+
+fn build_session_options_and_crate_config(matches: getopts::Matches) -> (Options, CfgSpecs) {
+    let sessopts = build_session_options(&matches);
+    let cfg = parse_cfgspecs(matches.opt_strs("cfg"));
+    (sessopts, cfg)
+}
+
+fn mk_session(matches: getopts::Matches) -> (Session, CfgSpecs) {
+    let registry = registry::Registry::new(&[]);
+    let (sessopts, cfg) = build_session_options_and_crate_config(matches);
+    let sess = build_session(sessopts, None, registry, process_configure_mod);
+    (sess, cfg)
 }
 
 fn new_public_extern_entry<S, I>(locations: I) -> ExternEntry
@@ -59,31 +66,19 @@ fn mk_map<K: Ord, V>(entries: Vec<(K, V)>) -> BTreeMap<K, V> {
 #[test]
 fn test_switch_implies_cfg_test() {
     syntax::with_default_globals(|| {
-        let matches = &match optgroups().parse(&["--test".to_string()]) {
-            Ok(m) => m,
-            Err(f) => panic!("test_switch_implies_cfg_test: {}", f),
-        };
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, cfg) = build_session_options_and_crate_config(matches);
-        let sess = build_session(sessopts, None, registry);
+        let matches = optgroups().parse(&["--test".to_string()]).unwrap();
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, to_crate_config(cfg));
         assert!(cfg.contains(&(sym::test, None)));
     });
 }
 
-// When the user supplies --test and --cfg test, don't implicitly add
-// another --cfg test
+// When the user supplies --test and --cfg test, don't implicitly add another --cfg test
 #[test]
 fn test_switch_implies_cfg_test_unless_cfg_test() {
     syntax::with_default_globals(|| {
-        let matches = &match optgroups().parse(&["--test".to_string(),
-                                                 "--cfg=test".to_string()]) {
-            Ok(m) => m,
-            Err(f) => panic!("test_switch_implies_cfg_test_unless_cfg_test: {}", f),
-        };
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, cfg) = build_session_options_and_crate_config(matches);
-        let sess = build_session(sessopts, None, registry);
+        let matches = optgroups().parse(&["--test".to_string(), "--cfg=test".to_string()]).unwrap();
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, to_crate_config(cfg));
         let mut test_items = cfg.iter().filter(|&&(name, _)| name == sym::test);
         assert!(test_items.next().is_some());
@@ -95,27 +90,21 @@ fn test_switch_implies_cfg_test_unless_cfg_test() {
 fn test_can_print_warnings() {
     syntax::with_default_globals(|| {
         let matches = optgroups().parse(&["-Awarnings".to_string()]).unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(!sess.diagnostic().can_emit_warnings());
     });
 
     syntax::with_default_globals(|| {
         let matches = optgroups()
             .parse(&["-Awarnings".to_string(), "-Dwarnings".to_string()])
             .unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(sess.diagnostic().can_emit_warnings());
     });
 
     syntax::with_default_globals(|| {
         let matches = optgroups().parse(&["-Adead_code".to_string()]).unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(sess.diagnostic().can_emit_warnings());
     });
 }
@@ -704,6 +693,6 @@ fn test_edition_parsing() {
     let matches = optgroups()
         .parse(&["--edition=2018".to_string()])
         .unwrap();
-    let (sessopts, _) = build_session_options_and_crate_config(&matches);
+    let (sessopts, _) = build_session_options_and_crate_config(matches);
     assert!(sessopts.edition == Edition::Edition2018)
 }