Stop emitting spans from proc macro compile time in quote expansion

Before this commit if the proc_macro::quote!{} macro was used, the span
of each token as written in the source of the proc macro itself would be
saved in the crate metadata of the proc macro and then recovered at proc
macro runtime to forward this to the macro expansion of the proc macro.
This commit stops doing this and instead generates def-site spans for
each token. This removes the only case where spans from the proc macro
source have a semantic effect on the compilation of crates that use the
proc macro. This makes it easier to stop requiring all dependencies of
proc macros to be present when using the proc macro. And will make it
easier to stop requiring a proc macro to be present when using a crate
that used this proc macro internally but doesn't expose it as part of
it's public api. The latter is necessary to be able to cross-compile
tools that link against rustc internals without requiring to be built as
part of rustc with the -Zdual-proc-macro hack. It may also enable using
proc macros inside the standard library or it's dependencies without
breaking cross-compilation.

Author: bjorn3

compiler/rustc_expand/src/base.rs

@@ -19,7 +19,7 @@ use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, RegisteredTools};
 use rustc_parse::{parser, MACRO_ARGUMENTS};
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::{parse::ParseSess, Limit, Session};
-use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
+use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId, MacroKind};
 use rustc_span::source_map::SourceMap;
@@ -1015,10 +1015,6 @@ pub trait ResolverExpand {
         path: &ast::Path,
     ) -> Result<bool, Indeterminate>;
 
-    /// Decodes the proc-macro quoted span in the specified crate, with the specified id.
-    /// No caching is performed.
-    fn get_proc_macro_quoted_span(&self, krate: CrateNum, id: usize) -> Span;
-
     /// The order of items in the HIR is unrelated to the order of
     /// items in the AST. However, we generate proc macro harnesses
     /// based on the AST order, and later refer to these harnesses

compiler/rustc_expand/src/proc_macro_server.rs

@@ -9,13 +9,11 @@ use rustc_ast::token;
 use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
 use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parse_stream_from_source_str;
 use rustc_session::parse::ParseSess;
-use rustc_span::def_id::CrateNum;
 use rustc_span::symbol::{self, sym, Symbol};
 use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
 use smallvec::{smallvec, SmallVec};
@@ -419,8 +417,6 @@ pub(crate) struct Rustc<'a, 'b> {
     def_site: Span,
     call_site: Span,
     mixed_site: Span,
-    krate: CrateNum,
-    rebased_spans: FxHashMap<usize, Span>,
 }
 
 impl<'a, 'b> Rustc<'a, 'b> {
@@ -430,8 +426,6 @@ impl<'a, 'b> Rustc<'a, 'b> {
             def_site: ecx.with_def_site_ctxt(expn_data.def_site),
             call_site: ecx.with_call_site_ctxt(expn_data.call_site),
             mixed_site: ecx.with_mixed_site_ctxt(expn_data.call_site),
-            krate: expn_data.macro_def_id.unwrap().krate,
-            rebased_spans: FxHashMap::default(),
             ecx,
         }
     }
@@ -781,43 +775,6 @@ impl server::Span for Rustc<'_, '_> {
     fn source_text(&mut self, span: Self::Span) -> Option<String> {
         self.psess().source_map().span_to_snippet(span).ok()
     }
-
-    /// Saves the provided span into the metadata of
-    /// *the crate we are currently compiling*, which must
-    /// be a proc-macro crate. This id can be passed to
-    /// `recover_proc_macro_span` when our current crate
-    /// is *run* as a proc-macro.
-    ///
-    /// Let's suppose that we have two crates - `my_client`
-    /// and `my_proc_macro`. The `my_proc_macro` crate
-    /// contains a procedural macro `my_macro`, which
-    /// is implemented as: `quote! { "hello" }`
-    ///
-    /// When we *compile* `my_proc_macro`, we will execute
-    /// the `quote` proc-macro. This will save the span of
-    /// "hello" into the metadata of `my_proc_macro`. As a result,
-    /// the body of `my_proc_macro` (after expansion) will end
-    /// up containing a call that looks like this:
-    /// `proc_macro::Ident::new("hello", proc_macro::Span::recover_proc_macro_span(0))`
-    ///
-    /// where `0` is the id returned by this function.
-    /// When `my_proc_macro` *executes* (during the compilation of `my_client`),
-    /// the call to `recover_proc_macro_span` will load the corresponding
-    /// span from the metadata of `my_proc_macro` (which we have access to,
-    /// since we've loaded `my_proc_macro` from disk in order to execute it).
-    /// In this way, we have obtained a span pointing into `my_proc_macro`
-    fn save_span(&mut self, span: Self::Span) -> usize {
-        self.psess().save_proc_macro_span(span)
-    }
-
-    fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span {
-        let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site);
-        *self.rebased_spans.entry(id).or_insert_with(|| {
-            // FIXME: `SyntaxContext` for spans from proc macro crates is lost during encoding,
-            // replace it with a def-site context until we are encoding it properly.
-            resolver.get_proc_macro_quoted_span(krate, id).with_ctxt(def_site.ctxt())
-        })
-    }
 }
 
 impl server::Symbol for Rustc<'_, '_> {

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1475,15 +1475,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.native_libraries.decode((self, sess))
     }
 
-    fn get_proc_macro_quoted_span(self, index: usize, sess: &Session) -> Span {
-        self.root
-            .tables
-            .proc_macro_quoted_spans
-            .get(self, index)
-            .unwrap_or_else(|| panic!("Missing proc macro quoted span: {index:?}"))
-            .decode((self, sess))
-    }
-
     fn get_foreign_modules(self, sess: &'a Session) -> impl Iterator<Item = ForeignModule> + '_ {
         self.root.foreign_modules.decode((self, sess))
     }

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -579,15 +579,6 @@ impl CStore {
         self.get_crate_data(cnum).num_def_ids()
     }
 
-    pub fn get_proc_macro_quoted_span_untracked(
-        &self,
-        cnum: CrateNum,
-        id: usize,
-        sess: &Session,
-    ) -> Span {
-        self.get_crate_data(cnum).get_proc_macro_quoted_span(id, sess)
-    }
-
     pub fn set_used_recursively(&mut self, cnum: CrateNum) {
         let cmeta = self.get_crate_data_mut(cnum);
         if !cmeta.used {

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1809,10 +1809,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let stability = tcx.lookup_stability(CRATE_DEF_ID);
             let macros =
                 self.lazy_array(tcx.resolutions(()).proc_macros.iter().map(|p| p.local_def_index));
-            for (i, span) in self.tcx.sess.psess.proc_macro_quoted_spans() {
-                let span = self.lazy(span);
-                self.tables.proc_macro_quoted_spans.set_some(i, span);
-            }
 
             self.tables.def_kind.set_some(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
             record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -453,7 +453,6 @@ define_tables! {
     // `DefPathTable` up front, since we may only ever use a few
     // definitions from any given crate.
     def_keys: Table<DefIndex, LazyValue<DefKey>>,
-    proc_macro_quoted_spans: Table<usize, LazyValue<Span>>,
     variant_data: Table<DefIndex, LazyValue<VariantData>>,
     assoc_container: Table<DefIndex, ty::AssocItemContainer>,
     macro_definition: Table<DefIndex, LazyValue<ast::DelimArgs>>,

compiler/rustc_resolve/src/macros.rs

@@ -20,7 +20,7 @@ use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::compile_declarative_macro;
 use rustc_expand::expand::{AstFragment, Invocation, InvocationKind, SupportsMacroExpansion};
 use rustc_hir::def::{self, DefKind, Namespace, NonMacroAttrKind};
-use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::middle::stability;
 use rustc_middle::ty::RegisteredTools;
 use rustc_middle::ty::{TyCtxt, Visibility};
@@ -437,10 +437,6 @@ impl<'a, 'tcx> ResolverExpand for Resolver<'a, 'tcx> {
         self.path_accessible(expn_id, path, &[MacroNS])
     }
 
-    fn get_proc_macro_quoted_span(&self, krate: CrateNum, id: usize) -> Span {
-        self.cstore().get_proc_macro_quoted_span_untracked(krate, id, self.tcx.sess)
-    }
-
     fn declare_proc_macro(&mut self, id: NodeId) {
         self.proc_macros.push(id)
     }

compiler/rustc_session/src/parse.rs

@@ -227,9 +227,6 @@ pub struct ParseSess {
     pub file_depinfo: Lock<FxIndexSet<Symbol>>,
     /// Whether cfg(version) should treat the current release as incomplete
     pub assume_incomplete_release: bool,
-    /// Spans passed to `proc_macro::quote_span`. Each span has a numerical
-    /// identifier represented by its position in the vector.
-    proc_macro_quoted_spans: AppendOnlyVec<Span>,
     /// Used to generate new `AttrId`s. Every `AttrId` is unique.
     pub attr_id_generator: AttrIdGenerator,
 }
@@ -264,7 +261,6 @@ impl ParseSess {
             env_depinfo: Default::default(),
             file_depinfo: Default::default(),
             assume_incomplete_release: false,
-            proc_macro_quoted_spans: Default::default(),
             attr_id_generator: AttrIdGenerator::new(),
         }
     }
@@ -316,14 +312,4 @@ impl ParseSess {
             });
         });
     }
-
-    pub fn save_proc_macro_span(&self, span: Span) -> usize {
-        self.proc_macro_quoted_spans.push(span)
-    }
-
-    pub fn proc_macro_quoted_spans(&self) -> impl Iterator<Item = (usize, Span)> + '_ {
-        // This is equivalent to `.iter().copied().enumerate()`, but that isn't possible for
-        // AppendOnlyVec, so we resort to this scheme.
-        self.proc_macro_quoted_spans.iter_enumerated()
-    }
 }

library/proc_macro/src/bridge/mod.rs

@@ -103,8 +103,6 @@ macro_rules! with_api {
                 fn subspan($self: $S::Span, start: Bound<usize>, end: Bound<usize>) -> Option<$S::Span>;
                 fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span;
                 fn source_text($self: $S::Span) -> Option<String>;
-                fn save_span($self: $S::Span) -> usize;
-                fn recover_proc_macro_span(id: usize) -> $S::Span;
             },
             Symbol {
                 fn normalize_and_validate_ident(string: &str) -> Result<$S::Symbol, ()>;

library/proc_macro/src/lib.rs

@@ -226,7 +226,7 @@ impl Default for TokenStream {
 }
 
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
-pub use quote::{quote, quote_span};
+pub use quote::quote;
 
 fn tree_to_bridge_tree(
     tree: TokenTree,
@@ -576,20 +576,6 @@ impl Span {
         self.0.source_text()
     }
 
-    // Used by the implementation of `Span::quote`
-    #[doc(hidden)]
-    #[unstable(feature = "proc_macro_internals", issue = "27812")]
-    pub fn save_span(&self) -> usize {
-        self.0.save_span()
-    }
-
-    // Used by the implementation of `Span::quote`
-    #[doc(hidden)]
-    #[unstable(feature = "proc_macro_internals", issue = "27812")]
-    pub fn recover_proc_macro_span(id: usize) -> Span {
-        Span(bridge::client::Span::recover_proc_macro_span(id))
-    }
-
     diagnostic_method!(error, Level::Error);
     diagnostic_method!(warning, Level::Warning);
     diagnostic_method!(note, Level::Note);

library/proc_macro/src/quote.rs

@@ -64,7 +64,6 @@ pub fn quote(stream: TokenStream) -> TokenStream {
     if stream.is_empty() {
         return quote!(crate::TokenStream::new());
     }
-    let proc_macro_crate = quote!(crate);
     let mut after_dollar = false;
     let tokens = stream
         .into_iter()
@@ -105,7 +104,7 @@ pub fn quote(stream: TokenStream) -> TokenStream {
                 ))),
                 TokenTree::Ident(tt) => quote!(crate::TokenTree::Ident(crate::Ident::new(
                     (@ TokenTree::from(Literal::string(&tt.to_string()))),
-                    (@ quote_span(proc_macro_crate.clone(), tt.span())),
+                    crate::Span::def_site(),
                 ))),
                 TokenTree::Literal(tt) => quote!(crate::TokenTree::Literal({
                     let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
@@ -115,7 +114,7 @@ pub fn quote(stream: TokenStream) -> TokenStream {
                     if let (Some(crate::TokenTree::Literal(mut lit)), None) =
                         (iter.next(), iter.next())
                     {
-                        lit.set_span((@ quote_span(proc_macro_crate.clone(), tt.span())));
+                        lit.set_span(crate::Span::def_site());
                         lit
                     } else {
                         unreachable!()
@@ -131,11 +130,3 @@ pub fn quote(stream: TokenStream) -> TokenStream {
 
     quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
 }
-
-/// Quote a `Span` into a `TokenStream`.
-/// This is needed to implement a custom quoter.
-#[unstable(feature = "proc_macro_quote", issue = "54722")]
-pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream {
-    let id = span.save_span();
-    quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
-}