Skip to content

Commit d4689f1

Browse files
borsbors
committed
Auto merge of #18151 - ChayimFriedman2:metavar-concat, r=Veykril
feat: Support the `${concat(...)}` metavariable expression I didn't follow rustc precisely, because I think it does some things wrongly (or they are FIXME), but I only allowed more code, not less. So we're all fine. Closes #18145.
2 parents c467115 + 8a50aec commit d4689f1

File tree

9 files changed

+304
-5
lines changed

9 files changed

+304
-5
lines changed

‎Cargo.lock

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs

+147
Original file line numberDiff line numberDiff line change
@@ -311,3 +311,150 @@ fn test() {
311311
"#]],
312312
);
313313
}
314+
315+
#[test]
316+
fn concat() {
317+
// FIXME: Should this error? rustc currently accepts it.
318+
check(
319+
r#"
320+
macro_rules! m {
321+
( $a:ident, $b:literal ) => {
322+
let ${concat($a, _, "123", _foo, $b, _, 123)};
323+
};
324+
}
325+
326+
fn test() {
327+
m!( abc, 456 );
328+
m!( def, "hello" );
329+
}
330+
"#,
331+
expect![[r#"
332+
macro_rules! m {
333+
( $a:ident, $b:literal ) => {
334+
let ${concat($a, _, "123", _foo, $b, _, 123)};
335+
};
336+
}
337+
338+
fn test() {
339+
let abc_123_foo456_123;;
340+
let def_123_foohello_123;;
341+
}
342+
"#]],
343+
);
344+
}
345+
346+
#[test]
347+
fn concat_less_than_two_elements() {
348+
// FIXME: Should this error? rustc currently accepts it.
349+
check(
350+
r#"
351+
macro_rules! m {
352+
() => {
353+
let ${concat(abc)};
354+
};
355+
}
356+
357+
fn test() {
358+
m!()
359+
}
360+
"#,
361+
expect![[r#"
362+
macro_rules! m {
363+
() => {
364+
let ${concat(abc)};
365+
};
366+
}
367+
368+
fn test() {
369+
/* error: macro definition has parse errors */
370+
}
371+
"#]],
372+
);
373+
}
374+
375+
#[test]
376+
fn concat_invalid_ident() {
377+
// FIXME: Should this error? rustc currently accepts it.
378+
check(
379+
r#"
380+
macro_rules! m {
381+
() => {
382+
let ${concat(abc, '"')};
383+
};
384+
}
385+
386+
fn test() {
387+
m!()
388+
}
389+
"#,
390+
expect![[r#"
391+
macro_rules! m {
392+
() => {
393+
let ${concat(abc, '"')};
394+
};
395+
}
396+
397+
fn test() {
398+
/* error: `${concat(..)}` is not generating a valid identifier */let __ra_concat_dummy;
399+
}
400+
"#]],
401+
);
402+
}
403+
404+
#[test]
405+
fn concat_invalid_fragment() {
406+
// FIXME: Should this error? rustc currently accepts it.
407+
check(
408+
r#"
409+
macro_rules! m {
410+
( $e:expr ) => {
411+
let ${concat(abc, $e)};
412+
};
413+
}
414+
415+
fn test() {
416+
m!(())
417+
}
418+
"#,
419+
expect![[r#"
420+
macro_rules! m {
421+
( $e:expr ) => {
422+
let ${concat(abc, $e)};
423+
};
424+
}
425+
426+
fn test() {
427+
/* error: metavariables of `${concat(..)}` must be of type `ident`, `literal` or `tt` */let abc;
428+
}
429+
"#]],
430+
);
431+
}
432+
433+
#[test]
434+
fn concat_repetition() {
435+
// FIXME: Should this error? rustc currently accepts it.
436+
check(
437+
r#"
438+
macro_rules! m {
439+
( $($i:ident)* ) => {
440+
let ${concat(abc, $i)};
441+
};
442+
}
443+
444+
fn test() {
445+
m!(a b c)
446+
}
447+
"#,
448+
expect![[r#"
449+
macro_rules! m {
450+
( $($i:ident)* ) => {
451+
let ${concat(abc, $i)};
452+
};
453+
}
454+
455+
fn test() {
456+
/* error: expected simple binding, found nested binding `i` */let abc;
457+
}
458+
"#]],
459+
);
460+
}

‎crates/mbe/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ rustc-hash.workspace = true
1818
smallvec.workspace = true
1919
tracing.workspace = true
2020
arrayvec.workspace = true
21+
ra-ap-rustc_lexer.workspace = true
2122

2223
# local deps
2324
syntax.workspace = true

‎crates/mbe/src/benchmark.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,11 @@ fn invocation_fixtures(
216216

217217
token_trees.push(subtree.into());
218218
}
219-
Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. } => {}
219+
Op::Ignore { .. }
220+
| Op::Index { .. }
221+
| Op::Count { .. }
222+
| Op::Len { .. }
223+
| Op::Concat { .. } => {}
220224
};
221225

222226
// Simple linear congruential generator for deterministic result

‎crates/mbe/src/expander/matcher.rs

+10-2
Original file line numberDiff line numberDiff line change
@@ -584,7 +584,11 @@ fn match_loop_inner<'t>(
584584
error_items.push(item);
585585
}
586586
OpDelimited::Op(
587-
Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. },
587+
Op::Ignore { .. }
588+
| Op::Index { .. }
589+
| Op::Count { .. }
590+
| Op::Len { .. }
591+
| Op::Concat { .. },
588592
) => {
589593
stdx::never!("metavariable expression in lhs found");
590594
}
@@ -879,7 +883,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(Symbol), pattern: &MetaTemplate)
879883
Op::Subtree { tokens, .. } => collect_vars(collector_fun, tokens),
880884
Op::Repeat { tokens, .. } => collect_vars(collector_fun, tokens),
881885
Op::Literal(_) | Op::Ident(_) | Op::Punct(_) => {}
882-
Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. } => {
886+
Op::Ignore { .. }
887+
| Op::Index { .. }
888+
| Op::Count { .. }
889+
| Op::Len { .. }
890+
| Op::Concat { .. } => {
883891
stdx::never!("metavariable expression in lhs found");
884892
}
885893
}

‎crates/mbe/src/expander/transcriber.rs

+78-2
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
33
44
use intern::{sym, Symbol};
5-
use span::Span;
5+
use span::{Edition, Span};
66
use tt::Delimiter;
77

88
use crate::{
99
expander::{Binding, Bindings, Fragment},
10-
parser::{MetaVarKind, Op, RepeatKind, Separator},
10+
parser::{ConcatMetaVarExprElem, MetaVarKind, Op, RepeatKind, Separator},
1111
ExpandError, ExpandErrorKind, ExpandResult, MetaTemplate,
1212
};
1313

@@ -312,6 +312,82 @@ fn expand_subtree(
312312
.into(),
313313
);
314314
}
315+
Op::Concat { elements, span: concat_span } => {
316+
let mut concatenated = String::new();
317+
for element in elements {
318+
match element {
319+
ConcatMetaVarExprElem::Ident(ident) => {
320+
concatenated.push_str(ident.sym.as_str())
321+
}
322+
ConcatMetaVarExprElem::Literal(lit) => {
323+
// FIXME: This isn't really correct wrt. escaping, but that's what rustc does and anyway
324+
// escaping is used most of the times for characters that are invalid in identifiers.
325+
concatenated.push_str(lit.symbol.as_str())
326+
}
327+
ConcatMetaVarExprElem::Var(var) => {
328+
// Handling of repetitions in `${concat}` isn't fleshed out in rustc, so we currently
329+
// err at it.
330+
// FIXME: Do what rustc does for repetitions.
331+
let var_value = match ctx.bindings.get_fragment(
332+
&var.sym,
333+
var.span,
334+
&mut ctx.nesting,
335+
marker,
336+
) {
337+
Ok(var) => var,
338+
Err(e) => {
339+
if err.is_none() {
340+
err = Some(e);
341+
};
342+
continue;
343+
}
344+
};
345+
let value = match &var_value {
346+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => {
347+
ident.sym.as_str()
348+
}
349+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) => {
350+
lit.symbol.as_str()
351+
}
352+
_ => {
353+
if err.is_none() {
354+
err = Some(ExpandError::binding_error(var.span, "metavariables of `${concat(..)}` must be of type `ident`, `literal` or `tt`"))
355+
}
356+
continue;
357+
}
358+
};
359+
concatenated.push_str(value);
360+
}
361+
}
362+
}
363+
364+
// `${concat}` span comes from the macro (at least for now).
365+
// See https://github.com/rust-lang/rust/blob/b0af276da341/compiler/rustc_expand/src/mbe/transcribe.rs#L724-L726.
366+
let mut result_span = *concat_span;
367+
marker(&mut result_span);
368+
369+
// FIXME: NFC normalize the result.
370+
if !rustc_lexer::is_ident(&concatenated) {
371+
if err.is_none() {
372+
err = Some(ExpandError::binding_error(
373+
*concat_span,
374+
"`${concat(..)}` is not generating a valid identifier",
375+
));
376+
}
377+
// Insert a dummy identifier for better parsing.
378+
concatenated.clear();
379+
concatenated.push_str("__ra_concat_dummy");
380+
}
381+
382+
let needs_raw =
383+
parser::SyntaxKind::from_keyword(&concatenated, Edition::LATEST).is_some();
384+
let is_raw = if needs_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No };
385+
arena.push(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
386+
is_raw,
387+
span: result_span,
388+
sym: Symbol::intern(&concatenated),
389+
})));
390+
}
315391
}
316392
}
317393
// drain the elements added in this instance of expand_subtree

‎crates/mbe/src/lib.rs

+5
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66
//! The tests for this functionality live in another crate:
77
//! `hir_def::macro_expansion_tests::mbe`.
88
9+
#[cfg(not(feature = "in-rust-tree"))]
10+
extern crate ra_ap_rustc_lexer as rustc_lexer;
11+
#[cfg(feature = "in-rust-tree")]
12+
extern crate rustc_lexer;
13+
914
mod expander;
1015
mod parser;
1116

‎crates/mbe/src/parser.rs

+50
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ pub(crate) enum Op {
8484
// FIXME: `usize`` once we drop support for 1.76
8585
depth: Option<usize>,
8686
},
87+
Concat {
88+
elements: Box<[ConcatMetaVarExprElem]>,
89+
span: Span,
90+
},
8791
Repeat {
8892
tokens: MetaTemplate,
8993
kind: RepeatKind,
@@ -98,6 +102,18 @@ pub(crate) enum Op {
98102
Ident(tt::Ident<Span>),
99103
}
100104

105+
#[derive(Clone, Debug, PartialEq, Eq)]
106+
pub(crate) enum ConcatMetaVarExprElem {
107+
/// There is NO preceding dollar sign, which means that this identifier should be interpreted
108+
/// as a literal.
109+
Ident(tt::Ident<Span>),
110+
/// There is a preceding dollar sign, which means that this identifier should be expanded
111+
/// and interpreted as a variable.
112+
Var(tt::Ident<Span>),
113+
/// For example, a number or a string.
114+
Literal(tt::Literal<Span>),
115+
}
116+
101117
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
102118
pub(crate) enum RepeatKind {
103119
ZeroOrMore,
@@ -384,6 +400,32 @@ fn parse_metavar_expr(src: &mut TtIter<'_, Span>) -> Result<Op, ()> {
384400
let depth = if try_eat_comma(&mut args) { Some(parse_depth(&mut args)?) } else { None };
385401
Op::Count { name: ident.sym.clone(), depth }
386402
}
403+
s if sym::concat == *s => {
404+
let mut elements = Vec::new();
405+
while let Some(next) = args.peek_n(0) {
406+
let element = if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = next {
407+
args.next().expect("already peeked");
408+
ConcatMetaVarExprElem::Literal(lit.clone())
409+
} else {
410+
let is_var = try_eat_dollar(&mut args);
411+
let ident = args.expect_ident_or_underscore()?.clone();
412+
413+
if is_var {
414+
ConcatMetaVarExprElem::Var(ident)
415+
} else {
416+
ConcatMetaVarExprElem::Ident(ident)
417+
}
418+
};
419+
elements.push(element);
420+
if args.peek_n(0).is_some() {
421+
args.expect_comma()?;
422+
}
423+
}
424+
if elements.len() < 2 {
425+
return Err(());
426+
}
427+
Op::Concat { elements: elements.into_boxed_slice(), span: func.span }
428+
}
387429
_ => return Err(()),
388430
};
389431

@@ -414,3 +456,11 @@ fn try_eat_comma(src: &mut TtIter<'_, Span>) -> bool {
414456
}
415457
false
416458
}
459+
460+
fn try_eat_dollar(src: &mut TtIter<'_, Span>) -> bool {
461+
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. }))) = src.peek_n(0) {
462+
let _ = src.next();
463+
return true;
464+
}
465+
false
466+
}

0 commit comments

Comments
 (0)