Skip to main content

rustc_macros/
symbols.rs

1//! Proc macro which builds the Symbol table
2//!
3//! # Debugging
4//!
5//! Since this proc-macro does some non-trivial work, debugging it is important.
6//! This proc-macro can be invoked as an ordinary unit test, like so:
7//!
8//! ```bash
9//! cd compiler/rustc_macros
10//! cargo test symbols::test_symbols -- --nocapture
11//! ```
12//!
13//! This unit test finds the `symbols!` invocation in `compiler/rustc_span/src/symbol.rs`
14//! and runs it. It verifies that the output token stream can be parsed as valid module
15//! items and that no errors were produced.
16//!
17//! You can also view the generated code by using `cargo expand`:
18//!
19//! ```bash
20//! cargo install cargo-expand          # this is necessary only once
21//! cd compiler/rustc_span
22//! # The specific version number in CFG_RELEASE doesn't matter.
23//! # The output is large.
24//! CFG_RELEASE="0.0.0" cargo +nightly expand > /tmp/rustc_span.rs
25//! ```
26
27use indexmap::IndexMap;
28use proc_macro2::{Span, TokenStream};
29use quote::quote;
30use syn::parse::{Parse, ParseStream, Result};
31use syn::punctuated::Punctuated;
32use syn::{Expr, Ident, Lit, LitStr, Macro, Token, braced};
33
34#[cfg(test)]
35mod tests;
36
37mod kw {
38    syn::custom_keyword!(Keywords);
39    syn::custom_keyword!(Symbols);
40}
41
42struct Keyword {
43    name: Ident,
44    value: LitStr,
45}
46
47impl Parse for Keyword {
48    fn parse(input: ParseStream<'_>) -> Result<Self> {
49        let name = input.parse()?;
50        input.parse::<Token![:]>()?;
51        let value = input.parse()?;
52
53        Ok(Keyword { name, value })
54    }
55}
56
57struct Symbol {
58    name: Ident,
59    value: Value,
60}
61
62enum Value {
63    SameAsName,
64    String(LitStr),
65    Env(LitStr, Macro),
66    Unsupported(Expr),
67}
68
69impl Parse for Symbol {
70    fn parse(input: ParseStream<'_>) -> Result<Self> {
71        let name = input.parse()?;
72        let colon_token: Option<Token![:]> = input.parse()?;
73        let value = if colon_token.is_some() { input.parse()? } else { Value::SameAsName };
74
75        Ok(Symbol { name, value })
76    }
77}
78
79impl Parse for Value {
80    fn parse(input: ParseStream<'_>) -> Result<Self> {
81        let expr: Expr = input.parse()?;
82        match &expr {
83            Expr::Lit(expr) => {
84                if let Lit::Str(lit) = &expr.lit {
85                    return Ok(Value::String(lit.clone()));
86                }
87            }
88            Expr::Macro(expr) => {
89                if expr.mac.path.is_ident("env")
90                    && let Ok(lit) = expr.mac.parse_body()
91                {
92                    return Ok(Value::Env(lit, expr.mac.clone()));
93                }
94            }
95            _ => {}
96        }
97        Ok(Value::Unsupported(expr))
98    }
99}
100
101struct Input {
102    keywords: Punctuated<Keyword, Token![,]>,
103    symbols: Punctuated<Symbol, Token![,]>,
104}
105
106impl Parse for Input {
107    fn parse(input: ParseStream<'_>) -> Result<Self> {
108        input.parse::<kw::Keywords>()?;
109        let content;
110        braced!(content in input);
111        let keywords = Punctuated::parse_terminated(&content)?;
112
113        input.parse::<kw::Symbols>()?;
114        let content;
115        braced!(content in input);
116        let symbols = Punctuated::parse_terminated(&content)?;
117
118        Ok(Input { keywords, symbols })
119    }
120}
121
122#[derive(Default)]
123struct Errors {
124    list: Vec<syn::Error>,
125}
126
127impl Errors {
128    fn error(&mut self, span: Span, message: String) {
129        self.list.push(syn::Error::new(span, message));
130    }
131}
132
133pub(super) fn symbols(input: TokenStream) -> TokenStream {
134    let (mut output, errors) = symbols_with_errors(input);
135
136    // If we generated any errors, then report them as compiler_error!() macro calls.
137    // This lets the errors point back to the most relevant span. It also allows us
138    // to report as many errors as we can during a single run.
139    output.extend(errors.into_iter().map(|e| e.to_compile_error()));
140
141    output
142}
143
144struct Predefined {
145    idx: u32,
146    span_of_name: Span,
147}
148
149struct Entries {
150    map: IndexMap<String, Predefined>,
151}
152
153impl Entries {
154    fn with_capacity(capacity: usize) -> Self {
155        Entries { map: IndexMap::with_capacity(capacity) }
156    }
157
158    fn insert(&mut self, span: Span, s: &str, errors: &mut Errors) -> u32 {
159        if let Some(prev) = self.map.get(s) {
160            errors.error(span, format!("Symbol `{s}` is duplicated"));
161            errors.error(prev.span_of_name, "location of previous definition".to_string());
162            prev.idx
163        } else {
164            let idx = self.len();
165            self.map.insert(s.to_string(), Predefined { idx, span_of_name: span });
166            idx
167        }
168    }
169
170    fn len(&self) -> u32 {
171        u32::try_from(self.map.len()).expect("way too many symbols")
172    }
173}
174
175fn symbols_with_errors(input: TokenStream) -> (TokenStream, Vec<syn::Error>) {
176    let mut errors = Errors::default();
177
178    let input: Input = match syn::parse2(input) {
179        Ok(input) => input,
180        Err(e) => {
181            // This allows us to display errors at the proper span, while minimizing
182            // unrelated errors caused by bailing out (and not generating code).
183            errors.list.push(e);
184            Input { keywords: Default::default(), symbols: Default::default() }
185        }
186    };
187
188    let mut keyword_stream = quote! {};
189    let mut symbols_stream = quote! {};
190    let mut prefill_stream = quote! {};
191    let prefill_ints = 0..=9;
192    let prefill_letters = ('A'..='Z').chain('a'..='z');
193    let mut entries = Entries::with_capacity(
194        input.keywords.len()
195            + input.symbols.len()
196            + prefill_ints.clone().count()
197            + prefill_letters.clone().count(),
198    );
199
200    // Generate the listed keywords.
201    for keyword in input.keywords.iter() {
202        let name = &keyword.name;
203        let value = &keyword.value;
204        let value_string = value.value();
205        let idx = entries.insert(keyword.name.span(), &value_string, &mut errors);
206        prefill_stream.extend(quote! {
207            #value,
208        });
209        keyword_stream.extend(quote! {
210            pub const #name: Symbol = Symbol::new(#idx);
211        });
212    }
213
214    // Generate the listed symbols.
215    for symbol in input.symbols.iter() {
216        let name = &symbol.name;
217
218        let value = match &symbol.value {
219            Value::SameAsName => name.to_string(),
220            Value::String(lit) => lit.value(),
221            Value::Env(..) => continue, // in another loop below
222            Value::Unsupported(expr) => {
223                errors.list.push(syn::Error::new_spanned(
224                    expr,
225                    concat!(
226                        "unsupported expression for symbol value; implement support for this in ",
227                        file!(),
228                    ),
229                ));
230                continue;
231            }
232        };
233        let idx = entries.insert(symbol.name.span(), &value, &mut errors);
234
235        prefill_stream.extend(quote! {
236            #value,
237        });
238        symbols_stream.extend(quote! {
239            pub const #name: Symbol = Symbol::new(#idx);
240        });
241    }
242
243    // Generate symbols for ascii letters and digits
244    for s in prefill_ints.map(|n| n.to_string()).chain(prefill_letters.map(|c| c.to_string())) {
245        entries.insert(Span::call_site(), &s, &mut errors);
246        prefill_stream.extend(quote! {
247            #s,
248        });
249    }
250
251    // Symbols whose value comes from an environment variable. It's allowed for
252    // these to have the same value as another symbol.
253    for symbol in &input.symbols {
254        let (env_var, expr) = match &symbol.value {
255            Value::Env(lit, expr) => (lit, expr),
256            Value::SameAsName | Value::String(_) | Value::Unsupported(_) => continue,
257        };
258
259        if !proc_macro::is_available() {
260            errors.error(
261                Span::call_site(),
262                "proc_macro::tracked_env is not available in unit test".to_owned(),
263            );
264            break;
265        }
266
267        let tracked_env = proc_macro::tracked::env_var(env_var.value());
268
269        let value = match tracked_env {
270            Ok(value) => value,
271            Err(err) => {
272                errors.list.push(syn::Error::new_spanned(expr, err));
273                continue;
274            }
275        };
276
277        let idx = if let Some(prev) = entries.map.get(&value) {
278            prev.idx
279        } else {
280            prefill_stream.extend(quote! {
281                #value,
282            });
283            entries.insert(symbol.name.span(), &value, &mut errors)
284        };
285
286        let name = &symbol.name;
287        symbols_stream.extend(quote! {
288            pub const #name: Symbol = Symbol::new(#idx);
289        });
290    }
291
292    let symbol_digits_base = entries.map["0"].idx;
293    let symbol_uppercase_letters_base = entries.map["A"].idx;
294    let symbol_lowercase_letters_base = entries.map["a"].idx;
295    let predefined_symbols_count = entries.len();
296    let output = quote! {
297        const SYMBOL_DIGITS_BASE: u32 = #symbol_digits_base;
298        const SYMBOL_UPPERCASE_LETTERS_BASE: u32 = #symbol_uppercase_letters_base;
299        const SYMBOL_LOWERCASE_LETTERS_BASE: u32 = #symbol_lowercase_letters_base;
300
301        /// The number of predefined symbols; this is the first index for
302        /// extra pre-interned symbols in an Interner created via
303        /// [`Interner::with_extra_symbols`].
304        pub const PREDEFINED_SYMBOLS_COUNT: u32 = #predefined_symbols_count;
305
306        #[doc(hidden)]
307        #[allow(non_upper_case_globals)]
308        mod kw_generated {
309            use super::Symbol;
310            #keyword_stream
311        }
312
313        #[allow(non_upper_case_globals)]
314        #[doc(hidden)]
315        pub mod sym_generated {
316            use super::Symbol;
317            #symbols_stream
318        }
319
320        impl Interner {
321            /// Creates an `Interner` with the predefined symbols from the `symbols!` macro and
322            /// any extra symbols provided by external drivers such as Clippy
323            pub(crate) fn with_extra_symbols(extra_symbols: &[&'static str]) -> Self {
324                Interner::prefill(
325                    &[#prefill_stream],
326                    extra_symbols,
327                )
328            }
329        }
330    };
331
332    (output, errors.list)
333}