From e321cb05a295ca20423bda4a14f60d71124b9b94 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 10 Oct 2025 22:42:22 +0200 Subject: [PATCH 1/8] WIP: implement `proc_macro` standalone backend --- library/proc_macro/src/bridge/client.rs | 73 +++++-- library/proc_macro/src/bridge/mod.rs | 1 + library/proc_macro/src/bridge/server.rs | 16 +- library/proc_macro/src/bridge/standalone.rs | 231 ++++++++++++++++++++ library/proc_macro/src/lib.rs | 22 ++ 5 files changed, 322 insertions(+), 21 deletions(-) create mode 100644 library/proc_macro/src/bridge/standalone.rs diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index bdaa865a998d6..2c2f0b8f3db23 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -5,6 +5,9 @@ use std::marker::PhantomData; use std::sync::atomic::AtomicU32; use super::*; +use crate::StandaloneLevel; +use crate::bridge::server::{Dispatcher, DispatcherTrait}; +use crate::bridge::standalone::NoRustc; macro_rules! define_client_handles { ( @@ -141,7 +144,10 @@ macro_rules! define_client_side { api_tags::Method::$name(api_tags::$name::$method).encode(&mut buf, &mut ()); $($arg.encode(&mut buf, &mut ());)* - buf = bridge.dispatch.call(buf); + buf = match &mut bridge.dispatch { + DispatchWay::Closure(f) => f.call(buf), + DispatchWay::Directly(disp) => disp.dispatch(buf), + }; let r = Result::<_, PanicMessage>::decode(&mut &buf[..], &mut ()); @@ -155,13 +161,18 @@ macro_rules! define_client_side { } with_api!(self, self, define_client_side); +enum DispatchWay<'a> { + Closure(closure::Closure<'a, Buffer, Buffer>), + Directly(Dispatcher), +} + struct Bridge<'a> { /// Reusable buffer (only `clear`-ed, never shrunk), primarily /// used for making requests. cached_buffer: Buffer, /// Server-side function that the client uses to make requests. - dispatch: closure::Closure<'a, Buffer, Buffer>, + dispatch: DispatchWay<'a>, /// Provided globals for this macro expansion. globals: ExpnGlobals, @@ -173,12 +184,33 @@ impl<'a> !Sync for Bridge<'a> {} #[allow(unsafe_code)] mod state { use std::cell::{Cell, RefCell}; + use std::marker::PhantomData; use std::ptr; use super::Bridge; + use crate::StandaloneLevel; + use crate::bridge::buffer::Buffer; + use crate::bridge::client::{COUNTERS, DispatchWay}; + use crate::bridge::server::{Dispatcher, HandleStore, MarkedTypes}; + use crate::bridge::{ExpnGlobals, Marked, standalone}; thread_local! { static BRIDGE_STATE: Cell<*const ()> = const { Cell::new(ptr::null()) }; + static STANDALONE: RefCell> = RefCell::new(standalone_bridge()); + pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::Never) }; + } + + fn standalone_bridge() -> Bridge<'static> { + let mut store = HandleStore::new(&COUNTERS); + let id = store.Span.alloc(Marked { value: standalone::Span::DUMMY, _marker: PhantomData }); + let dummy = super::Span { handle: id }; + let dispatcher = + Dispatcher { handle_store: store, server: MarkedTypes(standalone::NoRustc) }; + Bridge { + cached_buffer: Buffer::new(), + dispatch: DispatchWay::Directly(dispatcher), + globals: ExpnGlobals { call_site: dummy, def_site: dummy, mixed_site: dummy }, + } } pub(super) fn set<'bridge, R>(state: &RefCell>, f: impl FnOnce() -> R) -> R { @@ -199,16 +231,23 @@ mod state { pub(super) fn with( f: impl for<'bridge> FnOnce(Option<&RefCell>>) -> R, ) -> R { - let state = BRIDGE_STATE.get(); - // SAFETY: the only place where the pointer is set is in `set`. It puts - // back the previous value after the inner call has returned, so we know - // that as long as the pointer is not null, it came from a reference to - // a `RefCell` that outlasts the call to this function. Since `f` - // works the same for any lifetime of the bridge, including the actual - // one, we can lie here and say that the lifetime is `'static` without - // anyone noticing. - let bridge = unsafe { state.cast::>>().as_ref() }; - f(bridge) + let level = USE_STANDALONE.get(); + if level == StandaloneLevel::Always + || (level == StandaloneLevel::FallbackOnly && BRIDGE_STATE.get().is_null()) + { + STANDALONE.with(|bridge| f(Some(bridge))) + } else { + let state = BRIDGE_STATE.get(); + // SAFETY: the only place where the pointer is set is in `set`. It puts + // back the previous value after the inner call has returned, so we know + // that as long as the pointer is not null, it came from a reference to + // a `RefCell` that outlasts the call to this function. Since `f` + // works the same for any lifetime of the bridge, including the actual + // one, we can lie here and say that the lifetime is `'static` without + // anyone noticing. + let bridge = unsafe { state.cast::>>().as_ref() }; + f(bridge) + } } } @@ -228,6 +267,10 @@ pub(crate) fn is_available() -> bool { state::with(|s| s.is_some()) } +pub(crate) fn enable_standalone(level: StandaloneLevel) { + state::USE_STANDALONE.set(level); +} + /// A client-side RPC entry-point, which may be using a different `proc_macro` /// from the one used by the server, but can be invoked compatibly. /// @@ -292,7 +335,11 @@ fn run_client Decode<'a, 's, ()>, R: Encode<()>>( let (globals, input) = <(ExpnGlobals, A)>::decode(reader, &mut ()); // Put the buffer we used for input back in the `Bridge` for requests. - let state = RefCell::new(Bridge { cached_buffer: buf.take(), dispatch, globals }); + let state = RefCell::new(Bridge { + cached_buffer: buf.take(), + dispatch: DispatchWay::Closure(dispatch), + globals, + }); let output = state::set(&state, || f(input)); diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index b0ee9c0cc3027..4f5fe2a08b8b7 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -138,6 +138,7 @@ mod rpc; mod selfless_reify; #[forbid(unsafe_code)] pub mod server; +pub(crate) mod standalone; #[allow(unsafe_code)] mod symbol; diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index e9ef26c07f24f..051151b6c7c69 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -12,12 +12,12 @@ macro_rules! define_server_handles { ) => { #[allow(non_snake_case)] pub(super) struct HandleStore { - $($oty: handle::OwnedStore,)* - $($ity: handle::InternedStore,)* + $(pub(super) $oty: handle::OwnedStore,)* + $(pub(super) $ity: handle::InternedStore,)* } impl HandleStore { - fn new(handle_counters: &'static client::HandleCounters) -> Self { + pub(super) fn new(handle_counters: &'static client::HandleCounters) -> Self { HandleStore { $($oty: handle::OwnedStore::new(&handle_counters.$oty),)* $($ity: handle::InternedStore::new(&handle_counters.$ity),)* @@ -119,7 +119,7 @@ macro_rules! declare_server_traits { } with_api!(Self, self_, declare_server_traits); -pub(super) struct MarkedTypes(S); +pub(super) struct MarkedTypes(pub(super) S); impl Server for MarkedTypes { fn globals(&mut self) -> ExpnGlobals { @@ -150,9 +150,9 @@ macro_rules! define_mark_types_impls { } with_api!(Self, self_, define_mark_types_impls); -struct Dispatcher { - handle_store: HandleStore, - server: S, +pub(super) struct Dispatcher { + pub(super) handle_store: HandleStore>, + pub(super) server: MarkedTypes, } macro_rules! define_dispatcher_impl { @@ -167,7 +167,7 @@ macro_rules! define_dispatcher_impl { fn dispatch(&mut self, buf: Buffer) -> Buffer; } - impl DispatcherTrait for Dispatcher> { + impl DispatcherTrait for Dispatcher { $(type $name = as Types>::$name;)* fn dispatch(&mut self, mut buf: Buffer) -> Buffer { diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs new file mode 100644 index 0000000000000..a7b9589e0fe12 --- /dev/null +++ b/library/proc_macro/src/bridge/standalone.rs @@ -0,0 +1,231 @@ +#![allow(unused_variables)] +use std::cell::RefCell; +use std::ops::{Bound, Range}; + +use crate::bridge::client::Symbol; +use crate::bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree, server}; + +pub struct NoRustc; + +impl server::Span for NoRustc { + fn debug(&mut self, span: Self::Span) -> String { + format!("{} bytes({}..{})", span.hi - span.lo, span.lo, span.hi) + } + + fn parent(&mut self, span: Self::Span) -> Option { + todo!() + } + + fn source(&mut self, span: Self::Span) -> Self::Span { + todo!() + } + + fn byte_range(&mut self, span: Self::Span) -> Range { + todo!() + } + + fn start(&mut self, span: Self::Span) -> Self::Span { + Span { lo: span.lo, hi: span.lo } + } + + fn end(&mut self, span: Self::Span) -> Self::Span { + Span { lo: span.hi, hi: span.hi } + } + + fn line(&mut self, span: Self::Span) -> usize { + todo!() + } + + fn column(&mut self, span: Self::Span) -> usize { + todo!() + } + + fn file(&mut self, span: Self::Span) -> String { + todo!() + } + + fn local_file(&mut self, span: Self::Span) -> Option { + todo!() + } + + fn join(&mut self, span: Self::Span, other: Self::Span) -> Option { + todo!() + } + + fn subspan( + &mut self, + span: Self::Span, + start: Bound, + end: Bound, + ) -> Option { + let length = span.hi as usize - span.lo as usize; + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo.checked_add(1)?, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi.checked_add(1)?, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::MAX as usize + || end > u32::MAX as usize + || (u32::MAX - start as u32) < span.lo + || (u32::MAX - end as u32) < span.lo + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo + start as u32; + let new_hi = span.lo + end as u32; + Some(Span { lo: new_lo, hi: new_hi }) + } + + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + todo!() + } + + fn source_text(&mut self, span: Self::Span) -> Option { + todo!() + } + + fn save_span(&mut self, span: Self::Span) -> usize { + SAVED_SPANS.with_borrow_mut(|spans| { + let idx = spans.len(); + spans.push(span); + idx + }) + } + + fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { + SAVED_SPANS.with_borrow(|spans| spans[id]) + } +} + +thread_local! { + static SAVED_SPANS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +impl server::FreeFunctions for NoRustc { + fn injected_env_var(&mut self, var: &str) -> Option { + todo!() + } + + fn track_env_var(&mut self, _var: &str, _value: Option<&str>) {} + + fn track_path(&mut self, _path: &str) {} + + fn literal_from_str(&mut self, s: &str) -> Result, ()> { + todo!() + } + + fn emit_diagnostic(&mut self, diagnostic: Diagnostic) { + panic!("cannot emit diagnostic in standalone mode"); + } +} + +impl server::TokenStream for NoRustc { + fn is_empty(&mut self, tokens: &Self::TokenStream) -> bool { + tokens.0.is_empty() + } + + fn expand_expr(&mut self, tokens: &Self::TokenStream) -> Result { + todo!() + } + + fn from_str(&mut self, src: &str) -> Self::TokenStream { + todo!() + } + + fn to_string(&mut self, tokens: &Self::TokenStream) -> String { + todo!() + } + + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + TokenStream(vec![tree]) + } + + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut base = base.unwrap_or_else(TokenStream::new); + base.0.extend(trees); + base + } + + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut base = base.unwrap_or_else(TokenStream::new); + for stream in streams { + base = self.concat_trees(Some(base), stream.0); + } + base + } + + fn into_trees( + &mut self, + tokens: Self::TokenStream, + ) -> Vec> { + tokens.0 + } +} + +pub struct FreeFunctions; +#[derive(Clone, Default)] +pub struct TokenStream(Vec>); +impl TokenStream { + pub fn new() -> Self { + Self(Vec::new()) + } +} + +#[derive(Hash, PartialEq, Eq, Clone, Copy)] +pub struct Span { + pub lo: u32, + pub hi: u32, +} +impl Span { + pub const DUMMY: Self = Self { lo: 0, hi: 0 }; +} + +impl server::Types for NoRustc { + type FreeFunctions = FreeFunctions; + type TokenStream = TokenStream; + type Span = Span; + type Symbol = Symbol; +} + +impl server::Server for NoRustc { + fn globals(&mut self) -> ExpnGlobals { + ExpnGlobals { def_site: Span::DUMMY, call_site: Span::DUMMY, mixed_site: Span::DUMMY } + } + + fn intern_symbol(ident: &str) -> Self::Symbol { + Symbol::new(ident) + } + + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + symbol.with(f); + } +} + +impl server::Symbol for NoRustc { + fn normalize_and_validate_ident(&mut self, string: &str) -> Result { + todo!() + } +} diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 1aa6064633c3b..d8ffced199d3b 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -89,6 +89,28 @@ pub fn is_available() -> bool { bridge::client::is_available() } +/// Controls the extent to which the new standalone backend is used. +/// +/// When this will be stabilized, the default level will change from +/// `Never` to `FallbackOnly`. +#[unstable(feature = "proc_macro_standalone", issue = "130856")] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum StandaloneLevel { + /// The standalone implementation is never used. This is the default. + Never, + /// The standalone implementation is only used outside of procedural macros. + FallbackOnly, + /// The standalone implementation is always used, even in procedural macros. + Always, +} + +/// Enables the new experimental standalone backend, which allows calling the +/// functions in this crate outside of procedural macros. +#[unstable(feature = "proc_macro_standalone", issue = "130856")] +pub fn enable_standalone(level: StandaloneLevel) { + bridge::client::enable_standalone(level); +} + /// The main type provided by this crate, representing an abstract stream of /// tokens, or, more specifically, a sequence of token trees. /// The type provides interfaces for iterating over those token trees and, conversely, From cacb1f3d4f8dc4c29cee4246599026e5e674e961 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Tue, 14 Oct 2025 22:50:35 +0200 Subject: [PATCH 2/8] WIP: to_string and more --- library/proc_macro/src/bridge/standalone.rs | 82 +++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index a7b9589e0fe12..0295cea31931f 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -1,9 +1,12 @@ #![allow(unused_variables)] +#![warn(warnings)] use std::cell::RefCell; use std::ops::{Bound, Range}; +use crate::Delimiter; use crate::bridge::client::Symbol; -use crate::bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree, server}; +use crate::bridge::fxhash::FxHashMap; +use crate::bridge::{Diagnostic, ExpnGlobals, LitKind, Literal, TokenTree, server}; pub struct NoRustc; @@ -21,7 +24,7 @@ impl server::Span for NoRustc { } fn byte_range(&mut self, span: Self::Span) -> Range { - todo!() + span.lo as usize..span.hi as usize } fn start(&mut self, span: Self::Span) -> Self::Span { @@ -111,14 +114,18 @@ impl server::Span for NoRustc { thread_local! { static SAVED_SPANS: RefCell> = const { RefCell::new(Vec::new()) }; + static TRACKED_ENV_VARS: RefCell>> = RefCell::new(FxHashMap::default()); } impl server::FreeFunctions for NoRustc { fn injected_env_var(&mut self, var: &str) -> Option { - todo!() + TRACKED_ENV_VARS.with_borrow(|vars| vars.get(var)?.clone()) } - fn track_env_var(&mut self, _var: &str, _value: Option<&str>) {} + fn track_env_var(&mut self, var: &str, value: Option<&str>) { + TRACKED_ENV_VARS + .with_borrow_mut(|vars| vars.insert(var.to_string(), value.map(ToString::to_string))); + } fn track_path(&mut self, _path: &str) {} @@ -145,7 +152,72 @@ impl server::TokenStream for NoRustc { } fn to_string(&mut self, tokens: &Self::TokenStream) -> String { - todo!() + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + } + + let mut s = String::new(); + for tree in &tokens.0 { + s.push_str(&match tree { + TokenTree::Group(group) => { + let inner = if let Some(stream) = &group.stream { + self.to_string(stream) + } else { + String::new() + }; + match group.delimiter { + Delimiter::Parenthesis => format!("({inner})"), + Delimiter::Brace => format!("{{{inner}}}"), + Delimiter::Bracket => format!("[{inner}]"), + Delimiter::None => inner, + } + } + TokenTree::Ident(ident) => { + if ident.is_raw { + format!("r#{}", ident.sym) + } else { + ident.sym.to_string() + } + } + TokenTree::Literal(lit) => { + let inner = if let Some(suffix) = lit.suffix { + format!("{}{suffix}", lit.symbol) + } else { + lit.symbol.to_string() + }; + match lit.kind { + LitKind::Byte => todo!(), + LitKind::ByteStr => format!("b\"{inner}\""), + LitKind::ByteStrRaw(raw) => { + format!("br{0}\"{inner}\"{0}", get_hashes_str(raw)) + } + LitKind::CStr => format!("c\"{inner}\""), + LitKind::CStrRaw(raw) => { + format!("cr{0}\"{inner}\"{0}", get_hashes_str(raw)) + } + LitKind::Char => format!("'{inner}'"), + LitKind::ErrWithGuar => unreachable!(), + LitKind::Float | LitKind::Integer => inner, + LitKind::Str => format!("\"{inner}\""), + LitKind::StrRaw(raw) => format!("r{0}\"{inner}\"{0}", get_hashes_str(raw)), + } + } + TokenTree::Punct(punct) => (punct.ch as char).to_string(), + }); + s.push(' '); + } + s.pop(); + s } fn from_token_tree( From 6cca2415d3743ac965c95ff840b378118053a9a3 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Wed, 15 Oct 2025 11:45:25 +0200 Subject: [PATCH 3/8] extend to_string space handling --- library/proc_macro/src/bridge/standalone.rs | 47 ++++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index 0295cea31931f..717ce6642630d 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -167,8 +167,12 @@ impl server::TokenStream for NoRustc { } let mut s = String::new(); - for tree in &tokens.0 { - s.push_str(&match tree { + let mut last = String::new(); + let mut second_last = String::new(); + + for (idx, tree) in tokens.0.iter().enumerate() { + let mut space = true; + let new_part = match tree { TokenTree::Group(group) => { let inner = if let Some(stream) = &group.stream { self.to_string(stream) @@ -177,7 +181,13 @@ impl server::TokenStream for NoRustc { }; match group.delimiter { Delimiter::Parenthesis => format!("({inner})"), - Delimiter::Brace => format!("{{{inner}}}"), + Delimiter::Brace => { + if inner.is_empty() { + "{ }".to_string() + } else { + format!("{{ {inner} }}") + } + } Delimiter::Bracket => format!("[{inner}]"), Delimiter::None => inner, } @@ -212,11 +222,34 @@ impl server::TokenStream for NoRustc { LitKind::StrRaw(raw) => format!("r{0}\"{inner}\"{0}", get_hashes_str(raw)), } } - TokenTree::Punct(punct) => (punct.ch as char).to_string(), - }); - s.push(' '); + TokenTree::Punct(punct) => { + let c = punct.ch as char; + if c == '\'' { + space = false; + } + c.to_string() + } + }; + + const NON_SEPARATABLE_TOKENS: &[(char, char)] = &[(':', ':'), ('-', '>')]; + + for (first, second) in NON_SEPARATABLE_TOKENS { + if second_last == first.to_string() && last == second.to_string() && new_part != ":" + { + s.pop(); // pop ' ' + s.pop(); // pop `second` + s.pop(); // pop ' ' + s.push(*second); + s.push(' '); + } + } + s.push_str(&new_part); + second_last = last; + last = new_part; + if space && idx + 1 != tokens.0.len() { + s.push(' '); + } } - s.pop(); s } From cd57156316a3fecde87f90e6316d8728d0a48f0c Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 17 Oct 2025 23:32:08 +0200 Subject: [PATCH 4/8] reuse stuff --- library/proc_macro/src/bridge/client.rs | 7 +- library/proc_macro/src/bridge/standalone.rs | 83 ++++++++++++++++++--- library/proc_macro/src/lib.rs | 21 +++--- 3 files changed, 91 insertions(+), 20 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 2c2f0b8f3db23..9776cdab5e344 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::marker::PhantomData; +use std::num::NonZero; use std::sync::atomic::AtomicU32; use super::*; @@ -110,6 +111,10 @@ impl Clone for TokenStream { } impl Span { + pub(crate) fn dummy() -> Span { + Span { handle: NonZero::new(1).unwrap() } + } + pub(crate) fn def_site() -> Span { Bridge::with(|bridge| bridge.globals.def_site) } @@ -197,7 +202,7 @@ mod state { thread_local! { static BRIDGE_STATE: Cell<*const ()> = const { Cell::new(ptr::null()) }; static STANDALONE: RefCell> = RefCell::new(standalone_bridge()); - pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::Never) }; + pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::FallbackOnly) }; } fn standalone_bridge() -> Bridge<'static> { diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index 717ce6642630d..3779507cf43e3 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -3,10 +3,10 @@ use std::cell::RefCell; use std::ops::{Bound, Range}; -use crate::Delimiter; +use crate::{Delimiter, LEGAL_PUNCT_CHARS}; use crate::bridge::client::Symbol; use crate::bridge::fxhash::FxHashMap; -use crate::bridge::{Diagnostic, ExpnGlobals, LitKind, Literal, TokenTree, server}; +use crate::bridge::{server, DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Literal, Punct, TokenTree}; pub struct NoRustc; @@ -130,7 +130,22 @@ impl server::FreeFunctions for NoRustc { fn track_path(&mut self, _path: &str) {} fn literal_from_str(&mut self, s: &str) -> Result, ()> { - todo!() + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return Err(()); + }; + br""; + cr""; + + match first { + 'b' => todo!(), + 'c' => todo!(), + 'r' => todo!(), + '0'..='9' | '-' => todo!(), + '\'' => todo!(), + '"' => todo!(), + _ => Err(()) + } } fn emit_diagnostic(&mut self, diagnostic: Diagnostic) { @@ -148,10 +163,53 @@ impl server::TokenStream for NoRustc { } fn from_str(&mut self, src: &str) -> Self::TokenStream { - todo!() + /// Returns the delimiter, and whether it is the opening form. + fn char_to_delim(c: char) -> Option<(Delimiter, bool)> { + Some(match c { + '(' => (Delimiter::Parenthesis, true), + ')' => (Delimiter::Parenthesis, false), + '{' => (Delimiter::Brace, true), + '}' => (Delimiter::Brace, false), + '[' => (Delimiter::Bracket, true), + ']' => (Delimiter::Bracket, false), + _ => return None, + }) + } + + let mut unfinished_streams = vec![TokenStream::new()]; + let mut unclosed_delimiters = Vec::new(); + let mut current_ident = String::new(); + for c in src.chars() { + if let Some((delim, is_opening)) = char_to_delim(c) { + if is_opening { + unclosed_delimiters.push(delim); + unfinished_streams.push(TokenStream::new()); + } else if unclosed_delimiters.pop() == Some(delim) { + let group = TokenTree::<_, _, Symbol>::Group(Group { + delimiter: delim, + stream: unfinished_streams.pop(), + span: DelimSpan::from_single(Span::DUMMY) + }); + unfinished_streams.last_mut().unwrap().0.push(group); + } else { + panic!("cannot parse string into token stream") + } + } else if LEGAL_PUNCT_CHARS.contains(&c) { + unfinished_streams.last_mut().unwrap().0.push(TokenTree::Punct(Punct { + ch: c as u8, + joint: false, // TODO + span: Span::DUMMY, + })); + } + match c { + _ => todo!(), + } + } + unfinished_streams[0].clone() } fn to_string(&mut self, tokens: &Self::TokenStream) -> String { + /* /// Returns a string containing exactly `num` '#' characters. /// Uses a 256-character source string literal which is always safe to /// index with a `u8` index. @@ -164,7 +222,7 @@ impl server::TokenStream for NoRustc { "; const _: () = assert!(HASHES.len() == 256); &HASHES[..num as usize] - } + }*/ let mut s = String::new(); let mut last = String::new(); @@ -200,13 +258,20 @@ impl server::TokenStream for NoRustc { } } TokenTree::Literal(lit) => { - let inner = if let Some(suffix) = lit.suffix { + let respanned = Literal { + kind: lit.kind, + symbol: lit.symbol, + suffix: lit.suffix, + span: super::client::Span::dummy(), + }; + crate::Literal(respanned).to_string() + /*let inner = if let Some(suffix) = lit.suffix { format!("{}{suffix}", lit.symbol) } else { lit.symbol.to_string() }; match lit.kind { - LitKind::Byte => todo!(), + LitKind::Byte => format!("b'{inner}'"), LitKind::ByteStr => format!("b\"{inner}\""), LitKind::ByteStrRaw(raw) => { format!("br{0}\"{inner}\"{0}", get_hashes_str(raw)) @@ -220,7 +285,7 @@ impl server::TokenStream for NoRustc { LitKind::Float | LitKind::Integer => inner, LitKind::Str => format!("\"{inner}\""), LitKind::StrRaw(raw) => format!("r{0}\"{inner}\"{0}", get_hashes_str(raw)), - } + }*/ } TokenTree::Punct(punct) => { let c = punct.ch as char; @@ -231,7 +296,7 @@ impl server::TokenStream for NoRustc { } }; - const NON_SEPARATABLE_TOKENS: &[(char, char)] = &[(':', ':'), ('-', '>')]; + const NON_SEPARATABLE_TOKENS: &[(char, char)] = &[(':', ':'), ('-', '>'), ('=', '>')]; for (first, second) in NON_SEPARATABLE_TOKENS { if second_last == first.to_string() && last == second.to_string() && new_part != ":" diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index d8ffced199d3b..cb87e27695b05 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -969,6 +969,11 @@ pub enum Spacing { Alone, } +pub(crate) const LEGAL_PUNCT_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', +]; + impl Punct { /// Creates a new `Punct` from the given character and spacing. /// The `ch` argument must be a valid punctuation character permitted by the language, @@ -978,11 +983,7 @@ impl Punct { /// which can be further configured with the `set_span` method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(ch: char, spacing: Spacing) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { + if !LEGAL_PUNCT_CHARS.contains(&ch) { panic!("unsupported character `{:?}`", ch); } Punct(bridge::Punct { @@ -1178,7 +1179,7 @@ macro_rules! unsuffixed_int_literals { /// specified on this token, meaning that invocations like /// `Literal::i8_unsuffixed(1)` are equivalent to /// `Literal::u32_unsuffixed(1)`. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// Literals created through this method have the `Span::call_site()` @@ -1241,7 +1242,7 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1266,7 +1267,7 @@ impl Literal { /// specified is the preceding part of the token and `f32` is the suffix of /// the token. This token will always be inferred to be an `f32` in the /// compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1286,7 +1287,7 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1311,7 +1312,7 @@ impl Literal { /// specified is the preceding part of the token and `f64` is the suffix of /// the token. This token will always be inferred to be an `f64` in the /// compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics From 3a9bdaddfe0f1f5dd074996d86420427e4d40de1 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Sat, 18 Oct 2025 13:41:28 +0200 Subject: [PATCH 5/8] unit spans --- library/proc_macro/src/bridge/client.rs | 2 +- library/proc_macro/src/bridge/standalone.rs | 126 ++++++++------------ library/proc_macro/src/lib.rs | 4 +- 3 files changed, 50 insertions(+), 82 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 9776cdab5e344..62ed501b2e480 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -207,7 +207,7 @@ mod state { fn standalone_bridge() -> Bridge<'static> { let mut store = HandleStore::new(&COUNTERS); - let id = store.Span.alloc(Marked { value: standalone::Span::DUMMY, _marker: PhantomData }); + let id = store.Span.alloc(Marked { value: standalone::Span, _marker: PhantomData }); let dummy = super::Span { handle: id }; let dispatcher = Dispatcher { handle_store: store, server: MarkedTypes(standalone::NoRustc) }; diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index 3779507cf43e3..8e55ffac7b8a5 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -1,119 +1,93 @@ -#![allow(unused_variables)] #![warn(warnings)] -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::ops::{Bound, Range}; -use crate::{Delimiter, LEGAL_PUNCT_CHARS}; use crate::bridge::client::Symbol; use crate::bridge::fxhash::FxHashMap; -use crate::bridge::{server, DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Literal, Punct, TokenTree}; +use crate::bridge::{DelimSpan, Diagnostic, ExpnGlobals, Group, Literal, Punct, TokenTree, server}; +use crate::{Delimiter, LEGAL_PUNCT_CHARS}; pub struct NoRustc; impl server::Span for NoRustc { - fn debug(&mut self, span: Self::Span) -> String { - format!("{} bytes({}..{})", span.hi - span.lo, span.lo, span.hi) + fn debug(&mut self, _: Self::Span) -> String { + "Span".to_string() } - fn parent(&mut self, span: Self::Span) -> Option { - todo!() + fn parent(&mut self, _: Self::Span) -> Option { + None } - fn source(&mut self, span: Self::Span) -> Self::Span { - todo!() + fn source(&mut self, _: Self::Span) -> Self::Span { + Span } - fn byte_range(&mut self, span: Self::Span) -> Range { - span.lo as usize..span.hi as usize + fn byte_range(&mut self, _: Self::Span) -> Range { + todo!() } - fn start(&mut self, span: Self::Span) -> Self::Span { - Span { lo: span.lo, hi: span.lo } + fn start(&mut self, _: Self::Span) -> Self::Span { + Span } - fn end(&mut self, span: Self::Span) -> Self::Span { - Span { lo: span.hi, hi: span.hi } + fn end(&mut self, _: Self::Span) -> Self::Span { + Span } - fn line(&mut self, span: Self::Span) -> usize { + fn line(&mut self, _: Self::Span) -> usize { todo!() } - fn column(&mut self, span: Self::Span) -> usize { + fn column(&mut self, _: Self::Span) -> usize { todo!() } - fn file(&mut self, span: Self::Span) -> String { - todo!() + fn file(&mut self, _: Self::Span) -> String { + "".to_string() } - fn local_file(&mut self, span: Self::Span) -> Option { - todo!() + fn local_file(&mut self, _: Self::Span) -> Option { + None } - fn join(&mut self, span: Self::Span, other: Self::Span) -> Option { - todo!() + fn join(&mut self, _: Self::Span, _: Self::Span) -> Option { + Some(Span) } fn subspan( &mut self, - span: Self::Span, - start: Bound, - end: Bound, + _: Self::Span, + _start: Bound, + _end: Bound, ) -> Option { - let length = span.hi as usize - span.lo as usize; - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo.checked_add(1)?, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi.checked_add(1)?, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::MAX as usize - || end > u32::MAX as usize - || (u32::MAX - start as u32) < span.lo - || (u32::MAX - end as u32) < span.lo - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo + start as u32; - let new_hi = span.lo + end as u32; - Some(Span { lo: new_lo, hi: new_hi }) + Some(Span) } - fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { - todo!() + fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span { + Span } - fn source_text(&mut self, span: Self::Span) -> Option { - todo!() + fn source_text(&mut self, _: Self::Span) -> Option { + None } - fn save_span(&mut self, span: Self::Span) -> usize { - SAVED_SPANS.with_borrow_mut(|spans| { - let idx = spans.len(); - spans.push(span); - idx - }) + fn save_span(&mut self, _: Self::Span) -> usize { + let n = SAVED_SPAN_COUNT.get(); + SAVED_SPAN_COUNT.set(n + 1); + n } fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { - SAVED_SPANS.with_borrow(|spans| spans[id]) + if id < SAVED_SPAN_COUNT.get() { + Span + } else { + panic!("recovered span index out of bounds"); + } } } thread_local! { - static SAVED_SPANS: RefCell> = const { RefCell::new(Vec::new()) }; + static SAVED_SPAN_COUNT: Cell = const { Cell::new(0) }; static TRACKED_ENV_VARS: RefCell>> = RefCell::new(FxHashMap::default()); } @@ -144,11 +118,11 @@ impl server::FreeFunctions for NoRustc { '0'..='9' | '-' => todo!(), '\'' => todo!(), '"' => todo!(), - _ => Err(()) + _ => Err(()), } } - fn emit_diagnostic(&mut self, diagnostic: Diagnostic) { + fn emit_diagnostic(&mut self, _: Diagnostic) { panic!("cannot emit diagnostic in standalone mode"); } } @@ -188,7 +162,7 @@ impl server::TokenStream for NoRustc { let group = TokenTree::<_, _, Symbol>::Group(Group { delimiter: delim, stream: unfinished_streams.pop(), - span: DelimSpan::from_single(Span::DUMMY) + span: DelimSpan::from_single(Span), }); unfinished_streams.last_mut().unwrap().0.push(group); } else { @@ -198,7 +172,7 @@ impl server::TokenStream for NoRustc { unfinished_streams.last_mut().unwrap().0.push(TokenTree::Punct(Punct { ch: c as u8, joint: false, // TODO - span: Span::DUMMY, + span: Span, })); } match c { @@ -365,13 +339,7 @@ impl TokenStream { } #[derive(Hash, PartialEq, Eq, Clone, Copy)] -pub struct Span { - pub lo: u32, - pub hi: u32, -} -impl Span { - pub const DUMMY: Self = Self { lo: 0, hi: 0 }; -} +pub struct Span; impl server::Types for NoRustc { type FreeFunctions = FreeFunctions; @@ -382,7 +350,7 @@ impl server::Types for NoRustc { impl server::Server for NoRustc { fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { def_site: Span::DUMMY, call_site: Span::DUMMY, mixed_site: Span::DUMMY } + ExpnGlobals { def_site: Span, call_site: Span, mixed_site: Span } } fn intern_symbol(ident: &str) -> Self::Symbol { diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index cb87e27695b05..9b0da7d482379 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -970,8 +970,8 @@ pub enum Spacing { } pub(crate) const LEGAL_PUNCT_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', ':', '#', + '$', '?', '\'', ]; impl Punct { From 1339d6ee69e56e2a49ebf650b644f02576aeee1e Mon Sep 17 00:00:00 2001 From: cyrgani Date: Sun, 19 Oct 2025 21:38:55 +0200 Subject: [PATCH 6/8] wip: some parsing --- library/proc_macro/src/bridge/client.rs | 2 +- library/proc_macro/src/bridge/standalone.rs | 116 +++++++++++++++++--- 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 62ed501b2e480..40ca1e221b15b 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -202,7 +202,7 @@ mod state { thread_local! { static BRIDGE_STATE: Cell<*const ()> = const { Cell::new(ptr::null()) }; static STANDALONE: RefCell> = RefCell::new(standalone_bridge()); - pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::FallbackOnly) }; + pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::Never) }; } fn standalone_bridge() -> Bridge<'static> { diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index 8e55ffac7b8a5..80ec3ff4bdef1 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -4,7 +4,9 @@ use std::ops::{Bound, Range}; use crate::bridge::client::Symbol; use crate::bridge::fxhash::FxHashMap; -use crate::bridge::{DelimSpan, Diagnostic, ExpnGlobals, Group, Literal, Punct, TokenTree, server}; +use crate::bridge::{ + DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Literal, Punct, TokenTree, server, +}; use crate::{Delimiter, LEGAL_PUNCT_CHARS}; pub struct NoRustc; @@ -23,7 +25,7 @@ impl server::Span for NoRustc { } fn byte_range(&mut self, _: Self::Span) -> Range { - todo!() + 0..0 } fn start(&mut self, _: Self::Span) -> Self::Span { @@ -35,11 +37,11 @@ impl server::Span for NoRustc { } fn line(&mut self, _: Self::Span) -> usize { - todo!() + 1 } fn column(&mut self, _: Self::Span) -> usize { - todo!() + 1 } fn file(&mut self, _: Self::Span) -> String { @@ -91,6 +93,82 @@ thread_local! { static TRACKED_ENV_VARS: RefCell>> = RefCell::new(FxHashMap::default()); } +fn parse_maybe_raw_str( + mut s: &str, + raw_variant: fn(u8) -> LitKind, + regular_variant: LitKind, +) -> Result, ()> { + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + } + let mut hash_count = None; + + if s.starts_with('r') { + s = s.strip_prefix('r').unwrap(); + let mut h = 0; + for c in s.chars() { + if c == '#' { + if h == u8::MAX { + return Err(()); + } + h += 1; + } else { + break; + } + } + hash_count = Some(h); + let hashes = get_hashes_str(h); + s = s.strip_prefix(hashes).unwrap(); + s = s.strip_suffix(hashes).ok_or(())?; + } + let sym = parse_plain_str(s)?; + + Ok(make_literal( + if let Some(h) = hash_count { raw_variant(h) } else { regular_variant }, + sym, + None, + )) +} + +fn parse_char(s: &str) -> Result, ()> { + if s.chars().count() == 1 { + Ok(make_literal(LitKind::Char, Symbol::new(s), None)) + } else { + Err(()) + } +} + +fn parse_plain_str(mut s: &str) -> Result { + s = s.strip_prefix("\"").ok_or(())?.strip_suffix('\"').ok_or(())?; + Ok(Symbol::new(s)) +} + +fn parse_numeral(s: &str) -> Result, ()> { + /*if s.ends_with("f16") + || s.ends_with("f32") + || s.ends_with("f64") + || s.ends_with("f128") + { + Literal { kind: todo!(), symbol: todo!(), suffix: todo!(), span: Span }; + } + todo!()*/ + todo!() +} + +fn make_literal(kind: LitKind, symbol: Symbol, suffix: Option) -> Literal { + Literal { kind, symbol, suffix, span: Span } +} + impl server::FreeFunctions for NoRustc { fn injected_env_var(&mut self, var: &str) -> Option { TRACKED_ENV_VARS.with_borrow(|vars| vars.get(var)?.clone()) @@ -108,16 +186,24 @@ impl server::FreeFunctions for NoRustc { let Some(first) = chars.next() else { return Err(()); }; - br""; - cr""; + let rest = &s[1..]; match first { - 'b' => todo!(), - 'c' => todo!(), - 'r' => todo!(), - '0'..='9' | '-' => todo!(), - '\'' => todo!(), - '"' => todo!(), + 'b' => { + if chars.next() == Some('\'') { + parse_char(rest).map(|mut lit| { + lit.kind = LitKind::Byte; + lit + }) + } else { + parse_maybe_raw_str(rest, LitKind::ByteStrRaw, LitKind::ByteStr) + } + } + 'c' => parse_maybe_raw_str(rest, LitKind::CStrRaw, LitKind::CStr), + 'r' => parse_maybe_raw_str(rest, LitKind::StrRaw, LitKind::Str), + '0'..='9' | '-' => parse_numeral(s), + '\'' => parse_char(s), + '"' => Ok(make_literal(LitKind::Str, parse_plain_str(s)?, None)), _ => Err(()), } } @@ -132,8 +218,8 @@ impl server::TokenStream for NoRustc { tokens.0.is_empty() } - fn expand_expr(&mut self, tokens: &Self::TokenStream) -> Result { - todo!() + fn expand_expr(&mut self, _tokens: &Self::TokenStream) -> Result { + todo!("`expand_expr` is not yet supported in the standalone backend") } fn from_str(&mut self, src: &str) -> Self::TokenStream { @@ -171,7 +257,7 @@ impl server::TokenStream for NoRustc { } else if LEGAL_PUNCT_CHARS.contains(&c) { unfinished_streams.last_mut().unwrap().0.push(TokenTree::Punct(Punct { ch: c as u8, - joint: false, // TODO + joint: todo!(), span: Span, })); } From 58e89175b4bd8114ccd2b57d105cd5d7a7eda599 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Tue, 21 Oct 2025 15:44:06 +0200 Subject: [PATCH 7/8] wip: more parsing --- library/proc_macro/src/bridge/standalone.rs | 111 ++++++++++++++------ 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs index 80ec3ff4bdef1..6f2696552de08 100644 --- a/library/proc_macro/src/bridge/standalone.rs +++ b/library/proc_macro/src/bridge/standalone.rs @@ -1,14 +1,17 @@ #![warn(warnings)] use std::cell::{Cell, RefCell}; -use std::ops::{Bound, Range}; +use std::ops::{Bound, Range, RangeBounds}; use crate::bridge::client::Symbol; use crate::bridge::fxhash::FxHashMap; use crate::bridge::{ - DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Literal, Punct, TokenTree, server, + self, DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Punct, TokenTree, server, }; use crate::{Delimiter, LEGAL_PUNCT_CHARS}; +type Result = std::result::Result; +type Literal = bridge::Literal; + pub struct NoRustc; impl server::Span for NoRustc { @@ -97,7 +100,7 @@ fn parse_maybe_raw_str( mut s: &str, raw_variant: fn(u8) -> LitKind, regular_variant: LitKind, -) -> Result, ()> { +) -> Result { /// Returns a string containing exactly `num` '#' characters. /// Uses a 256-character source string literal which is always safe to /// index with a `u8` index. @@ -133,40 +136,86 @@ fn parse_maybe_raw_str( } let sym = parse_plain_str(s)?; - Ok(make_literal( - if let Some(h) = hash_count { raw_variant(h) } else { regular_variant }, - sym, - None, - )) + Ok(make_literal(if let Some(h) = hash_count { raw_variant(h) } else { regular_variant }, sym)) } -fn parse_char(s: &str) -> Result, ()> { - if s.chars().count() == 1 { - Ok(make_literal(LitKind::Char, Symbol::new(s), None)) - } else { - Err(()) - } +fn parse_char(s: &str) -> Result { + if s.chars().count() == 1 { Ok(make_literal(LitKind::Char, Symbol::new(s))) } else { Err(()) } } -fn parse_plain_str(mut s: &str) -> Result { +fn parse_plain_str(mut s: &str) -> Result { s = s.strip_prefix("\"").ok_or(())?.strip_suffix('\"').ok_or(())?; Ok(Symbol::new(s)) } -fn parse_numeral(s: &str) -> Result, ()> { - /*if s.ends_with("f16") - || s.ends_with("f32") - || s.ends_with("f64") - || s.ends_with("f128") +const INT_SUFFIXES: &[&str] = + &["u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64", "u128", "i128"]; +const FLOAT_SUFFIXES: &[&str] = &["f16", "f32", "f64", "f128"]; + +fn parse_numeral(mut s: &str) -> Result { + for suffix in INT_SUFFIXES { + if s.ends_with(suffix) { + return parse_integer(s); + } + } + let is_negative = s.starts_with('-'); + let non_negative = s.strip_prefix('-').unwrap(); + if non_negative.starts_with("0b") + || non_negative.starts_with("0o") + || non_negative.starts_with("0x") { - Literal { kind: todo!(), symbol: todo!(), suffix: todo!(), span: Span }; + return parse_integer(s); + } + let (s, suffix) = strip_number_suffix(s, FLOAT_SUFFIXES); + + Ok(Literal { kind: LitKind::Float, symbol: todo!(), suffix, span: Span }) +} + +fn parse_integer(mut s: &str) -> Result { + let is_negative = s.starts_with('-'); + s = s.strip_prefix('-').unwrap(); + + let (s, valid_chars) = if let Some(s) = s.strip_prefix("0b") { + (s, '0'..='1') + } else if let Some(s) = s.strip_prefix("0o") { + (s, '0'..='7') + } else if let Some(s) = s.strip_prefix("0x") { + (s, '0'..='F') + } else { + (s, '0'..='9') + }; + + let (s, suffix) = strip_number_suffix(s, INT_SUFFIXES); + + let mut any_found = false; + for c in s.chars() { + if c == '_' { + continue; + } + if valid_chars.contains(&c) { + any_found = true; + continue; + } + return Err(()); + } + if !any_found { + return Err(()); + } + + Ok(Literal { kind: LitKind::Integer, symbol: Symbol::new(s), suffix, span: Span }) +} + +fn strip_number_suffix<'a>(s: &'a str, suffixes: &[&str]) -> (&'a str, Option) { + for suf in suffixes { + if let Some(new_s) = s.strip_suffix(suf) { + return (new_s, Some(Symbol::new(suf))); + } } - todo!()*/ - todo!() + (s, None) } -fn make_literal(kind: LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { kind, symbol, suffix, span: Span } +fn make_literal(kind: LitKind, symbol: Symbol) -> Literal { + Literal { kind, symbol, suffix: None, span: Span } } impl server::FreeFunctions for NoRustc { @@ -181,7 +230,7 @@ impl server::FreeFunctions for NoRustc { fn track_path(&mut self, _path: &str) {} - fn literal_from_str(&mut self, s: &str) -> Result, ()> { + fn literal_from_str(&mut self, s: &str) -> Result { let mut chars = s.chars(); let Some(first) = chars.next() else { return Err(()); @@ -203,7 +252,7 @@ impl server::FreeFunctions for NoRustc { 'r' => parse_maybe_raw_str(rest, LitKind::StrRaw, LitKind::Str), '0'..='9' | '-' => parse_numeral(s), '\'' => parse_char(s), - '"' => Ok(make_literal(LitKind::Str, parse_plain_str(s)?, None)), + '"' => Ok(make_literal(LitKind::Str, parse_plain_str(s)?)), _ => Err(()), } } @@ -218,7 +267,7 @@ impl server::TokenStream for NoRustc { tokens.0.is_empty() } - fn expand_expr(&mut self, _tokens: &Self::TokenStream) -> Result { + fn expand_expr(&mut self, _tokens: &Self::TokenStream) -> Result { todo!("`expand_expr` is not yet supported in the standalone backend") } @@ -318,7 +367,7 @@ impl server::TokenStream for NoRustc { } } TokenTree::Literal(lit) => { - let respanned = Literal { + let respanned = bridge::Literal { kind: lit.kind, symbol: lit.symbol, suffix: lit.suffix, @@ -419,7 +468,7 @@ pub struct FreeFunctions; #[derive(Clone, Default)] pub struct TokenStream(Vec>); impl TokenStream { - pub fn new() -> Self { + fn new() -> Self { Self(Vec::new()) } } @@ -449,7 +498,7 @@ impl server::Server for NoRustc { } impl server::Symbol for NoRustc { - fn normalize_and_validate_ident(&mut self, string: &str) -> Result { + fn normalize_and_validate_ident(&mut self, string: &str) -> Result { todo!() } } From 7267ab070c203899bd2ecf351a0dd2ffcbd86e06 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Wed, 22 Oct 2025 16:09:14 +0200 Subject: [PATCH 8/8] hide `StandaloneLevel` --- library/proc_macro/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 9b0da7d482379..5591e38a6c0c8 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -95,20 +95,25 @@ pub fn is_available() -> bool { /// `Never` to `FallbackOnly`. #[unstable(feature = "proc_macro_standalone", issue = "130856")] #[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum StandaloneLevel { +enum StandaloneLevel { /// The standalone implementation is never used. This is the default. Never, /// The standalone implementation is only used outside of procedural macros. FallbackOnly, /// The standalone implementation is always used, even in procedural macros. + /// + /// This does not actually work and should be removed before merging. Always, } /// Enables the new experimental standalone backend, which allows calling the /// functions in this crate outside of procedural macros. +/// +/// When stabilizing this feature, this function will be removed and all programs +/// will have the fallback activated automatically. #[unstable(feature = "proc_macro_standalone", issue = "130856")] -pub fn enable_standalone(level: StandaloneLevel) { - bridge::client::enable_standalone(level); +pub fn enable_standalone() { + bridge::client::enable_standalone(StandaloneLevel::FallbackOnly); } /// The main type provided by this crate, representing an abstract stream of