Include files in vfs on demand

Author: HKalbasi

crates/hir-expand/src/builtin_fn_macro.rs

@@ -582,19 +582,28 @@ fn include_expand(
 }
 
 fn include_bytes_expand(
-    _db: &dyn AstDatabase,
-    _arg_id: MacroCallId,
+    db: &dyn AstDatabase,
+    arg_id: MacroCallId,
     tt: &tt::Subtree,
 ) -> ExpandResult<ExpandedEager> {
-    if let Err(e) = parse_string(tt) {
-        return ExpandResult::only_err(e);
-    }
+    let path = match parse_string(tt) {
+        Ok(it) => it,
+        Err(e) => return ExpandResult::only_err(e),
+    };
+
+    let file_id = match relative_file(db, arg_id, &path, true) {
+        Ok(file_id) => file_id,
+        Err(_) => {
+            return ExpandResult::ok(ExpandedEager::new(quote!(&[0u8; _])));
+        }
+    };
+
+    let text = db.file_text(file_id);
 
-    // FIXME: actually read the file here if the user asked for macro expansion
     let res = tt::Subtree {
         delimiter: None,
         token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
-            text: r#"b"""#.into(),
+            text: format!("b{text:?}").into(),
             id: tt::TokenId::unspecified(),
         }))],
     };
@@ -611,10 +620,6 @@ fn include_str_expand(
         Err(e) => return ExpandResult::only_err(e),
     };
 
-    // FIXME: we're not able to read excluded files (which is most of them because
-    // it's unusual to `include_str!` a Rust file), but we can return an empty string.
-    // Ideally, we'd be able to offer a precise expansion if the user asks for macro
-    // expansion.
     let file_id = match relative_file(db, arg_id, &path, true) {
         Ok(file_id) => file_id,
         Err(_) => {

crates/hir-expand/src/quote.rs

@@ -105,6 +105,7 @@ macro_rules! __quote {
     ( < ) => {$crate::__quote!(@PUNCT '<')};
     ( > ) => {$crate::__quote!(@PUNCT '>')};
     ( ! ) => {$crate::__quote!(@PUNCT '!')};
+    ( _ ) => {$crate::__quote!(@PUNCT '_')};
 
     ( $first:tt $($tail:tt)+ ) => {
         {
@@ -188,6 +189,7 @@ macro_rules! impl_to_to_tokentrees {
 }
 
 impl_to_to_tokentrees! {
+    u8 => self { tt::Literal{text: format!("{self}u8").into(), id: tt::TokenId::unspecified()} };
     u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
     usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
     i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };

crates/rust-analyzer/src/cli/load_cargo.rs

@@ -53,7 +53,7 @@ pub fn load_workspace(
 ) -> Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> {
     let (sender, receiver) = unbounded();
     let mut vfs = vfs::Vfs::default();
-    let mut loader = {
+    let loader = {
         let loader =
             vfs_notify::NotifyHandle::spawn(Box::new(move |msg| sender.send(msg).unwrap()));
         Box::new(loader)

crates/rust-analyzer/src/global_state.rs

@@ -52,7 +52,7 @@ pub(crate) struct GlobalState {
     sender: Sender<lsp_server::Message>,
     req_queue: ReqQueue,
     pub(crate) task_pool: Handle<TaskPool<Task>, Receiver<Task>>,
-    pub(crate) loader: Handle<Box<dyn vfs::loader::Handle>, Receiver<vfs::loader::Message>>,
+    pub(crate) loader: Handle<Arc<dyn vfs::loader::Handle>, Receiver<vfs::loader::Message>>,
     pub(crate) config: Arc<Config>,
     pub(crate) analysis_host: AnalysisHost,
     pub(crate) diagnostics: DiagnosticCollection,
@@ -123,11 +123,15 @@ impl std::panic::UnwindSafe for GlobalStateSnapshot {}
 
 impl GlobalState {
     pub(crate) fn new(sender: Sender<lsp_server::Message>, config: Config) -> GlobalState {
+        let vfs = Arc::new(RwLock::new((vfs::Vfs::default(), NoHashHashMap::default())));
         let loader = {
             let (sender, receiver) = unbounded::<vfs::loader::Message>();
             let handle: vfs_notify::NotifyHandle =
                 vfs::loader::Handle::spawn(Box::new(move |msg| sender.send(msg).unwrap()));
-            let handle = Box::new(handle) as Box<dyn vfs::loader::Handle>;
+            let (vfs, _) = &mut *vfs.write();
+            let handle = Arc::new(handle);
+            vfs.handle = Some(handle.clone());
+            let handle = handle as Arc<dyn vfs::loader::Handle>;
             Handle { handle, receiver }
         };
 
@@ -159,7 +163,7 @@ impl GlobalState {
             flycheck_sender,
             flycheck_receiver,
 
-            vfs: Arc::new(RwLock::new((vfs::Vfs::default(), NoHashHashMap::default()))),
+            vfs,
             vfs_config_version: 0,
             vfs_progress_config_version: 0,
             vfs_progress_n_total: 0,

crates/rust-analyzer/tests/slow-tests/main.rs

@@ -16,7 +16,12 @@ mod support;
 mod testdir;
 mod tidy;
 
-use std::{collections::HashMap, path::PathBuf, time::Instant};
+use std::{
+    collections::HashMap,
+    path::PathBuf,
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 use lsp_types::{
     notification::DidOpenTextDocument,
@@ -684,6 +689,78 @@ version = \"0.0.0\"
     );
 }
 
+#[test]
+fn include_non_rs_file() {
+    if skip_slow_tests() {
+        return;
+    }
+
+    let server = Project::with_fixture(
+        r###"
+//- /Cargo.toml
+[package]
+name = "foo"
+version = "0.0.0"
+
+//- /src/hello.non_rs
+fn hello_world() -> &'static str {
+    "hello world"
+}
+//- /src/some_bytes.txt
+12345
+//- /src/main.rs
+#[rustc_builtin_macro] macro_rules! include {}
+#[rustc_builtin_macro] macro_rules! include_bytes {}
+#[rustc_builtin_macro] macro_rules! include_str {}
+
+include!("hello.non_rs");
+
+fn main() {
+    let my_source = include_bytes!("main.rs");
+    let some_bytes = include_bytes!("some_bytes.txt");
+    let some_str = hello_world();
+    let some_other_bytes = include_bytes!("non_existent.txt");
+    const SOME_STRING: &str = include_str!("some_bytes.txt");
+}
+"###,
+    )
+    .with_config(serde_json::json!({
+        "cargo": {
+            "buildScripts": {
+                "enable": true
+            },
+            "noSysroot": true,
+        }
+    }))
+    .server()
+    .wait_until_workspace_is_loaded();
+
+    // For some reason, `wait_until_workspace_is_loaded` doesn't wait for loading non rs
+    // files, so here we sleep to make sure test will pass
+    sleep(Duration::from_secs(1));
+
+    let checker = |line, hover: &str| {
+        let res = server.send_request::<HoverRequest>(HoverParams {
+            text_document_position_params: TextDocumentPositionParams::new(
+                server.doc_id("src/main.rs"),
+                Position::new(line, 10),
+            ),
+            work_done_progress_params: Default::default(),
+        });
+        let res = res.to_string();
+        assert!(
+            res.contains(hover),
+            "{hover} not found in hover of line {line}.\nHover text is:\n{res}"
+        );
+    };
+
+    checker(7, "&[u8; 454]");
+    checker(8, "&[u8; 6]");
+    checker(9, "&str");
+    checker(10, "&[u8; _]");
+    checker(11, r#"\"12345\\n\""#);
+}
+
 #[test]
 fn out_dirs_check() {
     if skip_slow_tests() {

crates/vfs-notify/src/lib.rs

@@ -9,7 +9,10 @@
 
 #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)]
 
-use std::fs;
+use std::{
+    fs,
+    panic::{RefUnwindSafe, UnwindSafe},
+};
 
 use crossbeam_channel::{never, select, unbounded, Receiver, Sender};
 use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
@@ -24,10 +27,14 @@ pub struct NotifyHandle {
     _thread: jod_thread::JoinHandle,
 }
 
+impl UnwindSafe for NotifyHandle {}
+impl RefUnwindSafe for NotifyHandle {}
+
 #[derive(Debug)]
 enum Message {
     Config(loader::Config),
     Invalidate(AbsPathBuf),
+    Subscribe(AbsPathBuf),
 }
 
 impl loader::Handle for NotifyHandle {
@@ -41,15 +48,19 @@ impl loader::Handle for NotifyHandle {
         NotifyHandle { sender, _thread: thread }
     }
 
-    fn set_config(&mut self, config: loader::Config) {
+    fn set_config(&self, config: loader::Config) {
         self.sender.send(Message::Config(config)).unwrap();
     }
 
-    fn invalidate(&mut self, path: AbsPathBuf) {
+    fn subscribe(&self, path: AbsPathBuf) {
+        self.sender.send(Message::Subscribe(path)).unwrap();
+    }
+
+    fn invalidate(&self, path: AbsPathBuf) {
         self.sender.send(Message::Invalidate(path)).unwrap();
     }
 
-    fn load_sync(&mut self, path: &AbsPath) -> Option<Vec<u8>> {
+    fn load_sync(&self, path: &AbsPath) -> Option<Vec<u8>> {
         read(path)
     }
 }
@@ -126,6 +137,12 @@ impl NotifyActor {
                         let files = vec![(path, contents)];
                         self.send(loader::Message::Loaded { files });
                     }
+                    Message::Subscribe(path) => {
+                        let contents = read(path.as_path());
+                        let files = vec![(path.clone(), contents)];
+                        self.send(loader::Message::Loaded { files });
+                        self.watched_entries.push(loader::Entry::Files(vec![path]));
+                    }
                 },
                 Event::NotifyEvent(event) => {
                     if let Some(event) = log_notify_error(event) {

crates/vfs/src/file_set.rs

@@ -2,21 +2,34 @@
 //!
 //! Files which do not belong to any explicitly configured `FileSet` belong to
 //! the default `FileSet`.
-use std::fmt;
+use std::{
+    fmt,
+    panic::{RefUnwindSafe, UnwindSafe},
+    sync::Arc,
+};
 
 use fst::{IntoStreamer, Streamer};
 use rustc_hash::FxHashMap;
 use stdx::hash::NoHashHashMap;
 
-use crate::{AnchoredPath, FileId, Vfs, VfsPath};
+use crate::{loader::Handle, AnchoredPath, FileId, Vfs, VfsPath};
 
 /// A set of [`VfsPath`]s identified by [`FileId`]s.
-#[derive(Default, Clone, Eq, PartialEq)]
+#[derive(Default, Clone)]
 pub struct FileSet {
     files: FxHashMap<VfsPath, FileId>,
     paths: NoHashHashMap<FileId, VfsPath>,
+    handle: Option<Arc<dyn Handle + Sync + Send + UnwindSafe + RefUnwindSafe>>,
 }
 
+impl PartialEq for FileSet {
+    fn eq(&self, other: &Self) -> bool {
+        self.files == other.files && self.paths == other.paths
+    }
+}
+
+impl Eq for FileSet {}
+
 impl FileSet {
     /// Returns the number of stored paths.
     pub fn len(&self) -> usize {
@@ -31,7 +44,15 @@ impl FileSet {
         let mut base = self.paths[&path.anchor].clone();
         base.pop();
         let path = base.join(path.path)?;
-        self.files.get(&path).copied()
+        let answer = self.files.get(&path);
+        if answer.is_none() {
+            if let Some(handle) = &self.handle {
+                if let Some(x) = path.as_path() {
+                    handle.subscribe(x.to_owned());
+                }
+            }
+        }
+        answer.copied()
     }
 
     /// Get the id corresponding to `path` if it exists in the set.
@@ -110,7 +131,9 @@ impl FileSetConfig {
     /// Creates a new [`FileSet`] for every set of prefixes in `self`.
     pub fn partition(&self, vfs: &Vfs) -> Vec<FileSet> {
         let mut scratch_space = Vec::new();
-        let mut res = vec![FileSet::default(); self.len()];
+        let mut file_set = FileSet::default();
+        file_set.handle = vfs.handle.clone();
+        let mut res = vec![file_set; self.len()];
         for (file_id, path) in vfs.iter() {
             let root = self.classify(path, &mut scratch_space);
             res[root].insert(file_id, path.clone());

crates/vfs/src/lib.rs

@@ -46,14 +46,19 @@ pub mod loader;
 mod path_interner;
 mod vfs_path;
 
-use std::{fmt, mem};
+use std::{
+    fmt, mem,
+    panic::{RefUnwindSafe, UnwindSafe},
+    sync::Arc,
+};
 
 use crate::path_interner::PathInterner;
 
 pub use crate::{
     anchored_path::{AnchoredPath, AnchoredPathBuf},
     vfs_path::VfsPath,
 };
+use loader::Handle;
 pub use paths::{AbsPath, AbsPathBuf};
 
 /// Handle to a file in [`Vfs`]
@@ -77,6 +82,7 @@ pub struct Vfs {
     interner: PathInterner,
     data: Vec<Option<Vec<u8>>>,
     changes: Vec<ChangedFile>,
+    pub handle: Option<Arc<dyn Handle + Sync + Send + UnwindSafe + RefUnwindSafe>>,
 }
 
 /// Changed file in the [`Vfs`].

crates/vfs/src/loader.rs

@@ -64,14 +64,17 @@ pub trait Handle: fmt::Debug {
         Self: Sized;
 
     /// Set this handle's configuration.
-    fn set_config(&mut self, config: Config);
+    fn set_config(&self, config: Config);
 
     /// The file's content at `path` has been modified, and should be reloaded.
-    fn invalidate(&mut self, path: AbsPathBuf);
+    fn invalidate(&self, path: AbsPathBuf);
 
     /// Load the content of the given file, returning [`None`] if it does not
     /// exists.
-    fn load_sync(&mut self, path: &AbsPath) -> Option<Vec<u8>>;
+    fn load_sync(&self, path: &AbsPath) -> Option<Vec<u8>>;
+
+    /// Add a file to watch list
+    fn subscribe(&self, path: AbsPathBuf);
 }
 
 impl Entry {