[hyperlight_component_util] Support passing host resources to guest functions

Previously, the guest could acquire and use host resources through
host function calls, but the original call from the host into the
guest could not pass in owned or borrowed host resources.  This commit
fixes in a number of bugs in order to make that possible, and adds
rudimentary tests to witguest.

Author: syntactically

Justfile

@@ -140,7 +140,7 @@ like-ci config=default-target hypervisor="kvm":
     {{ if os() == "linux" { "just test-rust-tracing " + config + " " + if hypervisor == "mshv" { "mshv2" } else if hypervisor == "mshv3" { "mshv3" } else { "kvm" } } else { "" } }}
 
     @# Run benchmarks
-    just bench-ci main {{config}} {{ if hypervisor == "mshv" { "mshv2" } else if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }}
+    just bench-ci main {{ if hypervisor == "mshv" { "mshv2" } else if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }}
 
 # runs all tests
 test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features) (test-doc target features)

flake.nix

@@ -59,6 +59,11 @@
               channel = "stable";
               sha256 = "sha256-AJ6LX/Q/Er9kS15bn9iflkUwcgYqRQxiOIL2ToVAXaU=";
             };
+            "1.86" = {
+              date = "2025-04-03";
+              channel = "stable";
+              sha256 = "sha256-X/4ZBHO3iW0fOenQ3foEvscgAPJYl2abspaBThDOukI=";
+            };
           };
 
           rust-platform = makeRustPlatform {

src/hyperlight_component_util/src/emit.rs

@@ -196,6 +196,28 @@ impl Mod {
     }
 }
 
+/// Unlike [`tv::ResolvedTyvar`], which is mostly concerned with free
+/// variables and leaves bound variables alone, this tells us the most
+/// information that we have at codegen time for a top level bound
+/// variable.
+pub enum ResolvedBoundVar<'a> {
+    Definite {
+        /// The final variable offset (relative to s.var_offset) that
+        /// we followed to get to this definite type, used
+        /// occasionally to name things.
+        final_bound_var: u32,
+        /// The actual definite type that this resolved to
+        ty: Defined<'a>,
+    },
+    Resource {
+        /// A resource-type index. Currently a resource-type index is
+        /// the same as the de Bruijn index of the tyvar that
+        /// introduced the resource type, but is never affected by
+        /// e.g. s.var_offset.
+        rtidx: u32,
+    },
+}
+
 /// A whole grab-bag of useful state to have while emitting Rust
 #[derive(Debug)]
 pub struct State<'a, 'b> {
@@ -260,6 +282,8 @@ pub struct State<'a, 'b> {
     /// wasmtime guest emit. When that is refactored to use the host
     /// guest emit, this can go away.
     pub is_wasmtime_guest: bool,
+    /// Are we working on an export or an import of the component type?
+    pub is_export: bool,
 }
 
 /// Create a State with all of its &mut references pointing to
@@ -311,6 +335,7 @@ impl<'a, 'b> State<'a, 'b> {
             root_component_name: None,
             is_guest,
             is_wasmtime_guest,
+            is_export: false,
         }
     }
     pub fn clone<'c>(&'c mut self) -> State<'c, 'b> {
@@ -331,6 +356,7 @@ impl<'a, 'b> State<'a, 'b> {
             root_component_name: self.root_component_name.clone(),
             is_guest: self.is_guest,
             is_wasmtime_guest: self.is_wasmtime_guest,
+            is_export: self.is_export,
         }
     }
     /// Obtain a reference to the [`Mod`] that we are currently
@@ -508,9 +534,17 @@ impl<'a, 'b> State<'a, 'b> {
     }
     /// Add an import/export to [`State::origin`], reflecting that we are now
     /// looking at code underneath it
-    pub fn push_origin<'c>(&'c mut self, is_export: bool, name: &'b str) -> State<'c, 'b> {
+    ///
+    /// origin_was_export differs from s.is_export in that s.is_export
+    /// keeps track of whether the item overall was imported or exported
+    /// from the root component (taking into account positivity), whereas
+    /// origin_was_export just checks if this particular extern_decl was
+    /// imported or exported from its parent instance (and so e.g. an
+    /// export of an instance that is imported by the root component has
+    /// !s.is_export && origin_was_export)
+    pub fn push_origin<'c>(&'c mut self, origin_was_export: bool, name: &'b str) -> State<'c, 'b> {
         let mut s = self.clone();
-        s.origin.push(if is_export {
+        s.origin.push(if origin_was_export {
             ImportExport::Export(name)
         } else {
             ImportExport::Import(name)
@@ -588,15 +622,24 @@ impl<'a, 'b> State<'a, 'b> {
     /// up with a definition, in which case, let's get that, or it
     /// ends up with a resource type, in which case we return the
     /// resource index
-    pub fn resolve_tv(&self, n: u32) -> (u32, Option<Defined<'b>>) {
-        match &self.bound_vars[self.var_offset + n as usize].bound {
+    ///
+    /// Distinct from [`Ctx::resolve_tv`], which is mostly concerned
+    /// with free variables, because this is concerned entirely with
+    /// bound variables.
+    pub fn resolve_bound_var(&self, n: u32) -> ResolvedBoundVar<'b> {
+        let noff = self.var_offset as u32 + n;
+        match &self.bound_vars[noff as usize].bound {
             TypeBound::Eq(Defined::Handleable(Handleable::Var(Tyvar::Bound(nn)))) => {
-                self.resolve_tv(n + 1 + nn)
+                self.resolve_bound_var(n + 1 + nn)
             }
-            TypeBound::Eq(t) => (n, Some(t.clone())),
-            TypeBound::SubResource => (n, None),
+            TypeBound::Eq(t) => ResolvedBoundVar::Definite {
+                final_bound_var: n,
+                ty: t.clone(),
+            },
+            TypeBound::SubResource => ResolvedBoundVar::Resource { rtidx: noff },
         }
     }
+
     /// Construct a namespace path referring to the resource trait for
     /// a resource with the given name
     pub fn resource_trait_path(&self, r: Ident) -> Vec<Ident> {

src/hyperlight_component_util/src/guest.rs

@@ -18,8 +18,9 @@ use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
 
 use crate::emit::{
-    FnName, ResourceItemName, State, WitName, kebab_to_exports_name, kebab_to_fn, kebab_to_getter,
-    kebab_to_imports_name, kebab_to_namespace, kebab_to_type, kebab_to_var, split_wit_name,
+    FnName, ResolvedBoundVar, ResourceItemName, State, WitName, kebab_to_exports_name, kebab_to_fn,
+    kebab_to_getter, kebab_to_imports_name, kebab_to_namespace, kebab_to_type, kebab_to_var,
+    split_wit_name,
 };
 use crate::etypes::{Component, Defined, ExternDecl, ExternDesc, Handleable, Instance, Tyvar};
 use crate::hl::{
@@ -98,10 +99,10 @@ fn emit_import_extern_decl<'a, 'b, 'c>(
         ExternDesc::Type(t) => match t {
             Defined::Handleable(Handleable::Var(Tyvar::Bound(b))) => {
                 // only resources need something emitted
-                let (b, None) = s.resolve_tv(*b) else {
+                let ResolvedBoundVar::Resource { rtidx } = s.resolve_bound_var(*b) else {
                     return quote! {};
                 };
-                let rtid = format_ident!("HostResource{}", s.var_offset + b as usize);
+                let rtid = format_ident!("HostResource{}", rtidx as usize);
                 let path = s.resource_trait_path(kebab_to_type(ed.kebab_name));
                 s.root_mod
                     .r#impl(path, format_ident!("Host"))
@@ -314,6 +315,8 @@ fn emit_component<'a, 'b, 'c>(
 
     s.var_offset = 0;
 
+    s.is_export = true;
+
     let exports = ct
         .instance
         .unqualified

src/hyperlight_component_util/src/hl.rs

@@ -18,8 +18,8 @@ use itertools::Itertools;
 use proc_macro2::{Ident, TokenStream};
 use quote::{format_ident, quote};
 
-use crate::emit::{State, kebab_to_cons, kebab_to_var};
-use crate::etypes::{self, Defined, Handleable, TypeBound, Tyvar, Value};
+use crate::emit::{ResolvedBoundVar, State, kebab_to_cons, kebab_to_var};
+use crate::etypes::{self, Defined, Handleable, Tyvar, Value};
 use crate::rtypes;
 
 /// Construct a string that can be used "on the wire" to identify a
@@ -151,25 +151,16 @@ pub fn emit_hl_unmarshal_toplevel_value(
     }
 }
 
-/// Find the resource index that the given type variable refers to.
-///
-/// Precondition: this type variable does refer to a resource type
-fn resolve_tyvar_to_resource(s: &mut State, v: u32) -> u32 {
-    match s.bound_vars[v as usize].bound {
-        TypeBound::SubResource => v,
-        TypeBound::Eq(Defined::Handleable(Handleable::Var(Tyvar::Bound(vv)))) => {
-            resolve_tyvar_to_resource(s, v + vv + 1)
-        }
-        _ => panic!("impossible: resource var is not resource"),
-    }
-}
 /// Find the resource index that the given Handleable refers to.
 ///
 /// Precondition: this type variable does refer to a resource type
 pub fn resolve_handleable_to_resource(s: &mut State, ht: &Handleable) -> u32 {
     match ht {
         Handleable::Var(Tyvar::Bound(vi)) => {
-            resolve_tyvar_to_resource(s, s.var_offset as u32 + *vi)
+            let ResolvedBoundVar::Resource { rtidx } = s.resolve_bound_var(*vi) else {
+                panic!("impossible: resource var is not resource");
+            };
+            rtidx
         }
         _ => panic!("impossible handleable in type"),
     }
@@ -338,9 +329,29 @@ pub fn emit_hl_unmarshal_value(s: &mut State, id: Ident, vt: &Value) -> TokenStr
             log::debug!("resolved ht to r (2) {:?} {:?}", ht, vi);
             if s.is_guest {
                 let rid = format_ident!("HostResource{}", vi);
-                quote! {
-                    let i = u32::from_ne_bytes(#id[0..4].try_into().unwrap());
-                    (::wasmtime::component::Resource::<#rid>::new_borrow(i), 4)
+                if s.is_wasmtime_guest {
+                    quote! {
+                        let i = u32::from_ne_bytes(#id[0..4].try_into().unwrap());
+                        (::wasmtime::component::Resource::<#rid>::new_borrow(i), 4)
+                    }
+                } else {
+                    // TODO: When we add the Drop impl (#810), we need
+                    // to make sure it does not get called here
+                    //
+                    // If we tried to actually return a reference
+                    // here, rustc would get mad about the temporary
+                    // constructed here not living long enough, so
+                    // instead we return the temporary and construct
+                    // the reference elsewhere. It might be a bit more
+                    // principled to have a separate
+                    // HostResourceXXBorrow struct that implements
+                    // AsRef<HostResourceXX> or something in the
+                    // future...
+                    quote! {
+                        let i = u32::from_ne_bytes(#id[0..4].try_into().unwrap());
+
+                        (#rid { rep: i }, 4)
+                    }
                 }
             } else {
                 let rid = format_ident!("resource{}", vi);
@@ -358,7 +369,11 @@ pub fn emit_hl_unmarshal_value(s: &mut State, id: Ident, vt: &Value) -> TokenStr
             let Some(Tyvar::Bound(n)) = tv else {
                 panic!("impossible tyvar")
             };
-            let (n, Some(Defined::Value(vt))) = s.resolve_tv(*n) else {
+            let ResolvedBoundVar::Definite {
+                final_bound_var: n,
+                ty: Defined::Value(vt),
+            } = s.resolve_bound_var(*n)
+            else {
                 panic!("unresolvable tyvar (2)");
             };
             let vt = vt.clone();
@@ -644,7 +659,9 @@ pub fn emit_hl_marshal_value(s: &mut State, id: Ident, vt: &Value) -> TokenStrea
                 let rid = format_ident!("resource{}", vi);
                 quote! {
                     let i = rts.#rid.len();
-                    rts.#rid.push_back(::hyperlight_common::resource::ResourceEntry::lend(#id));
+                    let (lrg, re) = ::hyperlight_common::resource::ResourceEntry::lend(#id);
+                    to_cleanup.push(Box::new(lrg));
+                    rts.#rid.push_back(re);
                     alloc::vec::Vec::from(u32::to_ne_bytes(i as u32))
                 }
             }
@@ -653,7 +670,11 @@ pub fn emit_hl_marshal_value(s: &mut State, id: Ident, vt: &Value) -> TokenStrea
             let Some(Tyvar::Bound(n)) = tv else {
                 panic!("impossible tyvar")
             };
-            let (n, Some(Defined::Value(vt))) = s.resolve_tv(*n) else {
+            let ResolvedBoundVar::Definite {
+                final_bound_var: n,
+                ty: Defined::Value(vt),
+            } = s.resolve_bound_var(*n)
+            else {
                 panic!("unresolvable tyvar (2)");
             };
             let vt = vt.clone();
@@ -668,7 +689,20 @@ pub fn emit_hl_marshal_value(s: &mut State, id: Ident, vt: &Value) -> TokenStrea
 /// [`crate::rtypes`] module) of the given value type.
 pub fn emit_hl_unmarshal_param(s: &mut State, id: Ident, pt: &Value) -> TokenStream {
     let toks = emit_hl_unmarshal_value(s, id, pt);
-    quote! { { #toks }.0 }
+    // Slight hack to avoid rust complaints about deserialised
+    // resource borrow lifetimes.
+    fn is_borrow(vt: &Value) -> bool {
+        match vt {
+            Value::Borrow(_) => true,
+            Value::Var(_, vt) => is_borrow(vt),
+            _ => false,
+        }
+    }
+    if s.is_guest && !s.is_wasmtime_guest && is_borrow(pt) {
+        quote! { &({ #toks }.0) }
+    } else {
+        quote! { { #toks }.0 }
+    }
 }
 
 /// Emit code to unmarshal the result of a function with result type

src/hyperlight_component_util/src/host.rs

@@ -56,12 +56,20 @@ fn emit_export_extern_decl<'a, 'b, 'c>(
                     let unmarshal = emit_hl_unmarshal_result(s, ret.clone(), &ft.result);
                     quote! {
                         fn #n(&mut self, #(#param_decls),*) -> #result_decl {
+                            let mut to_cleanup = Vec::<Box<dyn Drop>>::new();
+                            let marshalled = {
+                                let mut rts = self.rt.lock().unwrap();
+                                #[allow(clippy::unused_unit)]
+                                (#(#marshal,)*)
+                            };
                             let #ret = ::hyperlight_host::sandbox::Callable::call::<::std::vec::Vec::<u8>>(&mut self.sb,
                                 #hln,
-                                (#(#marshal,)*)
+                                marshalled,
                             );
                             let ::std::result::Result::Ok(#ret) = #ret else { panic!("bad return from guest {:?}", #ret) };
                             #[allow(clippy::unused_unit)]
+                            let mut rts = self.rt.lock().unwrap();
+                            #[allow(clippy::unused_unit)]
                             #unmarshal
                         }
                     }
@@ -333,6 +341,8 @@ fn emit_component<'a, 'b, 'c>(s: &'c mut State<'a, 'b>, wn: WitName, ct: &'c Com
     s.root_component_name = Some((ns.clone(), wn.name));
     s.cur_trait = Some(export_trait.clone());
     s.import_param_var = Some(format_ident!("I"));
+    s.is_export = true;
+
     let exports = ct
         .instance
         .unqualified

src/hyperlight_component_util/src/rtypes.rs

@@ -345,7 +345,11 @@ pub fn emit_value(s: &mut State, vt: &Value) -> TokenStream {
                     }
                 } else {
                     let vr = emit_var_ref(s, tv);
-                    quote! { ::hyperlight_common::resource::BorrowedResourceGuard<#vr> }
+                    if s.is_export {
+                        quote! { &#vr }
+                    } else {
+                        quote! { ::hyperlight_common::resource::BorrowedResourceGuard<#vr> }
+                    }
                 }
             }
         },
@@ -607,16 +611,19 @@ fn emit_type_alias<F: Fn(&mut State) -> TokenStream>(
 
 /// Emit (via returning) a Rust trait item corresponding to this
 /// extern decl
+///
+/// See note on emit.rs push_origin for the difference between
+/// origin_was_export and s.is_export.
 fn emit_extern_decl<'a, 'b, 'c>(
-    is_export: bool,
+    origin_was_export: bool,
     s: &'c mut State<'a, 'b>,
     ed: &'c ExternDecl<'b>,
 ) -> TokenStream {
     log::debug!("  emitting decl {:?}", ed.kebab_name);
     match &ed.desc {
         ExternDesc::CoreModule(_) => panic!("core module (im/ex)ports are not supported"),
         ExternDesc::Func(ft) => {
-            let mut s = s.push_origin(is_export, ed.kebab_name);
+            let mut s = s.push_origin(origin_was_export, ed.kebab_name);
             match kebab_to_fn(ed.kebab_name) {
                 FnName::Plain(n) => {
                     let params = ft
@@ -681,7 +688,7 @@ fn emit_extern_decl<'a, 'b, 'c>(
                 TokenStream::new()
             }
             let edn: &'b str = ed.kebab_name;
-            let mut s: State<'_, 'b> = s.push_origin(is_export, edn);
+            let mut s: State<'_, 'b> = s.push_origin(origin_was_export, edn);
             if let Some((n, bound)) = s.is_var_defn(t) {
                 match bound {
                     TypeBound::Eq(t) => {
@@ -708,7 +715,7 @@ fn emit_extern_decl<'a, 'b, 'c>(
             }
         }
         ExternDesc::Instance(it) => {
-            let mut s = s.push_origin(is_export, ed.kebab_name);
+            let mut s = s.push_origin(origin_was_export, ed.kebab_name);
             let wn = split_wit_name(ed.kebab_name);
             emit_instance(&mut s, wn.clone(), it);
 
@@ -831,8 +838,8 @@ fn emit_component<'a, 'b, 'c>(s: &'c mut State<'a, 'b>, wn: WitName, ct: &'c Com
     s.cur_trait().items.extend(quote! { #(#imports)* });
 
     s.adjust_vars(ct.instance.evars.len() as u32);
-
     s.import_param_var = Some(format_ident!("I"));
+    s.is_export = true;
 
     let export_name = kebab_to_exports_name(wn.name);
     *s.bound_vars = ct