[vm/ffi] Expose a subset of dart_(native_)api.h for dynamic linking.

This CL introduces dart_api_dl.h which exposes a subset of dart_api.h
and dart_native_api.h for dynamic linking at runtime through the FFI.
Dynamic linking is done through including dart_api_dl.cc in a shared
library and passing NativeApi.initializeApiDLData to the init function.

This CL also includes Native API versioning to deal with possible
version skew between native api version against which native libraries
are compiled and the version in the DartVM the code is run on.

The subset of symbols in the CL includes handle related symbols, error
related symbols, handle scope symbols, and native port sumbols.

Design: http://go/dart-ffi-expose-dart-api

Closes: #40607
Closes: #36858
Closes: #41319
Closes: flutter/flutter#46887
Closes: flutter/flutter#47061

Misc:
Closes: #42260

Change-Id: I9e557808dbc99b341f23964cbddbb05f26d7a6c5
Cq-Include-Trybots: luci.dart.try:vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64-try,app-kernel-linux-debug-x64-try,vm-kernel-linux-debug-ia32-try,vm-kernel-win-debug-x64-try,vm-kernel-win-debug-ia32-try,vm-kernel-precomp-linux-debug-x64-try,vm-dartkb-linux-release-x64-abi-try,vm-kernel-precomp-android-release-arm64-try,vm-kernel-asan-linux-release-x64-try,vm-kernel-msan-linux-release-x64-try,vm-kernel-precomp-msan-linux-release-x64-try,vm-kernel-linux-release-simarm-try,vm-kernel-linux-release-simarm64-try,vm-kernel-precomp-android-release-arm_x64-try,vm-kernel-precomp-obfuscate-linux-release-x64-try,dart-sdk-linux-try,analyzer-analysis-server-linux-try,analyzer-linux-release-try,front-end-linux-release-x64-try,vm-kernel-precomp-win-release-x64-try,vm-kernel-mac-debug-x64-try,vm-precomp-ffi-qemu-linux-release-arm-try,vm-kernel-nnbd-linux-debug-x64-try,analyzer-nnbd-linux-release-try,front-end-nnbd-linux-release-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/145592
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>
Reviewed-by: Alexander Markov <[email protected]>

Author: dcharkes

pkg/kernel/lib/library_index.dart

@@ -12,6 +12,7 @@ import 'ast.dart';
 class LibraryIndex {
   static const String getterPrefix = 'get:';
   static const String setterPrefix = 'set:';
+  static const String tearoffPrefix = 'get#';
 
   /// A special class name that can be used to access the top-level members
   /// of a library.
@@ -237,7 +238,7 @@ class _MemberTable {
   String getDisambiguatedExtensionName(
       ExtensionMemberDescriptor extensionMember) {
     if (extensionMember.kind == ExtensionMemberKind.TearOff)
-      return 'get#' + extensionMember.name.name;
+      return LibraryIndex.tearoffPrefix + extensionMember.name.name;
     if (extensionMember.kind == ExtensionMemberKind.Getter)
       return LibraryIndex.getterPrefix + extensionMember.name.name;
     if (extensionMember.kind == ExtensionMemberKind.Setter)

pkg/vm/lib/transformations/ffi.dart

@@ -216,6 +216,8 @@ class FfiTransformer extends Transformer {
   final Map<NativeType, Procedure> storeMethods;
   final Map<NativeType, Procedure> elementAtMethods;
   final Procedure loadStructMethod;
+  final Procedure asFunctionTearoff;
+  final Procedure lookupFunctionTearoff;
 
   /// Classes corresponding to [NativeType], indexed by [NativeType].
   final List<Class> nativeTypesClasses;
@@ -274,7 +276,13 @@ class FfiTransformer extends Transformer {
           final name = nativeTypeClassNames[t.index];
           return index.getTopLevelMember('dart:ffi', "_elementAt$name");
         }),
-        loadStructMethod = index.getTopLevelMember('dart:ffi', '_loadStruct');
+        loadStructMethod = index.getTopLevelMember('dart:ffi', '_loadStruct'),
+        asFunctionTearoff = index.getMember('dart:ffi', 'NativeFunctionPointer',
+            LibraryIndex.tearoffPrefix + 'asFunction'),
+        lookupFunctionTearoff = index.getMember(
+            'dart:ffi',
+            'DynamicLibraryExtension',
+            LibraryIndex.tearoffPrefix + 'lookupFunction');
 
   /// Computes the Dart type corresponding to a ffi.[NativeType], returns null
   /// if it is not a valid NativeType.

pkg/vm/lib/transformations/ffi_use_sites.dart

@@ -122,8 +122,15 @@ class _FfiUseSiteTransformer extends FfiTransformer {
 
   @override
   visitProcedure(Procedure node) {
+    if (isFfiLibrary && node.isExtensionMember) {
+      if (node == asFunctionTearoff || node == lookupFunctionTearoff) {
+        // Skip static checks and transformation for the tearoffs.
+        return node;
+      }
+    }
+
     _staticTypeContext = new StaticTypeContext(node, env);
-    var result = super.visitProcedure(node);
+    final result = super.visitProcedure(node);
     _staticTypeContext = null;
     return result;
   }
@@ -159,15 +166,16 @@ class _FfiUseSiteTransformer extends FfiTransformer {
 
     final Member target = node.target;
     try {
-      if (target == lookupFunctionMethod && !isFfiLibrary) {
+      if (target == lookupFunctionMethod) {
         final DartType nativeType = InterfaceType(
             nativeFunctionClass, Nullability.legacy, [node.arguments.types[0]]);
         final DartType dartType = node.arguments.types[1];
 
         _ensureNativeTypeValid(nativeType, node);
         _ensureNativeTypeToDartType(nativeType, dartType, node);
+
         return _replaceLookupFunction(node);
-      } else if (target == asFunctionMethod && !isFfiLibrary) {
+      } else if (target == asFunctionMethod) {
         final DartType dartType = node.arguments.types[1];
         final DartType nativeType = InterfaceType(
             nativeFunctionClass, Nullability.legacy, [node.arguments.types[0]]);

runtime/BUILD.gn

@@ -210,8 +210,11 @@ source_set("dart_api") {
   public_configs = [ ":dart_public_config" ]
   sources = [
     "include/dart_api.h",
+    "include/dart_api_dl.h",
     "include/dart_native_api.h",
     "include/dart_tools_api.h",
+    "include/dart_version.h",
+    "include/internal/dart_api_dl_impl.h",
   ]
 }
 

runtime/bin/BUILD.gn

@@ -1139,11 +1139,14 @@ shared_library("ffi_test_dynamic_library") {
 shared_library("ffi_test_functions") {
   deps = [ ":dart" ]
 
-  # The two files here do not depend on each other.
-  # flutter/flutter integration tests will only use `ffi_test_functions.cc` -
-  # any test functionality using `dart_api.h` has to go into
-  # `ffi_test_functions_vmspecific.cc`.
   sources = [
+    # This file must be compiled in for dynamic linking.
+    "../include/dart_api_dl.cc",
+
+    # The two files here do not depend on each other.
+    # flutter/flutter integration tests will only use `ffi_test_functions.cc` -
+    # any test functionality using `dart_api.h` has to go into
+    # `ffi_test_functions_vmspecific.cc`.
     "ffi_test/ffi_test_functions.cc",
     "ffi_test/ffi_test_functions_vmspecific.cc",
   ]

runtime/bin/ffi_test/ffi_test_functions_vmspecific.cc

@@ -35,6 +35,8 @@
 #include "include/dart_api.h"
 #include "include/dart_native_api.h"
 
+#include "include/dart_api_dl.h"
+
 namespace dart {
 
 #define CHECK(X)                                                               \
@@ -270,31 +272,9 @@ DART_EXPORT intptr_t TestCallbackWrongIsolate(void (*fn)()) {
 #endif  // defined(TARGET_OS_LINUX)
 
 ////////////////////////////////////////////////////////////////////////////////
-// Dynamic linking of dart_native_api.h for the next two samples.
-typedef bool (*Dart_PostCObjectType)(Dart_Port port_id, Dart_CObject* message);
-Dart_PostCObjectType Dart_PostCObject_ = nullptr;
-
-DART_EXPORT void RegisterDart_PostCObject(
-    Dart_PostCObjectType function_pointer) {
-  Dart_PostCObject_ = function_pointer;
-}
-
-typedef Dart_Port (*Dart_NewNativePortType)(const char* name,
-                                            Dart_NativeMessageHandler handler,
-                                            bool handle_concurrently);
-Dart_NewNativePortType Dart_NewNativePort_ = nullptr;
-
-DART_EXPORT void RegisterDart_NewNativePort(
-    Dart_NewNativePortType function_pointer) {
-  Dart_NewNativePort_ = function_pointer;
-}
-
-typedef bool (*Dart_CloseNativePortType)(Dart_Port native_port_id);
-Dart_CloseNativePortType Dart_CloseNativePort_ = nullptr;
-
-DART_EXPORT void RegisterDart_CloseNativePort(
-    Dart_CloseNativePortType function_pointer) {
-  Dart_CloseNativePort_ = function_pointer;
+// Initialize `dart_api_dl.h`
+DART_EXPORT intptr_t InitDartApiDL(void* data) {
+  return Dart_InitializeApiDL(data);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -342,7 +322,7 @@ void NotifyDart(Dart_Port send_port, const Work* work) {
   dart_object.type = Dart_CObject_kInt64;
   dart_object.value.as_int64 = work_addr;
 
-  const bool result = Dart_PostCObject_(send_port, &dart_object);
+  const bool result = Dart_PostCObject_DL(send_port, &dart_object);
   if (!result) {
     FATAL("C   :  Posting message to port failed.");
   }
@@ -504,16 +484,16 @@ class PendingCall {
   PendingCall(void** buffer, size_t* length)
       : response_buffer_(buffer), response_length_(length) {
     receive_port_ =
-        Dart_NewNativePort_("cpp-response", &PendingCall::HandleResponse,
-                            /*handle_concurrently=*/false);
+        Dart_NewNativePort_DL("cpp-response", &PendingCall::HandleResponse,
+                              /*handle_concurrently=*/false);
   }
-  ~PendingCall() { Dart_CloseNativePort_(receive_port_); }
+  ~PendingCall() { Dart_CloseNativePort_DL(receive_port_); }
 
   Dart_Port port() const { return receive_port_; }
 
   void PostAndWait(Dart_Port port, Dart_CObject* object) {
     std::unique_lock<std::mutex> lock(mutex);
-    const bool success = Dart_PostCObject_(send_port_, object);
+    const bool success = Dart_PostCObject_DL(send_port_, object);
     if (!success) FATAL("Failed to send message, invalid port or isolate died");
 
     printf("C   :  Waiting for result.\n");
@@ -668,7 +648,7 @@ void MyCallback2(uint8_t a) {
   printf("C   :  Dart_PostCObject_(request: %" Px ", call: %" Px ").\n",
          reinterpret_cast<intptr_t>(&c_request),
          reinterpret_cast<intptr_t>(&c_pending_call));
-  Dart_PostCObject_(send_port_, &c_request);
+  Dart_PostCObject_DL(send_port_, &c_request);
 }
 
 // Simulated work for Thread #1.
@@ -793,7 +773,8 @@ DART_EXPORT void ThreadPoolTest_BarrierSync(
 ////////////////////////////////////////////////////////////////////////////////
 // Functions for handle tests.
 //
-// vmspecific_handle_test.dart
+// vmspecific_handle_test.dart (statically linked).
+// vmspecific_handle_dynamically_linked_test.dart (dynamically linked).
 
 static void RunFinalizer(void* isolate_callback_data,
                          Dart_WeakPersistentHandle handle,
@@ -912,4 +893,46 @@ DART_EXPORT Dart_Handle TrueHandle() {
   return Dart_True();
 }
 
+DART_EXPORT Dart_Handle PassObjectToCUseDynamicLinking(Dart_Handle h) {
+  auto persistent_handle = Dart_NewPersistentHandle_DL(h);
+
+  Dart_Handle handle_2 = Dart_HandleFromPersistent_DL(persistent_handle);
+  Dart_SetPersistentHandle_DL(persistent_handle, h);
+  Dart_DeletePersistentHandle_DL(persistent_handle);
+
+  auto weak_handle = Dart_NewWeakPersistentHandle_DL(
+      handle_2, reinterpret_cast<void*>(0x1234), 64, RunFinalizer);
+  Dart_Handle return_value = Dart_HandleFromWeakPersistent_DL(weak_handle);
+
+  Dart_DeleteWeakPersistentHandle_DL(weak_handle);
+
+  return return_value;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Example for doing closure callbacks with help of `dart_api.h`.
+//
+// sample_ffi_functions_callbacks_closures.dart
+
+void (*callback_)(Dart_Handle);
+Dart_PersistentHandle closure_to_callback_;
+
+DART_EXPORT void RegisterClosureCallbackFP(void (*callback)(Dart_Handle)) {
+  callback_ = callback;
+}
+
+DART_EXPORT void RegisterClosureCallback(Dart_Handle h) {
+  closure_to_callback_ = Dart_NewPersistentHandle_DL(h);
+}
+
+DART_EXPORT void InvokeClosureCallback() {
+  Dart_Handle closure_handle =
+      Dart_HandleFromPersistent_DL(closure_to_callback_);
+  callback_(closure_handle);
+}
+
+DART_EXPORT void ReleaseClosureCallback() {
+  Dart_DeletePersistentHandle_DL(closure_to_callback_);
+}
+
 }  // namespace dart

runtime/include/dart_api.h

@@ -243,6 +243,8 @@ typedef struct _Dart_IsolateGroup* Dart_IsolateGroup;
 typedef struct _Dart_Handle* Dart_Handle;
 typedef Dart_Handle Dart_PersistentHandle;
 typedef struct _Dart_WeakPersistentHandle* Dart_WeakPersistentHandle;
+// These three structs are versioned by DART_API_DL_MAJOR_VERSION, bump the
+// version when changing this struct.
 
 typedef void (*Dart_WeakPersistentHandleFinalizer)(
     void* isolate_callback_data,

runtime/include/dart_api_dl.cc

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+ * for details. All rights reserved. Use of this source code is governed by a
+ * BSD-style license that can be found in the LICENSE file.
+ */
+
+#include "include/dart_api_dl.h"
+#include "include/dart_version.h"
+#include "include/internal/dart_api_dl_impl.h"
+
+#include <string.h>
+
+#define DART_API_DL_DEFINITIONS(name)                                          \
+  using name##Type = decltype(&name);                                          \
+  name##Type name##_DL = nullptr;
+DART_API_ALL_DL_SYMBOLS(DART_API_DL_DEFINITIONS)
+#undef DART_API_DL_DEFINITIONS
+
+typedef void (*DartApiEntry_function)();
+
+DartApiEntry_function FindFunctionPointer(const DartApiEntry* entries,
+                                          const char* name) {
+  while (entries->name != nullptr) {
+    if (strcmp(entries->name, name) == 0) return entries->function;
+    entries++;
+  }
+  return nullptr;
+}
+
+intptr_t Dart_InitializeApiDL(void* data) {
+  DartApi* dart_api_data = reinterpret_cast<DartApi*>(data);
+
+  if (dart_api_data->major != DART_API_DL_MAJOR_VERSION) {
+    // If the DartVM we're running on does not have the same version as this
+    // file was compiled against, refuse to initialize. The symbols are not
+    // compatible.
+    return -1;
+  }
+  // Minor versions are allowed to be different.
+  // If the DartVM has a higher minor version, it will provide more symbols
+  // than we initialize here.
+  // If the DartVM has a lower minor version, it will not provide all symbols.
+  // In that case, we leave the missing symbols un-initialized. Those symbols
+  // should not be used by the Dart and native code. The client is responsible
+  // for checking the minor version number himself based on which symbols it
+  // is using.
+  // (If we would error out on this case, recompiling native code against a
+  // newer SDK would break all uses on older SDKs, which is too strict.)
+
+  const DartApiEntry* dart_api_function_pointers = dart_api_data->functions;
+
+#define DART_API_DL_INIT(name)                                                 \
+  name##_DL = reinterpret_cast<name##Type>(                                    \
+      FindFunctionPointer(dart_api_function_pointers, #name));
+  DART_API_ALL_DL_SYMBOLS(DART_API_DL_INIT)
+#undef DART_API_DL_INIT
+
+  return 0;
+}