8302599: Extend ASan support to Microsoft Visual C++

Reviewed-by: erikj, stuefe, ihse

Author: jcking

make/autoconf/jdk-options.m4

@@ -415,19 +415,28 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_ADDRESS_SANITIZER],
       CHECK_AVAILABLE: [
         AC_MSG_CHECKING([if AddressSanitizer (asan) is available])
         if test "x$TOOLCHAIN_TYPE" = "xgcc" ||
-            test "x$TOOLCHAIN_TYPE" = "xclang"; then
+           test "x$TOOLCHAIN_TYPE" = "xclang" ||
+           test "x$TOOLCHAIN_TYPE" = "xmicrosoft"; then
           AC_MSG_RESULT([yes])
         else
           AC_MSG_RESULT([no])
           AVAILABLE=false
         fi
       ],
       IF_ENABLED: [
-        # ASan is simply incompatible with gcc -Wstringop-truncation. See
-        # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85650
-        # It's harmless to be suppressed in clang as well.
-        ASAN_CFLAGS="-fsanitize=address -Wno-stringop-truncation -fno-omit-frame-pointer -fno-common -DADDRESS_SANITIZER"
-        ASAN_LDFLAGS="-fsanitize=address"
+        if test "x$TOOLCHAIN_TYPE" = "xgcc" ||
+           test "x$TOOLCHAIN_TYPE" = "xclang"; then
+          # ASan is simply incompatible with gcc -Wstringop-truncation. See
+          # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85650
+          # It's harmless to be suppressed in clang as well.
+          ASAN_CFLAGS="-fsanitize=address -Wno-stringop-truncation -fno-omit-frame-pointer -fno-common -DADDRESS_SANITIZER"
+          ASAN_LDFLAGS="-fsanitize=address"
+        elif test "x$TOOLCHAIN_TYPE" = "xmicrosoft"; then
+          # -Oy- is equivalent to -fno-omit-frame-pointer in GCC/Clang.
+          ASAN_CFLAGS="-fsanitize=address -Oy- -DADDRESS_SANITIZER"
+          # MSVC produces a warning if you pass -fsanitize=address to the linker.
+          ASAN_LDFLAGS=""
+        fi
         JVM_CFLAGS="$JVM_CFLAGS $ASAN_CFLAGS"
         JVM_LDFLAGS="$JVM_LDFLAGS $ASAN_LDFLAGS"
         CFLAGS_JDKLIB="$CFLAGS_JDKLIB $ASAN_CFLAGS"

make/data/asan/asan_default_options.c

@@ -33,6 +33,8 @@
 
 #if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(visibility)
 #define ATTRIBUTE_DEFAULT_VISIBILITY __attribute__((visibility("default")))
+#elif defined(_MSC_VER)
+#define ATTRIBUTE_DEFAULT_VISIBILITY __declspec(dllexport)
 #else
 #define ATTRIBUTE_DEFAULT_VISIBILITY
 #endif
@@ -43,12 +45,18 @@
 #define ATTRIBUTE_USED
 #endif
 
+#if defined(_MSC_VER)
+#define CDECL __cdecl
+#else
+#define CDECL
+#endif
+
 // Override weak symbol exposed by ASan to override default options. This is called by ASan
 // extremely early during library loading, before main is called. We need to override the default
 // options because LSan is enabled by default and Hotspot is not yet compatible with it.
 // Additionally we need to prevent ASan from handling SIGSEGV, so that Hotspot's crash handler is
 // used. You can override these options by setting the environment variable ASAN_OPTIONS.
-ATTRIBUTE_DEFAULT_VISIBILITY ATTRIBUTE_USED const char* __asan_default_options() {
+ATTRIBUTE_DEFAULT_VISIBILITY ATTRIBUTE_USED const char* CDECL __asan_default_options() {
   return
 #ifdef LEAK_SANITIZER
     "leak_check_at_exit=0,"