Skip to content

Commit 3b5b1c0

Browse files
gpsheadgpshead
authored
[3.6] bpo-35214: Initial clang MemorySanitizer support (GH-10479) (GH-10493)
Adds configure flags for msan and ubsan builds to make it easier to enable. These also encode the detail that address sanitizer and memory sanitizer should disable pymalloc. Define MEMORY_SANITIZER when appropriate at build time and adds workarounds to existing code to mark things as initialized where the sanitizer is otherwise unable to determine that. This lets our build succeed under the memory sanitizer. not all tests pass without sanitizer failures yet but we're in pretty good shape after this. (cherry picked from commit 1584a00) Contributed by Gregory P. Smith [Google LLC] Also includes a whitespace fix from make patchcheck to _posixsubprocess.c - unrelated to the main change that makes the CI happy so I'm just doing it now rather than creating a separate PR.
1 parent 9a7ba8c commit 3b5b1c0

File tree

9 files changed

+120
-9
lines changed

9 files changed

+120
-9
lines changed

‎Include/Python.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,15 @@
5353
#include "pyport.h"
5454
#include "pymacro.h"
5555

56+
/* A convenient way for code to know if clang's memory sanitizer is enabled. */
57+
#if defined(__has_feature)
58+
# if __has_feature(memory_sanitizer)
59+
# if !defined(MEMORY_SANITIZER)
60+
# define MEMORY_SANITIZER
61+
# endif
62+
# endif
63+
#endif
64+
5665
#include "pyatomic.h"
5766

5867
/* Debug-mode build with pymalloc implies PYMALLOC_DEBUG.

‎Misc/NEWS.d/next/Core and Builtins/2018-11-12-11-38-06.bpo-35214.PCHKbX.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The interpreter and extension modules have had annotations added so that
2+
they work properly under clang's Memory Sanitizer. A new configure flag
3+
--with-memory-sanitizer has been added to make test builds of this nature
4+
easier to perform.

‎Modules/_ctypes/callproc.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@
7575
#include <alloca.h>
7676
#endif
7777

78+
#ifdef MEMORY_SANITIZER
79+
#include <sanitizer/msan_interface.h>
80+
#endif
81+
7882
#if defined(_DEBUG) || defined(__MINGW32__)
7983
/* Don't use structured exception handling on Windows if this is defined.
8084
MingW, AFAIK, doesn't support it.
@@ -1136,6 +1140,13 @@ PyObject *_ctypes_callproc(PPROC pProc,
11361140
rtype = _ctypes_get_ffi_type(restype);
11371141
resbuf = alloca(max(rtype->size, sizeof(ffi_arg)));
11381142

1143+
#ifdef MEMORY_SANITIZER
1144+
/* ffi_call actually initializes resbuf, but from asm, which
1145+
* MemorySanitizer can't detect. Avoid false positives from MSan. */
1146+
if (resbuf != NULL) {
1147+
__msan_unpoison(resbuf, max(rtype->size, sizeof(ffi_arg)));
1148+
}
1149+
#endif
11391150
avalues = (void **)alloca(sizeof(void *) * argcount);
11401151
atypes = (ffi_type **)alloca(sizeof(ffi_type *) * argcount);
11411152
if (!resbuf || !avalues || !atypes) {

‎Modules/_posixsubprocess.c‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
#include <dirent.h>
2222
#endif
2323

24+
#ifdef MEMORY_SANITIZER
25+
# include <sanitizer/msan_interface.h>
26+
#endif
27+
2428
#if defined(__ANDROID__) && __ANDROID_API__ < 21 && !defined(SYS_getdents64)
2529
# include <sys/linux-syscalls.h>
2630
# define SYS_getdents64 __NR_getdents64
@@ -287,6 +291,9 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep)
287291
sizeof(buffer))) > 0) {
288292
struct linux_dirent64 *entry;
289293
int offset;
294+
#ifdef MEMORY_SANITIZER
295+
__msan_unpoison(buffer, bytes);
296+
#endif
290297
for (offset = 0; offset < bytes; offset += entry->d_reclen) {
291298
int fd;
292299
entry = (struct linux_dirent64 *)(buffer + offset);
@@ -791,11 +798,11 @@ static PyMethodDef module_methods[] = {
791798

792799

793800
static struct PyModuleDef _posixsubprocessmodule = {
794-
PyModuleDef_HEAD_INIT,
795-
"_posixsubprocess",
796-
module_doc,
797-
-1, /* No memory is needed. */
798-
module_methods,
801+
PyModuleDef_HEAD_INIT,
802+
"_posixsubprocess",
803+
module_doc,
804+
-1, /* No memory is needed. */
805+
module_methods,
799806
};
800807

801808
PyMODINIT_FUNC

‎Modules/faulthandler.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1393,7 +1393,7 @@ void _PyFaulthandler_Fini(void)
13931393
#ifdef HAVE_SIGALTSTACK
13941394
if (stack.ss_sp != NULL) {
13951395
/* Fetch the current alt stack */
1396-
stack_t current_stack;
1396+
stack_t current_stack = {};
13971397
if (sigaltstack(NULL, &current_stack) == 0) {
13981398
if (current_stack.ss_sp == stack.ss_sp) {
13991399
/* The current alt stack is the one that we installed.

‎Python/pymath.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ double _Py_force_double(double x)
1717

1818
/* inline assembly for getting and setting the 387 FPU control word on
1919
gcc/x86 */
20-
20+
#ifdef MEMORY_SANITIZER
21+
__attribute__((no_sanitize_memory))
22+
#endif
2123
unsigned short _Py_get_387controlword(void) {
2224
unsigned short cw;
2325
__asm__ __volatile__ ("fnstcw %0" : "=m" (cw));

‎Python/random.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
# endif
2121
#endif
2222

23+
#ifdef MEMORY_SANITIZER
24+
# include <sanitizer/msan_interface.h>
25+
#endif
26+
2327
#ifdef Py_DEBUG
2428
int _Py_HashSecret_Initialized = 0;
2529
#else
@@ -143,6 +147,11 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise)
143147
else {
144148
n = syscall(SYS_getrandom, dest, n, flags);
145149
}
150+
# ifdef MEMORY_SANITIZER
151+
if (n > 0) {
152+
__msan_unpoison(dest, n);
153+
}
154+
# endif
146155
#endif
147156

148157
if (n < 0) {

‎configure‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,8 @@ enable_optimizations
822822
with_lto
823823
with_hash_algorithm
824824
with_address_sanitizer
825+
with_memory_sanitizer
826+
with_undefined_behavior_sanitizer
825827
with_libs
826828
with_system_expat
827829
with_system_ffi
@@ -1520,7 +1522,10 @@ Optional Packages:
15201522
--with-hash-algorithm=[fnv|siphash24]
15211523
select hash algorithm
15221524
--with-address-sanitizer
1523-
enable AddressSanitizer
1525+
enable AddressSanitizer (asan)
1526+
--with-memory-sanitizer enable MemorySanitizer (msan)
1527+
--with-undefined-behavior-sanitizer
1528+
enable UndefinedBehaviorSanitizer (ubsan)
15241529
--with-libs='lib1 ...' link against additional libs
15251530
--with-system-expat build pyexpat module using an installed expat
15261531
library
@@ -9926,6 +9931,44 @@ if test "${with_address_sanitizer+set}" = set; then :
99269931
$as_echo "$withval" >&6; }
99279932
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
99289933
LDFLAGS="-fsanitize=address $LDFLAGS"
9934+
# ASan works by controlling memory allocation, our own malloc interferes.
9935+
with_pymalloc="no"
9936+
9937+
else
9938+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9939+
$as_echo "no" >&6; }
9940+
fi
9941+
9942+
9943+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5
9944+
$as_echo_n "checking for --with-memory-sanitizer... " >&6; }
9945+
9946+
# Check whether --with-memory_sanitizer was given.
9947+
if test "${with_memory_sanitizer+set}" = set; then :
9948+
withval=$with_memory_sanitizer;
9949+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
9950+
$as_echo "$withval" >&6; }
9951+
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
9952+
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
9953+
# MSan works by controlling memory allocation, our own malloc interferes.
9954+
with_pymalloc="no"
9955+
9956+
else
9957+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9958+
$as_echo "no" >&6; }
9959+
fi
9960+
9961+
9962+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5
9963+
$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; }
9964+
9965+
# Check whether --with-undefined_behavior_sanitizer was given.
9966+
if test "${with_undefined_behavior_sanitizer+set}" = set; then :
9967+
withval=$with_undefined_behavior_sanitizer;
9968+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
9969+
$as_echo "$withval" >&6; }
9970+
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
9971+
LDFLAGS="-fsanitize=undefined $LDFLAGS"
99299972

99309973
else
99319974
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5

‎configure.ac‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2833,11 +2833,37 @@ esac
28332833
AC_MSG_CHECKING(for --with-address-sanitizer)
28342834
AC_ARG_WITH(address_sanitizer,
28352835
AS_HELP_STRING([--with-address-sanitizer],
2836-
[enable AddressSanitizer]),
2836+
[enable AddressSanitizer (asan)]),
28372837
[
28382838
AC_MSG_RESULT($withval)
28392839
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
28402840
LDFLAGS="-fsanitize=address $LDFLAGS"
2841+
# ASan works by controlling memory allocation, our own malloc interferes.
2842+
with_pymalloc="no"
2843+
],
2844+
[AC_MSG_RESULT(no)])
2845+
2846+
AC_MSG_CHECKING(for --with-memory-sanitizer)
2847+
AC_ARG_WITH(memory_sanitizer,
2848+
AS_HELP_STRING([--with-memory-sanitizer],
2849+
[enable MemorySanitizer (msan)]),
2850+
[
2851+
AC_MSG_RESULT($withval)
2852+
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
2853+
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
2854+
# MSan works by controlling memory allocation, our own malloc interferes.
2855+
with_pymalloc="no"
2856+
],
2857+
[AC_MSG_RESULT(no)])
2858+
2859+
AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer)
2860+
AC_ARG_WITH(undefined_behavior_sanitizer,
2861+
AS_HELP_STRING([--with-undefined-behavior-sanitizer],
2862+
[enable UndefinedBehaviorSanitizer (ubsan)]),
2863+
[
2864+
AC_MSG_RESULT($withval)
2865+
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
2866+
LDFLAGS="-fsanitize=undefined $LDFLAGS"
28412867
],
28422868
[AC_MSG_RESULT(no)])
28432869

0 commit comments

Comments
 (0)