Malloc should fail on too large allocations

1: two new options to choose between fail by assert-then-assume and return null
2: `__CPROVER_max_malloc_size` is introduced
3: some tests for boundary conditions
4: user-level documentation

Author: petr-bauch

doc/cprover-manual/properties.md

@@ -319,3 +319,19 @@ example, replacing functions or setting global variables with the `__CPROVER`
 prefix might make analysis impossible. To avoid doing this by accident, negative
 lookahead can be used. For example, `(?!__).*` matches all names not starting
 with `__`.
+
+### Malloc failure mode
+
+|Flag                    |  Check                                          |
+|------------------------|-------------------------------------------------|
+| `--malloc-fail-null`   |  in case malloc fails return NULL               |
+| `--malloc-fail-assert` |  in case malloc fails report as failed property |
+
+Calling `malloc` may fail for a number of reasons and the function may return a
+NULL pointer. The users can choose if and how they want the `malloc`-related
+failures to occur. The option `--malloc-fail-null` results in `malloc` returning
+the NULL pointer when failing. The option `--malloc-fail-assert` places
+additional properties inside `malloc` that are checked and if failing the
+verification is terminated (by `assume(false)`). One such property is that the
+allocated size is not too large, i.e. internally representable. When neither of
+those two options are used, CBMC will assume that `malloc` does not fail.

regression/cbmc/malloc-too-large/largest_representable.c

@@ -0,0 +1,10 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+int main()
+{
+  int *p = malloc(__CPROVER_max_malloc_size); // try to allocate exactly the max
+
+  return 0;
+}

regression/cbmc/malloc-too-large/largest_representable.desc

@@ -0,0 +1,9 @@
+CORE
+largest_representable.c
+--malloc-fail-assert
+^EXIT=0$
+^SIGNAL=0$
+^\[malloc.assertion.\d+\] line \d+ max allocation size exceeded: SUCCESS$
+^VERIFICATION SUCCESSFUL$
+--
+^warning: ignoring

regression/cbmc/malloc-too-large/max_size.c

@@ -0,0 +1,10 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+int main()
+{
+  int *p = malloc(SIZE_MAX);
+
+  return 0;
+}

regression/cbmc/malloc-too-large/max_size.desc

@@ -0,0 +1,9 @@
+CORE
+max_size.c
+--malloc-fail-assert
+^EXIT=10$
+^SIGNAL=0$
+^\[malloc.assertion.\d+\] line \d+ max allocation size exceeded: FAILURE$
+^VERIFICATION FAILED$
+--
+^warning: ignoring

regression/cbmc/malloc-too-large/one_byte_too_large.c

@@ -0,0 +1,12 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+int main()
+{
+  // try to allocate the smallest violating amount
+  int *p = malloc(__CPROVER_max_malloc_size + 1);
+  assert(p);
+
+  return 0;
+}

regression/cbmc/malloc-too-large/one_byte_too_large.desc

@@ -0,0 +1,9 @@
+CORE
+one_byte_too_large.c
+--malloc-fail-null
+^EXIT=10$
+^SIGNAL=0$
+^\[main.assertion.\d+\] line \d+ assertion p: FAILURE$
+^VERIFICATION FAILED$
+--
+^warning: ignoring

src/ansi-c/ansi_c_internal_additions.cpp

@@ -155,6 +155,15 @@ void ansi_c_internal_additions(std::string &code)
     "void *" CPROVER_PREFIX "allocate("
       CPROVER_PREFIX "size_t size, " CPROVER_PREFIX "bool zero);\n"
     "const void *" CPROVER_PREFIX "alloca_object = 0;\n"
+    "int " CPROVER_PREFIX "malloc_failure_mode="+
+      std::to_string(config.ansi_c.malloc_failure_mode)+";\n"
+    "int " CPROVER_PREFIX "malloc_failure_mode_return_null="+
+    std::to_string(config.ansi_c.malloc_failure_mode_return_null)+";\n"
+    "int " CPROVER_PREFIX "malloc_failure_mode_assert_then_assume="+
+    std::to_string(config.ansi_c.malloc_failure_mode_assert_then_assume)+";\n"
+    CPROVER_PREFIX "size_t " CPROVER_PREFIX "max_malloc_size="+
+    std::to_string(1 << (config.ansi_c.pointer_width -
+                         config.bv_encoding.object_bits - 1))+";\n"
 
     // this is ANSI-C
     "extern " CPROVER_PREFIX "thread_local const char __func__["

src/ansi-c/library/cprover.h

@@ -16,6 +16,12 @@ extern const void *__CPROVER_malloc_object;
 extern __CPROVER_size_t __CPROVER_malloc_size;
 extern _Bool __CPROVER_malloc_is_new_array;
 extern const void *__CPROVER_memory_leak;
+extern int __CPROVER_malloc_failure_mode;
+extern __CPROVER_size_t __CPROVER_max_malloc_size;
+
+// malloc failure modes
+extern int __CPROVER_malloc_failure_mode_return_null;
+extern int __CPROVER_malloc_failure_mode_assert_then_assume;
 
 void __CPROVER_assume(__CPROVER_bool assumption) __attribute__((__noreturn__));
 void __CPROVER_assert(__CPROVER_bool assertion, const char *description);

src/ansi-c/library/stdlib.c

@@ -115,23 +115,47 @@ inline void *malloc(__CPROVER_size_t malloc_size)
   // realistically, malloc may return NULL,
   // and __CPROVER_allocate doesn't, but no one cares
   __CPROVER_HIDE:;
-  void *malloc_res;
-  malloc_res = __CPROVER_allocate(malloc_size, 0);
 
-  // make sure it's not recorded as deallocated
-  __CPROVER_deallocated=(malloc_res==__CPROVER_deallocated)?0:__CPROVER_deallocated;
+    if(
+      __CPROVER_malloc_failure_mode ==
+      __CPROVER_malloc_failure_mode_return_null)
+    {
+      if(malloc_size > __CPROVER_max_malloc_size)
+      {
+        return (void *)0;
+      }
+    }
+    else if(
+      __CPROVER_malloc_failure_mode ==
+      __CPROVER_malloc_failure_mode_assert_then_assume)
+    {
+      __CPROVER_assert(
+        malloc_size <= __CPROVER_max_malloc_size,
+        "max allocation size exceeded");
+      __CPROVER_assume(malloc_size <= __CPROVER_max_malloc_size);
+    }
 
-  // record the object size for non-determistic bounds checking
-  __CPROVER_bool record_malloc=__VERIFIER_nondet___CPROVER_bool();
-  __CPROVER_malloc_object=record_malloc?malloc_res:__CPROVER_malloc_object;
-  __CPROVER_malloc_size=record_malloc?malloc_size:__CPROVER_malloc_size;
-  __CPROVER_malloc_is_new_array=record_malloc?0:__CPROVER_malloc_is_new_array;
+    void *malloc_res;
+    malloc_res = __CPROVER_allocate(malloc_size, 0);
 
-  // detect memory leaks
-  __CPROVER_bool record_may_leak=__VERIFIER_nondet___CPROVER_bool();
-  __CPROVER_memory_leak=record_may_leak?malloc_res:__CPROVER_memory_leak;
+    // make sure it's not recorded as deallocated
+    __CPROVER_deallocated =
+      (malloc_res == __CPROVER_deallocated) ? 0 : __CPROVER_deallocated;
 
-  return malloc_res;
+    // record the object size for non-determistic bounds checking
+    __CPROVER_bool record_malloc = __VERIFIER_nondet___CPROVER_bool();
+    __CPROVER_malloc_object =
+      record_malloc ? malloc_res : __CPROVER_malloc_object;
+    __CPROVER_malloc_size = record_malloc ? malloc_size : __CPROVER_malloc_size;
+    __CPROVER_malloc_is_new_array =
+      record_malloc ? 0 : __CPROVER_malloc_is_new_array;
+
+    // detect memory leaks
+    __CPROVER_bool record_may_leak = __VERIFIER_nondet___CPROVER_bool();
+    __CPROVER_memory_leak =
+      record_may_leak ? malloc_res : __CPROVER_memory_leak;
+
+    return malloc_res;
 }
 
 /* FUNCTION: __builtin_alloca */