gh-111178: fix UBSan failures in Modules/{blake2,md5,sha1,sha2,sha3}module.c (GH-128248)

picnixz · web-flow · commit 922cfecbcefe · 2025-01-27T14:50:58.000+01:00
- fix UBSan failures in `blake2module.c`
- fix UBSan failures in `md5module.c`
- fix UBSan failures in `sha1module.c`
- fix UBSan failures in `sha2module.c`
- fix UBSan failures in `sha3module.c`

Cosmetics:

- suppress unused return values
- remove redundant casts in constructors
- suppress unused parameters in `{md5,sha*,blake2}module.c`
diff --git a/Modules/blake2module.c b/Modules/blake2module.c
@@ -366,6 +366,8 @@ typedef struct {
     PyMutex mutex;
 } Blake2Object;
 
+#define _Blake2Object_CAST(op)  ((Blake2Object *)(op))
+
 #include "clinic/blake2module.c.h"
 
 /*[clinic input]
@@ -849,24 +851,27 @@ static PyMethodDef py_blake2b_methods[] = {
 
 
 static PyObject *
-py_blake2b_get_name(Blake2Object *self, void *closure)
+py_blake2b_get_name(PyObject *op, void *Py_UNUSED(closure))
 {
+    Blake2Object *self = _Blake2Object_CAST(op);
     return PyUnicode_FromString(is_blake2b(self->impl) ? "blake2b" : "blake2s");
 }
 
 
 
 static PyObject *
-py_blake2b_get_block_size(Blake2Object *self, void *closure)
+py_blake2b_get_block_size(PyObject *op, void *Py_UNUSED(closure))
 {
+    Blake2Object *self = _Blake2Object_CAST(op);
     return PyLong_FromLong(is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_BLOCK_BYTES : HACL_HASH_BLAKE2S_BLOCK_BYTES);
 }
 
 
 
 static PyObject *
-py_blake2b_get_digest_size(Blake2Object *self, void *closure)
+py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure))
 {
+    Blake2Object *self = _Blake2Object_CAST(op);
     switch (self->impl) {
 #if HACL_CAN_COMPILE_SIMD256
         case Blake2b_256:
@@ -887,15 +892,13 @@ py_blake2b_get_digest_size(Blake2Object *self, void *closure)
 
 
 static PyGetSetDef py_blake2b_getsetters[] = {
-    {"name", (getter)py_blake2b_get_name,
-        NULL, NULL, NULL},
-    {"block_size", (getter)py_blake2b_get_block_size,
-        NULL, NULL, NULL},
-    {"digest_size", (getter)py_blake2b_get_digest_size,
-        NULL, NULL, NULL},
-    {NULL}
+    {"name", py_blake2b_get_name, NULL, NULL, NULL},
+    {"block_size", py_blake2b_get_block_size, NULL, NULL, NULL},
+    {"digest_size", py_blake2b_get_digest_size, NULL, NULL, NULL},
+    {NULL}  /* Sentinel */
 };
 
+
 static int
 py_blake2_clear(PyObject *op)
 {
diff --git a/Modules/md5module.c b/Modules/md5module.c
@@ -54,6 +54,8 @@ typedef struct {
     Hacl_Hash_MD5_state_t *hash_state;
 } MD5object;
 
+#define _MD5object_CAST(op)     ((MD5object *)(op))
+
 #include "clinic/md5module.c.h"
 
 
@@ -72,7 +74,7 @@ md5_get_state(PyObject *module)
 static MD5object *
 newMD5object(MD5State * st)
 {
-    MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type);
+    MD5object *md5 = PyObject_GC_New(MD5object, st->md5_type);
     if (!md5) {
         return NULL;
     }
@@ -91,10 +93,11 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg)
 }
 
 static void
-MD5_dealloc(MD5object *ptr)
+MD5_dealloc(PyObject *op)
 {
+    MD5object *ptr = _MD5object_CAST(op);
     Hacl_Hash_MD5_free(ptr->hash_state);
-    PyTypeObject *tp = Py_TYPE((PyObject*)ptr);
+    PyTypeObject *tp = Py_TYPE(op);
     PyObject_GC_UnTrack(ptr);
     PyObject_GC_Del(ptr);
     Py_DECREF(tp);
@@ -224,36 +227,27 @@ static PyMethodDef MD5_methods[] = {
 };
 
 static PyObject *
-MD5_get_block_size(PyObject *self, void *closure)
+MD5_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(MD5_BLOCKSIZE);
 }
 
 static PyObject *
-MD5_get_name(PyObject *self, void *closure)
+MD5_get_name(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyUnicode_FromStringAndSize("md5", 3);
 }
 
 static PyObject *
-md5_get_digest_size(PyObject *self, void *closure)
+md5_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(MD5_DIGESTSIZE);
 }
 
 static PyGetSetDef MD5_getseters[] = {
-    {"block_size",
-     (getter)MD5_get_block_size, NULL,
-     NULL,
-     NULL},
-    {"name",
-     (getter)MD5_get_name, NULL,
-     NULL,
-     NULL},
-    {"digest_size",
-     (getter)md5_get_digest_size, NULL,
-     NULL,
-     NULL},
+    {"block_size", MD5_get_block_size, NULL, NULL, NULL},
+    {"name", MD5_get_name, NULL, NULL, NULL},
+    {"digest_size", md5_get_digest_size, NULL, NULL, NULL},
     {NULL}  /* Sentinel */
 };
 
diff --git a/Modules/sha1module.c b/Modules/sha1module.c
@@ -55,6 +55,8 @@ typedef struct {
     Hacl_Hash_SHA1_state_t *hash_state;
 } SHA1object;
 
+#define _SHA1object_CAST(op)    ((SHA1object *)(op))
+
 #include "clinic/sha1module.c.h"
 
 
@@ -73,7 +75,7 @@ sha1_get_state(PyObject *module)
 static SHA1object *
 newSHA1object(SHA1State *st)
 {
-    SHA1object *sha = (SHA1object *)PyObject_GC_New(SHA1object, st->sha1_type);
+    SHA1object *sha = PyObject_GC_New(SHA1object, st->sha1_type);
     if (sha == NULL) {
         return NULL;
     }
@@ -93,8 +95,9 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg)
 }
 
 static void
-SHA1_dealloc(SHA1object *ptr)
+SHA1_dealloc(PyObject *op)
 {
+    SHA1object *ptr = _SHA1object_CAST(op);
     Hacl_Hash_SHA1_free(ptr->hash_state);
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
@@ -217,36 +220,27 @@ static PyMethodDef SHA1_methods[] = {
 };
 
 static PyObject *
-SHA1_get_block_size(PyObject *self, void *closure)
+SHA1_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(SHA1_BLOCKSIZE);
 }
 
 static PyObject *
-SHA1_get_name(PyObject *self, void *closure)
+SHA1_get_name(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyUnicode_FromStringAndSize("sha1", 4);
 }
 
 static PyObject *
-sha1_get_digest_size(PyObject *self, void *closure)
+sha1_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(SHA1_DIGESTSIZE);
 }
 
 static PyGetSetDef SHA1_getseters[] = {
-    {"block_size",
-     (getter)SHA1_get_block_size, NULL,
-     NULL,
-     NULL},
-    {"name",
-     (getter)SHA1_get_name, NULL,
-     NULL,
-     NULL},
-    {"digest_size",
-     (getter)sha1_get_digest_size, NULL,
-     NULL,
-     NULL},
+    {"block_size", SHA1_get_block_size, NULL, NULL, NULL},
+    {"name", SHA1_get_name, NULL, NULL, NULL},
+    {"digest_size", sha1_get_digest_size, NULL, NULL, NULL},
     {NULL}  /* Sentinel */
 };
 
@@ -346,7 +340,7 @@ _sha1_clear(PyObject *module)
 static void
 _sha1_free(void *module)
 {
-    _sha1_clear((PyObject *)module);
+    (void)_sha1_clear((PyObject *)module);
 }
 
 static int
diff --git a/Modules/sha2module.c b/Modules/sha2module.c
@@ -67,6 +67,9 @@ typedef struct {
     Hacl_Hash_SHA2_state_t_512 *state;
 } SHA512object;
 
+#define _SHA256object_CAST(op)  ((SHA256object *)(op))
+#define _SHA512object_CAST(op)  ((SHA512object *)(op))
+
 #include "clinic/sha2module.c.h"
 
 /* We shall use run-time type information in the remainder of this module to
@@ -101,8 +104,7 @@ static void SHA512copy(SHA512object *src, SHA512object *dest)
 static SHA256object *
 newSHA224object(sha2_state *state)
 {
-    SHA256object *sha = (SHA256object *)PyObject_GC_New(
-        SHA256object, state->sha224_type);
+    SHA256object *sha = PyObject_GC_New(SHA256object, state->sha224_type);
     if (!sha) {
         return NULL;
     }
@@ -115,8 +117,7 @@ newSHA224object(sha2_state *state)
 static SHA256object *
 newSHA256object(sha2_state *state)
 {
-    SHA256object *sha = (SHA256object *)PyObject_GC_New(
-        SHA256object, state->sha256_type);
+    SHA256object *sha = PyObject_GC_New(SHA256object, state->sha256_type);
     if (!sha) {
         return NULL;
     }
@@ -129,8 +130,7 @@ newSHA256object(sha2_state *state)
 static SHA512object *
 newSHA384object(sha2_state *state)
 {
-    SHA512object *sha = (SHA512object *)PyObject_GC_New(
-        SHA512object, state->sha384_type);
+    SHA512object *sha = PyObject_GC_New(SHA512object, state->sha384_type);
     if (!sha) {
         return NULL;
     }
@@ -143,8 +143,7 @@ newSHA384object(sha2_state *state)
 static SHA512object *
 newSHA512object(sha2_state *state)
 {
-    SHA512object *sha = (SHA512object *)PyObject_GC_New(
-        SHA512object, state->sha512_type);
+    SHA512object *sha = PyObject_GC_New(SHA512object, state->sha512_type);
     if (!sha) {
         return NULL;
     }
@@ -164,8 +163,9 @@ SHA2_traverse(PyObject *ptr, visitproc visit, void *arg)
 }
 
 static void
-SHA256_dealloc(SHA256object *ptr)
+SHA256_dealloc(PyObject *op)
 {
+    SHA256object *ptr = _SHA256object_CAST(op);
     Hacl_Hash_SHA2_free_256(ptr->state);
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
@@ -174,8 +174,9 @@ SHA256_dealloc(SHA256object *ptr)
 }
 
 static void
-SHA512_dealloc(SHA512object *ptr)
+SHA512_dealloc(PyObject *op)
 {
+    SHA512object *ptr = _SHA512object_CAST(op);
     Hacl_Hash_SHA2_free_512(ptr->state);
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
@@ -442,76 +443,62 @@ static PyMethodDef SHA512_methods[] = {
 };
 
 static PyObject *
-SHA256_get_block_size(PyObject *self, void *closure)
+SHA256_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(SHA256_BLOCKSIZE);
 }
 
 static PyObject *
-SHA512_get_block_size(PyObject *self, void *closure)
+SHA512_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure))
 {
     return PyLong_FromLong(SHA512_BLOCKSIZE);
 }
 
 static PyObject *
-SHA256_get_digest_size(SHA256object *self, void *closure)
+SHA256_get_digest_size(PyObject *op, void *Py_UNUSED(closure))
 {
+    SHA256object *self = _SHA256object_CAST(op);
     return PyLong_FromLong(self->digestsize);
 }
 
 static PyObject *
-SHA512_get_digest_size(SHA512object *self, void *closure)
+SHA512_get_digest_size(PyObject *op, void *Py_UNUSED(closure))
 {
+    SHA512object *self = _SHA512object_CAST(op);
     return PyLong_FromLong(self->digestsize);
 }
 
 static PyObject *
-SHA256_get_name(SHA256object *self, void *closure)
+SHA256_get_name(PyObject *op, void *Py_UNUSED(closure))
 {
+    SHA256object *self = _SHA256object_CAST(op);
     if (self->digestsize == 28) {
         return PyUnicode_FromStringAndSize("sha224", 6);
     }
     return PyUnicode_FromStringAndSize("sha256", 6);
 }
 
 static PyObject *
-SHA512_get_name(SHA512object *self, void *closure)
+SHA512_get_name(PyObject *op, void *Py_UNUSED(closure))
 {
+    SHA512object *self = _SHA512object_CAST(op);
     if (self->digestsize == 64) {
         return PyUnicode_FromStringAndSize("sha512", 6);
     }
     return PyUnicode_FromStringAndSize("sha384", 6);
 }
 
 static PyGetSetDef SHA256_getseters[] = {
-    {"block_size",
-     (getter)SHA256_get_block_size, NULL,
-     NULL,
-     NULL},
-    {"name",
-     (getter)SHA256_get_name, NULL,
-     NULL,
-     NULL},
-    {"digest_size",
-     (getter)SHA256_get_digest_size, NULL,
-     NULL,
-     NULL},
+    {"block_size", SHA256_get_block_size, NULL, NULL, NULL},
+    {"name", SHA256_get_name, NULL, NULL, NULL},
+    {"digest_size", SHA256_get_digest_size, NULL, NULL, NULL},
     {NULL}  /* Sentinel */
 };
 
 static PyGetSetDef SHA512_getseters[] = {
-    {"block_size",
-     (getter)SHA512_get_block_size, NULL,
-     NULL,
-     NULL},
-    {"name",
-     (getter)SHA512_get_name, NULL,
-     NULL,
-     NULL},
-    {"digest_size",
-     (getter)SHA512_get_digest_size, NULL,
-     NULL,
-     NULL},
+    {"block_size", SHA512_get_block_size, NULL, NULL, NULL},
+    {"name", SHA512_get_name, NULL, NULL, NULL},
+    {"digest_size", SHA512_get_digest_size, NULL, NULL, NULL},
     {NULL}  /* Sentinel */
 };
 
@@ -818,7 +805,7 @@ _sha2_clear(PyObject *module)
 static void
 _sha2_free(void *module)
 {
-    _sha2_clear((PyObject *)module);
+    (void)_sha2_clear((PyObject *)module);
 }
 
 /* Initialize this module. */
diff --git a/Modules/sha3module.c b/Modules/sha3module.c
