Skip to content

Commit 89f9875

Browse files
vstinnervstinner
authored
gh-106320: Move private _PyHash API to the internal C API (#107026)
* No longer export most private _PyHash symbols, only export the ones which are needed by shared extensions. * Modules/_xxtestfuzz/fuzzer.c now uses the internal C API.
1 parent 756add0 commit 89f9875

File tree

8 files changed

+94
-87
lines changed

8 files changed

+94
-87
lines changed

‎Include/internal/pycore_pyhash.h‎

Lines changed: 80 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,86 @@
1-
#ifndef Py_INTERNAL_HASH_H
2-
#define Py_INTERNAL_HASH_H
1+
#ifndef Py_INTERNAL_PYHASH_H
2+
#define Py_INTERNAL_PYHASH_H
33

44
#ifndef Py_BUILD_CORE
55
# error "this header requires Py_BUILD_CORE define"
66
#endif
77

8+
/* Helpers for hash functions */
9+
extern Py_hash_t _Py_HashDouble(PyObject *, double);
10+
// _decimal shared extensions uses _Py_HashPointer()
11+
PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*);
12+
// Similar to _Py_HashPointer(), but don't replace -1 with -2
13+
extern Py_hash_t _Py_HashPointerRaw(const void*);
14+
// _datetime shared extension uses _Py_HashBytes()
15+
PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t);
16+
17+
/* Prime multiplier used in string and various other hashes. */
18+
#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */
19+
20+
/* Parameters used for the numeric hash implementation. See notes for
21+
_Py_HashDouble in Python/pyhash.c. Numeric hashes are based on
22+
reduction modulo the prime 2**_PyHASH_BITS - 1. */
23+
24+
#if SIZEOF_VOID_P >= 8
25+
# define _PyHASH_BITS 61
26+
#else
27+
# define _PyHASH_BITS 31
28+
#endif
29+
30+
#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1)
31+
#define _PyHASH_INF 314159
32+
#define _PyHASH_IMAG _PyHASH_MULTIPLIER
33+
34+
/* Hash secret
35+
*
36+
* memory layout on 64 bit systems
37+
* cccccccc cccccccc cccccccc uc -- unsigned char[24]
38+
* pppppppp ssssssss ........ fnv -- two Py_hash_t
39+
* k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t
40+
* ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t
41+
* ........ ........ eeeeeeee pyexpat XML hash salt
42+
*
43+
* memory layout on 32 bit systems
44+
* cccccccc cccccccc cccccccc uc
45+
* ppppssss ........ ........ fnv -- two Py_hash_t
46+
* k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*)
47+
* ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t
48+
* ........ ........ eeee.... pyexpat XML hash salt
49+
*
50+
* (*) The siphash member may not be available on 32 bit platforms without
51+
* an unsigned int64 data type.
52+
*/
53+
typedef union {
54+
/* ensure 24 bytes */
55+
unsigned char uc[24];
56+
/* two Py_hash_t for FNV */
57+
struct {
58+
Py_hash_t prefix;
59+
Py_hash_t suffix;
60+
} fnv;
61+
/* two uint64 for SipHash24 */
62+
struct {
63+
uint64_t k0;
64+
uint64_t k1;
65+
} siphash;
66+
/* a different (!) Py_hash_t for small string optimization */
67+
struct {
68+
unsigned char padding[16];
69+
Py_hash_t suffix;
70+
} djbx33a;
71+
struct {
72+
unsigned char padding[16];
73+
Py_hash_t hashsalt;
74+
} expat;
75+
} _Py_HashSecret_t;
76+
77+
// _elementtree shared extension uses _Py_HashSecret.expat
78+
PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret;
79+
80+
#ifdef Py_DEBUG
81+
extern int _Py_HashSecret_Initialized;
82+
#endif
83+
884

985
struct pyhash_runtime_state {
1086
struct {
@@ -34,7 +110,6 @@ struct pyhash_runtime_state {
34110
}
35111

36112

37-
uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t);
38-
113+
extern uint64_t _Py_KeyedHash(uint64_t key, const void *src, Py_ssize_t src_sz);
39114

40-
#endif // Py_INTERNAL_HASH_H
115+
#endif // !Py_INTERNAL_PYHASH_H

‎Include/pyhash.h‎

Lines changed: 3 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,10 @@
11
#ifndef Py_HASH_H
2-
32
#define Py_HASH_H
43
#ifdef __cplusplus
54
extern "C" {
65
#endif
76

8-
/* Helpers for hash functions */
97
#ifndef Py_LIMITED_API
10-
PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double);
11-
PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*);
12-
// Similar to _Py_HashPointer(), but don't replace -1 with -2
13-
PyAPI_FUNC(Py_hash_t) _Py_HashPointerRaw(const void*);
14-
PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t);
15-
#endif
16-
17-
/* Prime multiplier used in string and various other hashes. */
18-
#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */
19-
20-
/* Parameters used for the numeric hash implementation. See notes for
21-
_Py_HashDouble in Python/pyhash.c. Numeric hashes are based on
22-
reduction modulo the prime 2**_PyHASH_BITS - 1. */
23-
24-
#if SIZEOF_VOID_P >= 8
25-
# define _PyHASH_BITS 61
26-
#else
27-
# define _PyHASH_BITS 31
28-
#endif
29-
30-
#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1)
31-
#define _PyHASH_INF 314159
32-
#define _PyHASH_IMAG _PyHASH_MULTIPLIER
33-
34-
35-
/* hash secret
36-
*
37-
* memory layout on 64 bit systems
38-
* cccccccc cccccccc cccccccc uc -- unsigned char[24]
39-
* pppppppp ssssssss ........ fnv -- two Py_hash_t
40-
* k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t
41-
* ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t
42-
* ........ ........ eeeeeeee pyexpat XML hash salt
43-
*
44-
* memory layout on 32 bit systems
45-
* cccccccc cccccccc cccccccc uc
46-
* ppppssss ........ ........ fnv -- two Py_hash_t
47-
* k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*)
48-
* ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t
49-
* ........ ........ eeee.... pyexpat XML hash salt
50-
*
51-
* (*) The siphash member may not be available on 32 bit platforms without
52-
* an unsigned int64 data type.
53-
*/
54-
#ifndef Py_LIMITED_API
55-
typedef union {
56-
/* ensure 24 bytes */
57-
unsigned char uc[24];
58-
/* two Py_hash_t for FNV */
59-
struct {
60-
Py_hash_t prefix;
61-
Py_hash_t suffix;
62-
} fnv;
63-
/* two uint64 for SipHash24 */
64-
struct {
65-
uint64_t k0;
66-
uint64_t k1;
67-
} siphash;
68-
/* a different (!) Py_hash_t for small string optimization */
69-
struct {
70-
unsigned char padding[16];
71-
Py_hash_t suffix;
72-
} djbx33a;
73-
struct {
74-
unsigned char padding[16];
75-
Py_hash_t hashsalt;
76-
} expat;
77-
} _Py_HashSecret_t;
78-
PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret;
79-
80-
#ifdef Py_DEBUG
81-
PyAPI_DATA(int) _Py_HashSecret_Initialized;
82-
#endif
83-
84-
858
/* hash function definition */
869
typedef struct {
8710
Py_hash_t (*const hash)(const void *, Py_ssize_t);
@@ -94,7 +17,7 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void);
9417
#endif
9518

9619

97-
/* cutoff for small string DJBX33A optimization in range [1, cutoff).
20+
/* Cutoff for small string DJBX33A optimization in range [1, cutoff).
9821
*
9922
* About 50% of the strings in a typical Python application are smaller than
10023
* 6 to 7 chars. However DJBX33A is vulnerable to hash collision attacks.
@@ -112,7 +35,7 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void);
11235
#endif /* Py_HASH_CUTOFF */
11336

11437

115-
/* hash algorithm selection
38+
/* Hash algorithm selection
11639
*
11740
* The values for Py_HASH_* are hard-coded in the
11841
* configure script.
@@ -140,5 +63,4 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void);
14063
#ifdef __cplusplus
14164
}
14265
#endif
143-
144-
#endif /* !Py_HASH_H */
66+
#endif // !Py_HASH_H

‎Modules/_elementtree.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "Python.h"
1919
#include "pycore_import.h" // _PyImport_GetModuleAttrString()
20+
#include "pycore_pyhash.h" // _Py_HashSecret
2021
#include "structmember.h" // PyMemberDef
2122
#include "expat.h"
2223
#include "pyexpat.h"

‎Modules/_hashopenssl.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@
2424

2525
#include "Python.h"
2626
#include "pycore_hashtable.h"
27-
#include "hashlib.h"
27+
#include "pycore_pyhash.h" // _Py_HashBytes()
2828
#include "pycore_strhex.h" // _Py_strhex()
29+
#include "hashlib.h"
2930

3031
/* EVP is the preferred interface to hashing in OpenSSL */
3132
#include <openssl/evp.h>

‎Modules/_xxtestfuzz/fuzzer.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@
1010
1111
See the source code for LLVMFuzzerTestOneInput for details. */
1212

13+
#ifndef Py_BUILD_CORE
14+
# define Py_BUILD_CORE 1
15+
#endif
16+
1317
#include <Python.h>
18+
#include "pycore_pyhash.h" // _Py_HashBytes()
1419
#include <stdlib.h>
1520
#include <inttypes.h>
1621

‎Modules/pyexpat.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
#include "Python.h"
66
#include "pycore_import.h" // _PyImport_SetModule()
7+
#include "pycore_pyhash.h" // _Py_HashSecret
78
#include <ctype.h>
89

910
#include "structmember.h" // PyMemberDef

‎Python/hashtable.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646

4747
#include "Python.h"
4848
#include "pycore_hashtable.h"
49+
#include "pycore_pyhash.h" // _Py_HashPointerRaw()
4950

5051
#define HASHTABLE_MIN_SIZE 16
5152
#define HASHTABLE_HIGH 0.50

‎Python/pyhash.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
All the utility functions (_Py_Hash*()) return "-1" to signify an error.
55
*/
66
#include "Python.h"
7+
#include "pycore_pyhash.h" // _Py_HashSecret_t
78

89
#ifdef __APPLE__
910
# include <libkern/OSByteOrder.h>

0 commit comments

Comments
 (0)