Skip to content

Commit ff9adfd

Browse files
fangererfangerer
committed
[GR-33944] Finish support for HPy 0.0.3.
PullRequest: graalpython/1975
2 parents 0ce45b9 + d1dd059 commit ff9adfd

File tree

13 files changed

+161
-51
lines changed

13 files changed

+161
-51
lines changed

‎graalpython/com.oracle.graal.python.cext/include/hpy/version.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/* MIT License
2+
*
3+
* Copyright (c) 2020, 2021, Oracle and/or its affiliates.
4+
* Copyright (c) 2019 pyhandle
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
25+
// automatically generated by setup.py:get_scm_config()
26+
#define HPY_VERSION "0.0.3"
27+
#define HPY_GIT_REVISION "2196f14"

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CArrayWrappers.java‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
*/
4141
package com.oracle.graal.python.builtins.objects.cext.common;
4242

43+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol.FUN_GET_BYTE_ARRAY_TYPE_ID;
44+
45+
import java.lang.reflect.Field;
46+
import java.nio.ByteBuffer;
47+
import java.nio.charset.StandardCharsets;
48+
4349
import com.oracle.graal.python.builtins.objects.cext.capi.CApiContext.LLVMType;
4450
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.GetLLVMType;
4551
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.IsPointerNode;
@@ -63,13 +69,8 @@
6369
import com.oracle.truffle.api.library.ExportLibrary;
6470
import com.oracle.truffle.api.library.ExportMessage;
6571
import com.oracle.truffle.llvm.spi.NativeTypeLibrary;
66-
import sun.misc.Unsafe;
6772

68-
import java.lang.reflect.Field;
69-
import java.nio.ByteBuffer;
70-
import java.nio.charset.StandardCharsets;
71-
72-
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol.FUN_GET_BYTE_ARRAY_TYPE_ID;
73+
import sun.misc.Unsafe;
7374

7475
/**
7576
* Native wrappers for managed objects such that they can be used as a C array by native code. The
@@ -98,10 +99,13 @@ private static Unsafe getUnsafe() {
9899
* the copies the contents to the native memory.
99100
*/
100101
@TruffleBoundary
101-
public static long byteArrayToNativeInt8(byte[] data) {
102-
int size = data.length * Byte.BYTES;
102+
public static long byteArrayToNativeInt8(byte[] data, boolean writeNullTerminator) {
103+
int size = (data.length + (writeNullTerminator ? 1 : 0)) * Byte.BYTES;
103104
long ptr = UNSAFE.allocateMemory(size);
104105
UNSAFE.copyMemory(data, Unsafe.ARRAY_BYTE_BASE_OFFSET, null, ptr, size);
106+
if (writeNullTerminator) {
107+
UNSAFE.putByte(ptr + data.length * Byte.BYTES, (byte) 0);
108+
}
105109
return ptr;
106110
}
107111

@@ -140,7 +144,7 @@ public static long stringToNativeUtf8Bytes(String string) {
140144
ByteBuffer encoded = StandardCharsets.UTF_8.encode(string);
141145
byte[] data = new byte[encoded.remaining() + 1];
142146
encoded.get(data, 0, data.length - 1);
143-
return byteArrayToNativeInt8(data);
147+
return byteArrayToNativeInt8(data, true);
144148
}
145149

146150
@TruffleBoundary
@@ -326,7 +330,7 @@ void toNative(
326330
@Exclusive @Cached InvalidateNativeObjectsAllManagedNode invalidateNode) {
327331
invalidateNode.execute();
328332
if (!lib.isNative(this)) {
329-
setNativePointer(byteArrayToNativeInt8(getByteArray(lib)));
333+
setNativePointer(byteArrayToNativeInt8(getByteArray(lib), true));
330334
}
331335
}
332336
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyContext.java‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -147,14 +147,14 @@
147147
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyContextFunctions.GraalHPyUnicodeFromString;
148148
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyContextFunctions.GraalHPyUnicodeFromWchar;
149149
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyContextFunctions.ReturnType;
150+
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodes.HPyAttachFunctionTypeNode;
150151
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodes.PCallHPyFunction;
151152
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodesFactory.HPyAsPythonObjectNodeGen;
152153
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodesFactory.HPyGetNativeSpacePointerNodeGen;
153154
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodesFactory.HPyRaiseNodeGen;
154155
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodesFactory.HPyTransformExceptionToNativeNodeGen;
155156
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodesFactory.PCallHPyFunctionNodeGen;
156157
import com.oracle.graal.python.builtins.objects.cext.hpy.HPyExternalFunctionNodes.HPyCheckFunctionResultNode;
157-
import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNodes.HPyAttachFunctionTypeNode;
158158
import com.oracle.graal.python.builtins.objects.common.EmptyStorage;
159159
import com.oracle.graal.python.builtins.objects.common.HashMapStorage;
160160
import com.oracle.graal.python.builtins.objects.common.HashingStorage;
@@ -1704,12 +1704,17 @@ final Object getMembers(@SuppressWarnings("unused") boolean includeInternal) {
17041704
@ExportMessage
17051705
@ImportStatic(HPyContextMember.class)
17061706
static class IsMemberReadable {
1707-
@Specialization(guards = "cachedKey.equals(key)")
1708-
static boolean isMemberReadable(@SuppressWarnings("unused") GraalHPyContext context, @SuppressWarnings("unused") String key,
1709-
@Cached(value = "key", allowUncached = true) @SuppressWarnings("unused") String cachedKey,
1710-
@Cached(value = "getIndex(key)", allowUncached = true) int cachedIdx) {
1707+
@Specialization(guards = "cachedKey.equals(key)", limit = "1")
1708+
static boolean isMemberReadableCached(@SuppressWarnings("unused") GraalHPyContext context, @SuppressWarnings("unused") String key,
1709+
@Cached(value = "key") @SuppressWarnings("unused") String cachedKey,
1710+
@Cached(value = "getIndex(key)") int cachedIdx) {
17111711
return cachedIdx != -1;
17121712
}
1713+
1714+
@Specialization(replaces = "isMemberReadableCached")
1715+
static boolean isMemberReadable(@SuppressWarnings("unused") GraalHPyContext context, String key) {
1716+
return HPyContextMember.getIndex(key) != -1;
1717+
}
17131718
}
17141719

17151720
@ExportMessage
@@ -1735,10 +1740,10 @@ abstract static class GraalHPyReadMemberNode extends Node {
17351740

17361741
public abstract Object execute(GraalHPyContext hpyContext, String key);
17371742

1738-
@Specialization(guards = "cachedKey.equals(key)")
1739-
static Object doMember(GraalHPyContext hpyContext, String key,
1740-
@Cached(value = "key", allowUncached = true) @SuppressWarnings("unused") String cachedKey,
1741-
@Cached(value = "getIndex(key)", allowUncached = true) int cachedIdx) {
1743+
@Specialization(guards = "cachedKey.equals(key)", limit = "1")
1744+
static Object doMemberCached(GraalHPyContext hpyContext, String key,
1745+
@Cached(value = "key") @SuppressWarnings("unused") String cachedKey,
1746+
@Cached(value = "getIndex(key)") int cachedIdx) {
17421747
// TODO(fa) once everything is implemented, remove this check
17431748
if (cachedIdx != -1) {
17441749
Object value = hpyContext.hpyContextMembers[cachedIdx];
@@ -1749,6 +1754,12 @@ static Object doMember(GraalHPyContext hpyContext, String key,
17491754
CompilerDirectives.transferToInterpreterAndInvalidate();
17501755
throw CompilerDirectives.shouldNotReachHere(String.format("context function %s not yet implemented: ", key));
17511756
}
1757+
1758+
@Specialization(replaces = "doMemberCached")
1759+
static Object doMember(GraalHPyContext hpyContext, String key,
1760+
@Cached(value = "key", allowUncached = true) @SuppressWarnings("unused") String cachedKey) {
1761+
return doMemberCached(hpyContext, key, key, HPyContextMember.getIndex(key));
1762+
}
17521763
}
17531764

17541765
@ExportMessage

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyNodes.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1850,6 +1850,7 @@ static Object doPointer(GraalHPyContext hpyContext, PythonNativeObject n, @Suppr
18501850
* int basicsize;
18511851
* int itemsize;
18521852
* unsigned int flags;
1853+
* int legacy;
18531854
* void *legacy_slots;
18541855
* HPyDef **defines;
18551856
* const char *doc;
@@ -1881,6 +1882,7 @@ static Object doGeneric(GraalHPyContext context, Object typeSpec, Object typeSpe
18811882
@Cached HPyCreateGetSetDescriptorNode createGetSetDescriptorNode,
18821883
@Cached GetSuperClassNode getSuperClassNode,
18831884
@Cached IsSameTypeNode isSameTypeNode,
1885+
@Cached ReadAttributeFromObjectNode readHPyTypeFlagsNode,
18841886
@Cached(parameters = "New") LookupCallableSlotInMRONode lookupNewNode,
18851887
@Cached HPyAsPythonObjectNode hPyAsPythonObjectNode,
18861888
@Cached PRaiseNode raiseNode) {
@@ -1930,6 +1932,13 @@ static Object doGeneric(GraalHPyContext context, Object typeSpec, Object typeSpe
19301932
throw raiseNode.raise(TypeError, "HPy_TPFLAGS_INTERNAL_PURE should not be used directly, set .legacy=true instead");
19311933
}
19321934

1935+
long legacy = castToLong(valueLib, ptrLib.readMember(typeSpec, "legacy"));
1936+
if (legacy != 0) {
1937+
flags &= ~GraalHPyDef.HPy_TPFLAGS_INTERNAL_PURE;
1938+
} else {
1939+
flags |= GraalHPyDef.HPy_TPFLAGS_INTERNAL_PURE;
1940+
}
1941+
19331942
long basicSize = castToLong(valueLib, ptrLib.readMember(typeSpec, "basicsize"));
19341943
long itemSize = castToLong(valueLib, ptrLib.readMember(typeSpec, "itemsize"));
19351944
writeAttributeToObjectNode.execute(newType, GraalHPyDef.TYPE_HPY_ITEMSIZE, itemSize);
@@ -1991,6 +2000,9 @@ static Object doGeneric(GraalHPyContext context, Object typeSpec, Object typeSpe
19912000
// process legacy slots; this is of type 'cpy_PyTypeSlot legacy_slots[]'
19922001
Object legacySlots = callHelperFunctionNode.call(context, GraalHPyNativeSymbol.GRAAL_HPY_TYPE_SPEC_GET_LEGECY_SLOTS, typeSpec);
19932002
if (!ptrLib.isNull(legacySlots)) {
2003+
if (legacy == 0) {
2004+
throw raiseNode.raise(TypeError, "cannot specify .legacy_slots without setting .legacy=true");
2005+
}
19942006
int nLegacySlots = PInt.intValueExact(ptrLib.getArraySize(legacySlots));
19952007
for (int i = 0; i < nLegacySlots; i++) {
19962008
Object legacySlotDef = ptrLib.readArrayElement(legacySlots, i);
@@ -2007,10 +2019,10 @@ static Object doGeneric(GraalHPyContext context, Object typeSpec, Object typeSpe
20072019
* constructors won't usually do that. So, we compute the constructor here and
20082020
* decorate it.
20092021
*/
2022+
Object baseClass = getSuperClassNode.execute(newType);
20102023
if (basicSize > 0 && !seenNew) {
20112024
Object inheritedConstructor = null;
20122025

2013-
Object baseClass = getSuperClassNode.execute(newType);
20142026
if (!isSameTypeNode.execute(baseClass, PythonBuiltinClassType.PythonObject)) {
20152027
// Lookup the inherited constructor and pass it to the HPy decorator.
20162028
inheritedConstructor = lookupNewNode.execute(baseClass);
@@ -2020,6 +2032,15 @@ static Object doGeneric(GraalHPyContext context, Object typeSpec, Object typeSpe
20202032
writeAttributeToObjectNode.execute(newType, SpecialMethodNames.__NEW__, constructorDecorator);
20212033
}
20222034

2035+
long baseFlags;
2036+
if (baseClass instanceof PythonClass) {
2037+
baseFlags = ((PythonClass) baseClass).flags;
2038+
} else {
2039+
Object baseFlagsObj = readHPyTypeFlagsNode.execute(baseClass, GraalHPyDef.TYPE_HPY_FLAGS);
2040+
baseFlags = baseFlagsObj != PNone.NO_VALUE ? (long) baseFlagsObj : 0;
2041+
}
2042+
checkInheritanceConstraints(flags, baseFlags, raiseNode);
2043+
20232044
return newType;
20242045
} catch (CannotCastException | InteropException e) {
20252046
throw raiseNode.raise(SystemError, "Could not create type from spec because: %m", e);
@@ -2100,6 +2121,24 @@ private static String[] splitName(String specName) {
21002121
return new String[]{null, specName};
21012122
}
21022123

2124+
private static void checkInheritanceConstraints(long flags, long baseFlags, PRaiseNode raiseNode) {
2125+
boolean isPure = (flags & GraalHPyDef.HPy_TPFLAGS_INTERNAL_PURE) != 0;
2126+
boolean isBasePure = (baseFlags & GraalHPyDef.HPy_TPFLAGS_INTERNAL_PURE) != 0;
2127+
// Pure types may inherit from:
2128+
//
2129+
// * pure types, or
2130+
// * PyBaseObject_Type, or
2131+
// * other builtin or legacy types as long as long as they do not
2132+
// access the struct layout (e.g. by using HPy_AsStruct or defining
2133+
// a deallocator with HPy_tp_destroy).
2134+
//
2135+
// It would be nice to relax these restrictions or check them here.
2136+
// See https://github.com/hpyproject/hpy/issues/169 for details.
2137+
if (!isPure && isBasePure) {
2138+
throw raiseNode.raise(TypeError, "A legacy type should not inherit its memory layout from a pure type");
2139+
}
2140+
}
2141+
21032142
private static long castToLong(InteropLibrary lib, Object value) throws OverflowException {
21042143
if (lib.fitsInLong(value)) {
21052144
try {

‎graalpython/lib-graalpython/modules/hpy/devel/__init__.py‎

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2222
# SOFTWARE.
2323

24+
import sys
2425
import os.path
2526
import functools
27+
import re
2628
from pathlib import Path
2729
from distutils import log
2830
from distutils.ccompiler import new_compiler
@@ -35,6 +37,9 @@
3537
# NOTE: this file is also imported by PyPy tests, so it must be compatible
3638
# with both Python 2.7 and Python 3.x
3739

40+
DEFAULT_HPY_ABI = 'universal'
41+
if hasattr(sys, 'implementation') and sys.implementation.name == 'cpython':
42+
DEFAULT_HPY_ABI = 'cpython'
3843

3944
class HPyDevel:
4045
""" Extra sources for building HPy extensions with hpy.devel. """
@@ -129,6 +134,17 @@ def fix_distribution(self, dist):
129134
base_build_ext = dist.cmdclass.get("build_ext", build_ext)
130135
orig_bdist_egg_write_stub = bdist_egg_mod.write_stub
131136

137+
if isinstance(base_build_ext, type):
138+
assert ('setuptools.command.build_ext', 'build_ext') in [
139+
(c.__module__, c.__name__) for c in base_build_ext.__mro__
140+
], (
141+
"dist.cmdclass['build_ext'] does not inherit from"
142+
" setuptools.command.build_ext.build_ext. The HPy build"
143+
" system does not currently support any other build_ext"
144+
" classes. If you are using distutils.commands.build_ext,"
145+
" please use setuptools.commands.build_ext instead."
146+
)
147+
132148
class build_hpy_ext(build_hpy_ext_mixin, base_build_ext, object):
133149
_base_build_ext = base_build_ext
134150

@@ -167,30 +183,30 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules):
167183
assert attr == 'hpy_ext_modules'
168184

169185
# add a global option --hpy-abi to setup.py
170-
dist.__class__.hpy_abi = 'cpython'
186+
dist.__class__.hpy_abi = DEFAULT_HPY_ABI
171187
dist.__class__.global_options += [
172-
('hpy-abi=', None, 'Specify the HPy ABI mode (default: cpython)')
188+
('hpy-abi=', None, 'Specify the HPy ABI mode (default: %s)' % DEFAULT_HPY_ABI)
173189
]
174190
hpydevel = HPyDevel()
175191
hpydevel.fix_distribution(dist)
176192

177193

178194
_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE = """
179-
class Spec:
180-
def __init__(self, name, origin):
181-
self.name = name
182-
self.origin = origin
183-
195+
# DO NOT EDIT THIS FILE!
196+
# This file is automatically generated by hpy
184197
185198
def __bootstrap__():
199+
186200
import sys, pkg_resources
187-
from hpy.universal import load_from_spec
201+
from hpy.universal import load
188202
ext_filepath = pkg_resources.resource_filename(__name__, {ext_file!r})
189-
m = load_from_spec(Spec({module_name!r}, ext_filepath))
203+
m = load({module_name!r}, ext_filepath)
190204
m.__file__ = ext_filepath
191205
m.__loader__ = __loader__
206+
m.__name__ = __name__
192207
m.__package__ = __package__
193208
m.__spec__ = __spec__
209+
m.__spec__.origin = ext_filepath
194210
sys.modules[__name__] = m
195211
196212
__bootstrap__()
@@ -280,9 +296,12 @@ def _finalize_hpy_ext(self, ext):
280296
if ext.hpy_abi == 'cpython':
281297
ext.extra_objects.append(self.hpydevel.get_ctx_lib())
282298
ext._hpy_needs_stub = False
283-
if ext.hpy_abi == 'universal':
299+
elif ext.hpy_abi == 'universal':
284300
ext.define_macros.append(('HPY_UNIVERSAL_ABI', None))
285301
ext._hpy_needs_stub = True
302+
else:
303+
raise DistutilsError('Unknown HPy ABI: %s. Valid values are: '
304+
'cpython, universal' % ext.hpy_abi)
286305

287306
def finalize_options(self):
288307
self._extensions = self.distribution.ext_modules or []
@@ -337,12 +356,23 @@ def write_stub(self, output_dir, ext, compile=False):
337356
log.info(
338357
"writing hpy universal stub loader for %s to %s",
339358
ext._full_name, stub_file)
340-
if compile and os.path.exists(stub_file):
341-
raise DistutilsError(stub_file + " already exists! Please delete.")
359+
342360
ext_file = os.path.basename(ext._file_name)
343361
module_name = ext_file.split(".")[0]
344362
if not self.dry_run:
345363
with open(stub_file, 'w') as f:
346364
f.write(_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format(
347365
ext_file=ext_file, module_name=module_name)
348366
)
367+
368+
def get_export_symbols(self, ext):
369+
""" Override .get_export_symbols to replace "PyInit_<module_name>"
370+
with "HPyInit_<module_name>.
371+
372+
Only relevant on Windows, where the .pyd file (DLL) must export the
373+
module "HPyInit_" function.
374+
"""
375+
exports = self._base_build_ext.get_export_symbols(self, ext)
376+
if hasattr(ext, "hpy_abi") and ext.hpy_abi == 'universal':
377+
exports = [re.sub(r"^PyInit_", "HPyInit_", name) for name in exports]
378+
return exports

‎graalpython/lib-graalpython/modules/hpy/devel/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@
2323

2424
# automatically generated by setup.py:get_scm_config()
2525
__version__ = "0.0.3"
26-
__git_revision__ = "33c8692"
26+
__git_revision__ = "2196f14"

‎graalpython/lib-graalpython/modules/hpy/test/test_00_basic.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,6 @@ def test_close(self):
151151
""")
152152
assert mod.f(41.5) == 42.5
153153

154-
# TODO: enable test once supported
155-
@pytest.mark.xfail
156154
def test_bool(self):
157155
mod = self.make_module("""
158156
HPyDef_METH(f, "f", f_impl, HPyFunc_O)
@@ -205,7 +203,6 @@ def test_varargs(self):
205203
""")
206204
assert mod.f(4, 5) == 45
207205

208-
@pytest.mark.skip
209206
def test_builtin_handles(self):
210207
mod = self.make_module("""
211208
HPyDef_METH(f, "f", f_impl, HPyFunc_O)
@@ -361,7 +358,6 @@ def test_repr_str_ascii_bytes(self):
361358
assert mod.f3("\u1234") == "'\\u1234'"
362359
assert mod.f4(bytearray(b"foo")) == b"foo"
363360

364-
@pytest.mark.xfail
365361
def test_is_true(self):
366362
mod = self.make_module("""
367363
HPyDef_METH(f, "f", f_impl, HPyFunc_O)

0 commit comments

Comments
 (0)