From 62f2340ad341026af5d0f6284dfe611418cedf11 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 26 Mar 2022 16:00:35 -0700 Subject: [PATCH 1/9] Array4: __cuda_array_interface__ v2 Start implementing the `__cuda_array_interface__` for zero-copy data exchange on Nvidia CUDA GPUs. --- src/Base/Array4.cpp | 119 ++++++++++++++++++++++++++++------------- tests/test_array4.py | 113 ++++++++++++++++++++++++++------------ tests/test_multifab.py | 53 ++++++++++++++++++ 3 files changed, 214 insertions(+), 71 deletions(-) diff --git a/src/Base/Array4.cpp b/src/Base/Array4.cpp index 783e9cfb..f23feefc 100644 --- a/src/Base/Array4.cpp +++ b/src/Base/Array4.cpp @@ -19,6 +19,61 @@ namespace py = pybind11; using namespace amrex; +namespace +{ + /** CPU: __array_interface__ v3 + * + * https://numpy.org/doc/stable/reference/arrays.interface.html + */ + template + py::dict + array_interface(Array4 const & a4) + { + auto d = py::dict(); + auto const len = length(a4); + // F->C index conversion here + // p[(i-begin.x)+(j-begin.y)*jstride+(k-begin.z)*kstride+n*nstride]; + // Buffer dimensions: zero-size shall not skip dimension + auto shape = py::make_tuple( + a4.ncomp, + len.z <= 0 ? 1 : len.z, + len.y <= 0 ? 1 : len.y, + len.x <= 0 ? 1 : len.x // fastest varying index + ); + // buffer protocol strides are in bytes, AMReX strides are elements + auto const strides = py::make_tuple( + sizeof(T) * a4.nstride, + sizeof(T) * a4.kstride, + sizeof(T) * a4.jstride, + sizeof(T) // fastest varying index + ); + bool const read_only = false; + d["data"] = py::make_tuple(std::intptr_t(a4.dataPtr()), read_only); + // note: if we want to keep the same global indexing with non-zero + // box small_end as in AMReX, then we can explore playing with + // this offset as well + //d["offset"] = 0; // default + //d["mask"] = py::none(); // default + + d["shape"] = shape; + // we could also set this after checking the strides are C-style contiguous: + //if (is_contiguous(shape, strides)) + // d["strides"] = py::none(); // C-style contiguous + //else + d["strides"] = strides; + + // type description + // for more complicated types, e.g., tuples/structs + //d["descr"] = ...; + // we currently only need this + d["typestr"] = py::format_descriptor::format(); + + d["version"] = 3; + return d; + } +} + + template< typename T > void make_Array4(py::module &m, std::string typestr) { @@ -85,56 +140,44 @@ void make_Array4(py::module &m, std::string typestr) return a4; })) + + // CPU: __array_interface__ v3 + // https://numpy.org/doc/stable/reference/arrays.interface.html .def_property_readonly("__array_interface__", [](Array4 const & a4) { - auto d = py::dict(); - auto const len = length(a4); - // F->C index conversion here - // p[(i-begin.x)+(j-begin.y)*jstride+(k-begin.z)*kstride+n*nstride]; - // Buffer dimensions: zero-size shall not skip dimension - auto shape = py::make_tuple( - a4.ncomp, - len.z <= 0 ? 1 : len.z, - len.y <= 0 ? 1 : len.y, - len.x <= 0 ? 1 : len.x // fastest varying index - ); - // buffer protocol strides are in bytes, AMReX strides are elements - auto const strides = py::make_tuple( - sizeof(T) * a4.nstride, - sizeof(T) * a4.kstride, - sizeof(T) * a4.jstride, - sizeof(T) // fastest varying index - ); - bool const read_only = false; - d["data"] = py::make_tuple(std::intptr_t(a4.dataPtr()), read_only); - // note: if we want to keep the same global indexing with non-zero - // box small_end as in AMReX, then we can explore playing with - // this offset as well - //d["offset"] = 0; // default - //d["mask"] = py::none(); // default - - d["shape"] = shape; - // we could also set this after checking the strides are C-style contiguous: - //if (is_contiguous(shape, strides)) - // d["strides"] = py::none(); // C-style contiguous - //else - d["strides"] = strides; - - d["typestr"] = py::format_descriptor::format(); - d["version"] = 3; - return d; + return array_interface(a4); }) + // CPU: __array_function__ interface (TODO) + // + // NEP 18 — A dispatch mechanism for NumPy's high level array functions. + // https://numpy.org/neps/nep-0018-array-function-protocol.html + // This enables code using NumPy to be directly operated on Array4 arrays. + // __array_function__ feature requires NumPy 1.16 or later. + - // TODO: __cuda_array_interface__ + // Nvidia GPUs: __cuda_array_interface__ v2 // https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html + .def_property_readonly("__cuda_array_interface__", [](Array4 const & a4) { + auto d = array_interface(a4); + + // data: + // Because the user of the interface may or may not be in the same context, the most common case is to use cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_DEVICE_POINTER in the CUDA driver API (or the equivalent CUDA Runtime API) to retrieve a device pointer that is usable in the currently active context. + // TODO For zero-size arrays, use 0 here. + + // ... TODO: wasn't there some stream or device info? + + d["version"] = 2; + return d; + }) - // TODO: __dlpack__ + // TODO: __dlpack__ __dlpack_device__ // DLPack protocol (CPU, NVIDIA GPU, AMD GPU, Intel GPU, etc.) // https://dmlc.github.io/dlpack/latest/ // https://data-apis.org/array-api/latest/design_topics/data_interchange.html // https://github.com/data-apis/consortium-feedback/issues/1 // https://github.com/dmlc/dlpack/blob/master/include/dlpack/dlpack.h + // https://docs.cupy.dev/en/stable/user_guide/interoperability.html#dlpack-data-exchange-protocol .def("contains", &Array4::contains) diff --git a/tests/test_array4.py b/tests/test_array4.py index 44f295c3..4fe72471 100644 --- a/tests/test_array4.py +++ b/tests/test_array4.py @@ -69,36 +69,83 @@ def test_array4(): x[1, 1, 1] = 44 assert v_carr2np[0, 1, 1, 1] == 44 - # from cupy - - # to numpy - - # to cupy - - return - - # Check indexing - assert obj[0] == 1 - assert obj[1] == 2 - assert obj[2] == 3 - assert obj[-1] == 3 - assert obj[-2] == 2 - assert obj[-3] == 1 - with pytest.raises(IndexError): - obj[-4] - with pytest.raises(IndexError): - obj[3] - - # Check assignment - obj[0] = 2 - obj[1] = 3 - obj[2] = 4 - assert obj[0] == 2 - assert obj[1] == 3 - assert obj[2] == 4 - - -# def test_iv_conversions(): -# obj = amrex.IntVect.max_vector().numpy() -# assert(isinstance(obj, np.ndarray)) -# assert(obj.dtype == np.int32) + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_array4_numba(): + # https://numba.pydata.org/numba-doc/dev/cuda/cuda_array_interface.html + import numba + + # numba -> AMReX Array4 + x = np.ones( + ( + 2, + 3, + 4, + ) + ) # type: numpy.ndarray + + # host-to-device copy + x_numba = numba.cuda.to_device( + x + ) # type: numba.cuda.cudadrv.devicearray.DeviceNDArray + # x_cupy = cupy.asarray(x_numba) # type: cupy.ndarray + x_arr = amrex.Array4_double(x_numba) # type: amrex.Array4_double + + assert ( + x_arr.__cuda_array_interface__["data"][0] + == x_numba.__cuda_array_interface__["data"][0] + ) + + # AMReX -> numba + # arr_numba = cuda.as_cuda_array(arr4) + # ... or as MultiFab test + # TODO + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_array4_cupy(): + # https://docs.cupy.dev/en/stable/user_guide/interoperability.html + import cupy as cp + + # cupy -> AMReX Array4 + x = np.ones( + ( + 2, + 3, + 4, + ) + ) # TODO: merge into next line and create on device? + x_cupy = cp.asarray(x) # type: cupy.ndarray + print(f"x_cupy={x_cupy}") + print(x_cupy.__cuda_array_interface__) + + # cupy -> AMReX array4 + x_arr = amrex.Array4_double(x_cupy) # type: amrex.Array4_double + print(f"x_arr={x_arr}") + print(x_arr.__cuda_array_interface__) + + assert ( + x_arr.__cuda_array_interface__["data"][0] + == x_cupy.__cuda_array_interface__["data"][0] + ) + + # AMReX -> cupy + # arr_numba = cuda.as_cuda_array(arr4) + # ... or as MultiFab test + # TODO + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_array4_pytorch(): + # https://docs.cupy.dev/en/stable/user_guide/interoperability.html#pytorch + # arr_torch = torch.as_tensor(arr, device='cuda') + # assert(arr_torch.__cuda_array_interface__['data'][0] == arr.__cuda_array_interface__['data'][0]) + # TODO + + pass diff --git a/tests/test_multifab.py b/tests/test_multifab.py index fc92c912..59e09738 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -153,3 +153,56 @@ def test_mfab_mfiter(make_mfab): cnt += 1 assert iter(mfab).length == cnt + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_mfab_ops_cuda_numba(): + # https://numba.pydata.org/numba-doc/dev/cuda/cuda_array_interface.html + import numba + + # AMReX -> numba + # arr_numba = cuda.as_cuda_array(arr4) + # TODO + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_mfab_ops_cuda_cupy(): + # https://docs.cupy.dev/en/stable/user_guide/interoperability.html + import cupy as cp + + # AMReX -> cupy + # arr_numba = cuda.as_cuda_array(arr4) + # TODO + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_mfab_ops_cuda_pytorch(): + # https://docs.cupy.dev/en/stable/user_guide/interoperability.html#pytorch + import torch + + # AMReX -> pytorch + # arr_torch = torch.as_tensor(arr, device='cuda') + # assert(arr_torch.__cuda_array_interface__['data'][0] == arr.__cuda_array_interface__['data'][0]) + # TODO + + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +def test_mfab_ops_cuda_cuml(): + # https://github.com/rapidsai/cuml + # https://github.com/rapidsai/cudf + # maybe better for particles as a dataframe test + import cudf + import cuml + + # AMReX -> RAPIDSAI cuML + # arr_cuml = ... + # assert(arr_cuml.__cuda_array_interface__['data'][0] == arr.__cuda_array_interface__['data'][0]) + # TODO From 271a0213092bb6b7f9d0c5242eac3763c39a30d5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 6 Oct 2022 18:10:59 -0700 Subject: [PATCH 2/9] MultiFab: CuPy Test --- tests/conftest.py | 26 ++++++++++++ tests/test_multifab.py | 89 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d06729be..9ca60c17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,3 +84,29 @@ def create(): return mfab return create + +@pytest.mark.skipif( + amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" +) +@pytest.fixture(scope="function", params=list(itertools.product([1, 3], [0, 1]))) +def make_mfab_device(boxarr, distmap, request): + """MultiFab that resides purely on the device: + The MultiFab object itself is not a fixture because we want to avoid caching + it between amrex.initialize/finalize calls of various tests. + https://github.com/pytest-dev/pytest/discussions/10387 + https://github.com/pytest-dev/pytest/issues/5642#issuecomment-1279612764 + """ + + def create(): + num_components = request.param[0] + num_ghost = request.param[1] + mfab = amrex.MultiFab( + boxarr, + distmap, + num_components, + num_ghost, + amrex.MFInfo().set_arena(amrex.The_Device_Arena()), + ) + mfab.set_val(0.0, 0, num_components) + + return create diff --git a/tests/test_multifab.py b/tests/test_multifab.py index 59e09738..431de075 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -170,19 +170,97 @@ def test_mfab_ops_cuda_numba(): @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) -def test_mfab_ops_cuda_cupy(): +@pytest.mark.parametrize("nghost", [0, 1]) +def test_mfab_ops_cuda_cupy(make_mfab_device, nghost): + mfab_device = make_mfab_device() # https://docs.cupy.dev/en/stable/user_guide/interoperability.html import cupy as cp + import cupy.prof # AMReX -> cupy - # arr_numba = cuda.as_cuda_array(arr4) - # TODO + ngv = mfab_device.nGrowVect + print(f"\n mfab_device={mfab_device}, mfab_device.nGrowVect={ngv}") + + # assign 3 + with cupy.prof.time_range("assign 3 [()]", color_id=0): + for mfi in mfab_device: + bx = mfi.tilebox().grow(ngv) + marr = mfab_device.array(mfi) + marr_cupy = cp.array(marr, copy=False) + # print(marr_cupy.shape) # 1, 32, 32, 32 + # print(marr_cupy.dtype) # float64 + + # write and read into the marr_cupy + marr_cupy[()] = 3.0 + + # verify result with a .sum_unique + with cupy.prof.time_range("verify 3", color_id=0): + shape = 32**3 * 8 + # print(mfab_device.shape) + sum_threes = mfab_device.sum_unique(comp=0, local=False) + assert sum_threes == shape * 3 + + # assign 2 + with cupy.prof.time_range("assign 2 (set_val)", color_id=1): + mfab_device.set_val(2.0) + with cupy.prof.time_range("verify 2", color_id=1): + sum_twos = mfab_device.sum_unique(comp=0, local=False) + assert sum_twos == shape * 2 + + # assign 5 + with cupy.prof.time_range("assign 5 (ones-like)", color_id=2): + + def set_to_five(mm): + xp = cp.get_array_module(mm) + assert xp.__name__ == "cupy" + mm = xp.ones_like(mm) * 10.0 + mm /= 2.0 + return mm + + for mfi in mfab_device: + bx = mfi.tilebox().grow(ngv) + marr = mfab_device.array(mfi) + marr_cupy = cp.array(marr, copy=False) + + # write and read into the marr_cupy + fives_cp = set_to_five(marr_cupy) + marr_cupy[()] = 0.0 + marr_cupy += fives_cp + + # verify + with cupy.prof.time_range("verify 5", color_id=2): + sum = mfab_device.sum_unique(comp=0, local=False) + assert sum == shape * 5 + + # assign 7 + with cupy.prof.time_range("assign 7 (fuse)", color_id=3): + + @cp.fuse(kernel_name="set_to_seven") + def set_to_seven(x): + x += 7.0 + + for mfi in mfab_device: + bx = mfi.tilebox().grow(ngv) + marr = mfab_device.array(mfi) + marr_cupy = cp.array(marr, copy=False) + + # write and read into the marr_cupy + marr_cupy[()] = 0.0 + set_to_seven(marr_cupy) + + # verify + with cupy.prof.time_range("verify 7", color_id=3): + sum = mfab_device.sum_unique(comp=0, local=False) + assert sum == shape * 7 + + # TODO: @jit.rawkernel() @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) -def test_mfab_ops_cuda_pytorch(): +def test_mfab_ops_cuda_pytorch(make_mfab_device): + mfab_device = make_mfab_device() # https://docs.cupy.dev/en/stable/user_guide/interoperability.html#pytorch import torch @@ -195,7 +273,8 @@ def test_mfab_ops_cuda_pytorch(): @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) -def test_mfab_ops_cuda_cuml(): +def test_mfab_ops_cuda_cuml(make_mfab_device): + mfab_device = make_mfab_device() # https://github.com/rapidsai/cuml # https://github.com/rapidsai/cudf # maybe better for particles as a dataframe test From 5acd36ae3df77c4901ff421de814145335c6a2da Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 11 Oct 2022 16:44:04 -0700 Subject: [PATCH 3/9] `MFIter`: `Finalize()` on `StopIteration` Since `for` loops create no scope in Python, we need to trigger finalize logic, including stream syncs, before the destructor of `MultiFab` iterators are called. --- src/Base/MultiFab.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Base/MultiFab.cpp b/src/Base/MultiFab.cpp index 9c354a37..007a4406 100644 --- a/src/Base/MultiFab.cpp +++ b/src/Base/MultiFab.cpp @@ -94,6 +94,7 @@ void init_MultiFab(py::module &m) { if( !mfi.isValid() ) { first_or_done = true; + mfi.Finalize(); throw py::stop_iteration(); } return mfi; From 6965f9a95541bc66d4501598b7dc0135c05a6e27 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 14 Oct 2022 11:05:02 -0700 Subject: [PATCH 4/9] Add numba test incl. 3D kernel launch --- tests/test_multifab.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/tests/test_multifab.py b/tests/test_multifab.py index 431de075..cfa87ed1 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import math + import numpy as np import pytest @@ -158,13 +160,37 @@ def test_mfab_mfiter(make_mfab): @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) -def test_mfab_ops_cuda_numba(): +def test_mfab_ops_cuda_numba(make_mfab_device): + mfab_device = make_mfab_device() # https://numba.pydata.org/numba-doc/dev/cuda/cuda_array_interface.html - import numba + from numba import cuda - # AMReX -> numba - # arr_numba = cuda.as_cuda_array(arr4) - # TODO + ngv = mfab_device.nGrowVect + + # assign 3: define kernel + @cuda.jit + def set_to_three(array): + i, j, k = cuda.grid(3) + if i < array.shape[0] and j < array.shape[1] and k < array.shape[2]: + array[i, j, k] = 3.0 + + # assign 3: loop through boxes and launch kernels + for mfi in mfab_device: + bx = mfi.tilebox().grow(ngv) + marr = mfab_device.array(mfi) + marr_numba = cuda.as_cuda_array(marr) + + # kernel launch + threadsperblock = (4, 4, 4) + blockspergrid = tuple( + [math.ceil(s / b) for s, b in zip(marr_numba.shape, threadsperblock)] + ) + set_to_three[blockspergrid, threadsperblock](marr_numba) + + # Check results + shape = 32**3 * 8 + sum_threes = mfab_device.sum_unique(comp=0, local=False) + assert sum_threes == shape * 3 @pytest.mark.skipif( From 9f539f918792b169b267992732942b6c5d4008fb Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 14 Oct 2022 16:07:04 -0700 Subject: [PATCH 5/9] Add pytorch --- tests/test_multifab.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_multifab.py b/tests/test_multifab.py index cfa87ed1..3fc298f2 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -290,10 +290,16 @@ def test_mfab_ops_cuda_pytorch(make_mfab_device): # https://docs.cupy.dev/en/stable/user_guide/interoperability.html#pytorch import torch - # AMReX -> pytorch - # arr_torch = torch.as_tensor(arr, device='cuda') - # assert(arr_torch.__cuda_array_interface__['data'][0] == arr.__cuda_array_interface__['data'][0]) - # TODO + # assign 3: loop through boxes and launch kernel + for mfi in mfab_device: + marr = mfab_device.array(mfi) + marr_torch = torch.as_tensor(marr, device="cuda") + marr_torch[:, :, :] = 3 + + # Check results + shape = 32**3 * 8 + sum_threes = mfab_device.sum_unique(comp=0, local=False) + assert sum_threes == shape * 3 @pytest.mark.skipif( From 4175194045ad4bbc5da6ae5136494482fa59e287 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 14 Oct 2022 16:47:00 -0700 Subject: [PATCH 6/9] CuPy Fuse: Avoid Extra Memset --- tests/test_multifab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_multifab.py b/tests/test_multifab.py index 3fc298f2..23ef37ab 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -263,7 +263,7 @@ def set_to_five(mm): @cp.fuse(kernel_name="set_to_seven") def set_to_seven(x): - x += 7.0 + x[...] = 7.0 for mfi in mfab_device: bx = mfi.tilebox().grow(ngv) @@ -271,7 +271,6 @@ def set_to_seven(x): marr_cupy = cp.array(marr, copy=False) # write and read into the marr_cupy - marr_cupy[()] = 0.0 set_to_seven(marr_cupy) # verify From 6eb2da4983db484971a05abab747497c1930c51b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 16 Oct 2022 21:34:14 -0700 Subject: [PATCH 7/9] MultiFab Device Test: Fixes --- tests/conftest.py | 2 ++ tests/test_multifab.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9ca60c17..6c3079c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,6 +85,7 @@ def create(): return create + @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) @@ -108,5 +109,6 @@ def create(): amrex.MFInfo().set_arena(amrex.The_Device_Arena()), ) mfab.set_val(0.0, 0, num_components) + return mfab return create diff --git a/tests/test_multifab.py b/tests/test_multifab.py index 23ef37ab..79b14224 100644 --- a/tests/test_multifab.py +++ b/tests/test_multifab.py @@ -196,8 +196,7 @@ def set_to_three(array): @pytest.mark.skipif( amrex.Config.gpu_backend != "CUDA", reason="Requires AMReX_GPU_BACKEND=CUDA" ) -@pytest.mark.parametrize("nghost", [0, 1]) -def test_mfab_ops_cuda_cupy(make_mfab_device, nghost): +def test_mfab_ops_cuda_cupy(make_mfab_device): mfab_device = make_mfab_device() # https://docs.cupy.dev/en/stable/user_guide/interoperability.html import cupy as cp From 7f6d80bba3981f3afcab1093e4418d7229344c5c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 16 Oct 2022 21:45:24 -0700 Subject: [PATCH 8/9] Update to v3 --- src/Base/Array4.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Base/Array4.cpp b/src/Base/Array4.cpp index f23feefc..726a50a7 100644 --- a/src/Base/Array4.cpp +++ b/src/Base/Array4.cpp @@ -164,9 +164,16 @@ void make_Array4(py::module &m, std::string typestr) // Because the user of the interface may or may not be in the same context, the most common case is to use cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_DEVICE_POINTER in the CUDA driver API (or the equivalent CUDA Runtime API) to retrieve a device pointer that is usable in the currently active context. // TODO For zero-size arrays, use 0 here. - // ... TODO: wasn't there some stream or device info? - - d["version"] = 2; + // None or integer + // An optional stream upon which synchronization must take place at the point of consumption, either by synchronizing on the stream or enqueuing operations on the data on the given stream. Integer values in this entry are as follows: + // 0: This is disallowed as it would be ambiguous between None and the default stream, and also between the legacy and per-thread default streams. Any use case where 0 might be given should either use None, 1, or 2 instead for clarity. + // 1: The legacy default stream. + // 2: The per-thread default stream. + // Any other integer: a cudaStream_t represented as a Python integer. + // When None, no synchronization is required. + d["stream"] = py::none(); + + d["version"] = 3; return d; }) From e65cd411b99122f06756c5f42a0513b6db8633d0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 17 Oct 2022 00:03:30 -0700 Subject: [PATCH 9/9] Array4: TODO from CUDA A bit tricky to implement this caster as new constructor. Not currently needed, but adds comments where to do this. --- src/Base/Array4.cpp | 4 ++++ tests/test_array4.py | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Base/Array4.cpp b/src/Base/Array4.cpp index 726a50a7..623af5b7 100644 --- a/src/Base/Array4.cpp +++ b/src/Base/Array4.cpp @@ -140,6 +140,10 @@ void make_Array4(py::module &m, std::string typestr) return a4; })) + /* init from __cuda_array_interface__: non-owning view + * TODO + */ + // CPU: __array_interface__ v3 // https://numpy.org/doc/stable/reference/arrays.interface.html diff --git a/tests/test_array4.py b/tests/test_array4.py index 4fe72471..25efbb40 100644 --- a/tests/test_array4.py +++ b/tests/test_array4.py @@ -75,7 +75,7 @@ def test_array4(): ) def test_array4_numba(): # https://numba.pydata.org/numba-doc/dev/cuda/cuda_array_interface.html - import numba + from numba import cuda # numba -> AMReX Array4 x = np.ones( @@ -87,9 +87,7 @@ def test_array4_numba(): ) # type: numpy.ndarray # host-to-device copy - x_numba = numba.cuda.to_device( - x - ) # type: numba.cuda.cudadrv.devicearray.DeviceNDArray + x_numba = cuda.to_device(x) # type: numba.cuda.cudadrv.devicearray.DeviceNDArray # x_cupy = cupy.asarray(x_numba) # type: cupy.ndarray x_arr = amrex.Array4_double(x_numba) # type: amrex.Array4_double