From ca87b6b11de2432b5b7c11a7a217ef5e555d8671 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 12 Jul 2023 15:38:38 -0700 Subject: [PATCH 1/9] MultiFab tutorial with pyamrex --- GuidedTutorials/MultiFab/main.py | 84 ++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 GuidedTutorials/MultiFab/main.py diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py new file mode 100644 index 00000000..174eac79 --- /dev/null +++ b/GuidedTutorials/MultiFab/main.py @@ -0,0 +1,84 @@ +import numpy as np +import amrex.space3d as amr + +# TODO How do we print pyamrex's version? +print("Hello world from pyamrex\n") + +# Initialize AMReX +amr.initialize([]) + +# Goals: +# * Define a MultiFab +# * Fill a MultiFab with data +# * Plot it + +# Parameters + +# Number of data components at each grid point in the MultiFab +ncomp = 1 +# How many grid cells in each direction over the problem domain +n_cell = 32 +# How many grid cells are allowed in each direction over each box +max_grid_size = 16 + +# BoxArray: abstract domain setup + +# Integer vector indicating the lower coordinate bounds +dom_lo = amr.IntVect(0,0,0) +# Integer vector indicating the upper coordinate bounds +dom_hi = amr.IntVect(n_cell-1, n_cell-1, n_cell-1) +# Box containing the coordinates of this domain +domain = amr.Box(dom_lo, dom_hi) + +# Will contain a list of boxes describing the problem domain +ba = amr.BoxArray(domain) + +# Chop the single grid into many small boxes +ba.max_size(max_grid_size) + +# Distribution Mapping +dm = amr.DistributionMapping(ba) + +# Define MultiFab +mf = amr.MultiFab(ba, dm, ncomp, 0) +mf.set_val(0.) + +# Geometry: physical properties for data on our domain +real_box = amr.RealBox([0., 0., 0.], [1. , 1., 1.]) + +# TODO Should this be &real_box (reference) instead? +coord = 0 # Cartesian +is_per = [0, 0, 0] # periodicity +geom = amr.Geometry(domain, real_box, coord, is_per) + +# Calculate cell sizes +dx = geom.data().CellSize() # dx[0]=dx dx[1]=dy dx[2]=dz + +# Fill a MultiFab with data +mf_val = 0. +for mfi in mf: + bx = mfi.validbox() + mf_array = mf.array(mfi) + for i, j, k in bx: + x = (i+0.5) * dx[0] + y = (j+0.5) * dx[1] + z = (k+0.5) * dx[2] + r_squared = ((x-0.5)*(x-0.5)+(y-0.5)*(y-0.5)+(z-0.5)*(z-0.5))/0.01 + mf_array[i,j,k,0] = 1.0 + np.exp(-r_squared) + # Check that mf_array has been filled correctly for a given set of indices + ii = bx.small_end[0] + jj = bx.small_end[1] + kk = bx.small_end[2] + xx = (ii+0.5) * dx[0] + yy = (jj+0.5) * dx[1] + zz = (kk+0.5) * dx[2] + rr_squared = ((xx-0.5)*(xx-0.5)+(yy-0.5)*(yy-0.5)+(zz-0.5)*(zz-0.5))/0.01 + mf_refval = 1.0 + np.exp(-rr_squared) + assert(mf_array[i,j,k,0] == mf_refval) + +# TODO How do we write plotfiles? +# Plot MultiFab data +# WriteSingleLevelPlotfile("plt001", mf, {"comp0"}, geom, 0., 0); + +# Finalize AMReX +amr.finalize() From e61d3603fb8ef18d131042a458cf220ca74c0a98 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 19 Jul 2023 15:06:05 -0700 Subject: [PATCH 2/9] Replace `for` loop with NumPy operations --- GuidedTutorials/MultiFab/main.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py index 174eac79..7d1fc880 100644 --- a/GuidedTutorials/MultiFab/main.py +++ b/GuidedTutorials/MultiFab/main.py @@ -58,23 +58,26 @@ mf_val = 0. for mfi in mf: bx = mfi.validbox() - mf_array = mf.array(mfi) - for i, j, k in bx: - x = (i+0.5) * dx[0] - y = (j+0.5) * dx[1] - z = (k+0.5) * dx[2] - r_squared = ((x-0.5)*(x-0.5)+(y-0.5)*(y-0.5)+(z-0.5)*(z-0.5))/0.01 - mf_array[i,j,k,0] = 1.0 + np.exp(-r_squared) + # mf_array is indexed in reversed order (z,y,x) and indices are local, + # that is, indices range from 0 to the box size + mf_array = np.array(mf.array(mfi), copy=False) + x = (np.arange(bx.small_end[0], bx.big_end[0])+0.5)*dx[0] + y = (np.arange(bx.small_end[1], bx.big_end[1])+0.5)*dx[1] + z = (np.arange(bx.small_end[2], bx.big_end[2])+0.5)*dx[2] + v = (x[np.newaxis,np.newaxis,:]*10 + + y[np.newaxis,:,np.newaxis]*100 + + z[:,np.newaxis,np.newaxis]*1000) + mf_array = 1. + np.exp(-v) # Check that mf_array has been filled correctly for a given set of indices ii = bx.small_end[0] jj = bx.small_end[1] kk = bx.small_end[2] - xx = (ii+0.5) * dx[0] - yy = (jj+0.5) * dx[1] - zz = (kk+0.5) * dx[2] - rr_squared = ((xx-0.5)*(xx-0.5)+(yy-0.5)*(yy-0.5)+(zz-0.5)*(zz-0.5))/0.01 - mf_refval = 1.0 + np.exp(-rr_squared) - assert(mf_array[i,j,k,0] == mf_refval) + xx = (ii+0.5)*dx[0] + yy = (jj+0.5)*dx[1] + zz = (kk+0.5)*dx[2] + vv = xx*10 + yy*100 + zz*1000 + mf_refval = 1.0 + np.exp(-vv) + assert(mf_array[0,0,0] == mf_refval) # TODO How do we write plotfiles? # Plot MultiFab data From 193c499bead6f53f65e52b07b370551aad3539a7 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 4 Aug 2023 16:39:28 -0700 Subject: [PATCH 3/9] Print only from IO process, write plotfile --- GuidedTutorials/MultiFab/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py index 7d1fc880..7ec14836 100644 --- a/GuidedTutorials/MultiFab/main.py +++ b/GuidedTutorials/MultiFab/main.py @@ -1,12 +1,13 @@ import numpy as np import amrex.space3d as amr -# TODO How do we print pyamrex's version? -print("Hello world from pyamrex\n") - # Initialize AMReX amr.initialize([]) +# TODO How do we print pyamrex's version? +if amr.ParallelDescriptor.IOProcessor(): + print("Hello world from pyamrex\n") + # Goals: # * Define a MultiFab # * Fill a MultiFab with data @@ -79,9 +80,10 @@ mf_refval = 1.0 + np.exp(-vv) assert(mf_array[0,0,0] == mf_refval) -# TODO How do we write plotfiles? # Plot MultiFab data -# WriteSingleLevelPlotfile("plt001", mf, {"comp0"}, geom, 0., 0); +plotfile = amr.concatenate(root="plt", num=1, mindigits=3) +varnames = amr.Vector_string(["comp0"]) +amr.write_single_level_plotfile(plotfile, mf, varnames, geom, time=0., level_step=0) # Finalize AMReX amr.finalize() From e72e171b42784d5c20e49cef4fc8746fcfb44e33 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 4 Aug 2023 17:20:05 -0700 Subject: [PATCH 4/9] Two methods to fill array (NumPy, Array4) --- GuidedTutorials/MultiFab/main.py | 41 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py index 7ec14836..50254e37 100644 --- a/GuidedTutorials/MultiFab/main.py +++ b/GuidedTutorials/MultiFab/main.py @@ -59,26 +59,31 @@ mf_val = 0. for mfi in mf: bx = mfi.validbox() - # mf_array is indexed in reversed order (z,y,x) and indices are local, - # that is, indices range from 0 to the box size + # Preferred way to fill array using NumPy operations: + # - mf_array is indexed in reversed order (n,z,y,x) + # - indices are local (range from 0 to box size) mf_array = np.array(mf.array(mfi), copy=False) - x = (np.arange(bx.small_end[0], bx.big_end[0])+0.5)*dx[0] - y = (np.arange(bx.small_end[1], bx.big_end[1])+0.5)*dx[1] - z = (np.arange(bx.small_end[2], bx.big_end[2])+0.5)*dx[2] - v = (x[np.newaxis,np.newaxis,:]*10 - + y[np.newaxis,:,np.newaxis]*100 - + z[:,np.newaxis,np.newaxis]*1000) + x = (np.arange(bx.small_end[0], bx.big_end[0]+1)+0.5)*dx[0] + y = (np.arange(bx.small_end[1], bx.big_end[1]+1)+0.5)*dx[1] + z = (np.arange(bx.small_end[2], bx.big_end[2]+1)+0.5)*dx[2] + v = (x[np.newaxis,np.newaxis,:] + + y[np.newaxis,:,np.newaxis]*0.1 + + z[:,np.newaxis,np.newaxis]*0.01) mf_array = 1. + np.exp(-v) - # Check that mf_array has been filled correctly for a given set of indices - ii = bx.small_end[0] - jj = bx.small_end[1] - kk = bx.small_end[2] - xx = (ii+0.5)*dx[0] - yy = (jj+0.5)*dx[1] - zz = (kk+0.5)*dx[2] - vv = xx*10 + yy*100 + zz*1000 - mf_refval = 1.0 + np.exp(-vv) - assert(mf_array[0,0,0] == mf_refval) + # Alternative way to fill array using standard for loop over box indices: + # - mf_array_ref is indexed in standard order (x,y,z,n) + # - indices are global + mf_array_ref = mf.array(mfi) + for i, j, k in bx: + x = (i+0.5)*dx[0] + y = (j+0.5)*dx[1] + z = (k+0.5)*dx[2] + v = x + y*0.1 + z*0.01 + mf_array_ref[i,j,k,0] = 1. + np.exp(-v) + # Check that mf_array has been filled correctly, compare it with + # mf_array_ref (filled as Array4 and transformed to NumPy array) + mf_array_ref_np = np.array(mf_array_ref, copy=False) + assert np.allclose(mf_array, mf_array_ref_np, rtol=1e-16, atol=1e-16) # Plot MultiFab data plotfile = amr.concatenate(root="plt", num=1, mindigits=3) From 7c5010e763b85c1bdc0e34d05a1d26a836a9805e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 11 Aug 2023 14:42:05 -0700 Subject: [PATCH 5/9] Finalize TODOs --- GuidedTutorials/MultiFab/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py index 50254e37..dd186069 100644 --- a/GuidedTutorials/MultiFab/main.py +++ b/GuidedTutorials/MultiFab/main.py @@ -4,9 +4,8 @@ # Initialize AMReX amr.initialize([]) -# TODO How do we print pyamrex's version? if amr.ParallelDescriptor.IOProcessor(): - print("Hello world from pyamrex\n") + print(f"Hello world from pyAMReX version {amr.__version__}\n") # Goals: # * Define a MultiFab @@ -47,7 +46,6 @@ # Geometry: physical properties for data on our domain real_box = amr.RealBox([0., 0., 0.], [1. , 1., 1.]) -# TODO Should this be &real_box (reference) instead? coord = 0 # Cartesian is_per = [0, 0, 0] # periodicity geom = amr.Geometry(domain, real_box, coord, is_per) From 3ff1f80293f50bf43b4b2451ec03b7883225bf70 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 11 Aug 2023 15:20:18 -0700 Subject: [PATCH 6/9] Docs: Python --- Docs/source/Python_Tutorial.rst | 17 +++++++++++++++++ Docs/source/index.rst | 4 ++++ 2 files changed, 21 insertions(+) create mode 100644 Docs/source/Python_Tutorial.rst diff --git a/Docs/source/Python_Tutorial.rst b/Docs/source/Python_Tutorial.rst new file mode 100644 index 00000000..37c6fc7d --- /dev/null +++ b/Docs/source/Python_Tutorial.rst @@ -0,0 +1,17 @@ +.. _tutorials_python: + +====== +Python +====== + +These examples show how to use AMReX from Python. +AMReX applications can also be interfaced to Python with the same logic. + +In order to build the Python tutorials, add ``-DTUTORIAL_PYTHON=ON`` to the CMake configuration options. +Then install with ``cmake --build build --target pyamrex_pip_install``. + +Examples: + +- ``GuidedTutorials/MultiFab/main.py`` + +Please see `pyAMReX `__ for more details. diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 555b56c3..d9995d95 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -50,6 +50,7 @@ sorted by the following categories: - :ref:`Linear Solvers` -- Examples of several linear solvers. - :ref:`MUI` -- Incorporates the MxUI/MUI (Multiscale Universal interface) frame into AMReX. - :ref:`Particles` -- Basic usage of AMReX's particle data structures. +- :ref:`Python` -- Using AMReX and interfacing with AMReX applications form Python - via `pyAMReX `__ - :ref:`SDC` -- Example usage of a "Multi-Implicit" Spectral Deferred Corrections (MISDC) integrator to solve a scalar advection-diffusion-reaction equation. - :ref:`SENSEI` -- In situ data analysis and visualization through a unified interface. @@ -71,6 +72,7 @@ sorted by the following categories: ML_Tutorial MUI_Tutorial Particles_Tutorial + Python_Tutorial SDC_Tutorial SENSEI_Tutorial SUNDIALS_Tutorial @@ -95,6 +97,8 @@ sorted by the following categories: .. _`Particles`: Particles_Tutorial.html +.. _`Python`: Python_Tutorial.html + .. _`SDC`: SDC_Tutorial.html .. _`SENSEI`: SENSEI_Tutorial.html From 121be95d789b8ec2c88dab05dfca85319c6bab5b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 11 Aug 2023 17:33:19 -0700 Subject: [PATCH 7/9] Python: Shared Libs --- ExampleCodes/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ExampleCodes/CMakeLists.txt b/ExampleCodes/CMakeLists.txt index 75e2688b..54e5e5de 100644 --- a/ExampleCodes/CMakeLists.txt +++ b/ExampleCodes/CMakeLists.txt @@ -25,6 +25,14 @@ set ( AMReX_GIT_REPO "https://github.com/AMReX-Codes/amrex.git/" set ( SUNDIALS_GIT_REPO "https://github.com/LLNL/sundials.git" CACHE STRING "The URL identifying the repo to fetch SUNDIALS from" ) +# +# For Python, we need AMReX as a a sahred library, otherwise we cannot share all +# its global variables between multiple Python modules +# +if(TUTORIAL_PYTHON) + set(AMReX_BUILD_SHARED_LIBS ON CACHE BOOL "Build AMReX shared library" FORCE) +endif() + if( NOT DEFINED AMReX_DIR ) # # Fetch amrex repo From 545561ef8ed06d07a706ed319dac5033c08e406b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 11 Aug 2023 15:14:41 -0700 Subject: [PATCH 8/9] CI: Python --- .../workflows/dependencies/dependencies.sh | 6 +- .../dependencies/dependencies_clang6.sh | 6 +- .../dependencies/dependencies_gcc10.sh | 6 +- .../dependencies/dependencies_nofortran.sh | 6 +- .github/workflows/linux.yml | 68 ++++++++++++------- ExampleCodes/CMakeLists.txt | 42 +++++++++++- 6 files changed, 103 insertions(+), 31 deletions(-) diff --git a/.github/workflows/dependencies/dependencies.sh b/.github/workflows/dependencies/dependencies.sh index d0e86e99..ecaa6eca 100755 --- a/.github/workflows/dependencies/dependencies.sh +++ b/.github/workflows/dependencies/dependencies.sh @@ -13,4 +13,8 @@ sudo apt-get install -y --no-install-recommends\ build-essential \ g++ gfortran \ libopenmpi-dev \ - openmpi-bin + openmpi-bin \ + python3 \ + python3-pip + +python3 -m pip install -U pip setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_clang6.sh b/.github/workflows/dependencies/dependencies_clang6.sh index 19b348b9..596dd8c3 100755 --- a/.github/workflows/dependencies/dependencies_clang6.sh +++ b/.github/workflows/dependencies/dependencies_clang6.sh @@ -11,4 +11,8 @@ sudo apt-get update sudo apt-get install -y \ build-essential \ - clang gfortran + clang gfortran \ + python3 \ + python3-pip + +python3 -m pip install -U pip setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_gcc10.sh b/.github/workflows/dependencies/dependencies_gcc10.sh index 583c5018..66573abc 100755 --- a/.github/workflows/dependencies/dependencies_gcc10.sh +++ b/.github/workflows/dependencies/dependencies_gcc10.sh @@ -14,4 +14,8 @@ sudo apt-get install -y --no-install-recommends \ build-essential \ g++-10 gfortran-10 \ libopenmpi-dev \ - openmpi-bin + openmpi-bin \ + python3 \ + python3-pip + +python3 -m pip install -U pip setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_nofortran.sh b/.github/workflows/dependencies/dependencies_nofortran.sh index 36d759f6..d00c1ff8 100755 --- a/.github/workflows/dependencies/dependencies_nofortran.sh +++ b/.github/workflows/dependencies/dependencies_nofortran.sh @@ -17,4 +17,8 @@ sudo apt-get install -y --no-install-recommends\ build-essential \ g++ \ libopenmpi-dev \ - openmpi-bin + openmpi-bin \ + python3 \ + python3-pip + +python3 -m pip install -U pip setuptools wheel diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 25f93f6e..428d21e8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -16,17 +16,21 @@ jobs: - name: Build & Install run: | cd ExampleCodes - mkdir build - cd build - cmake .. \ + cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_LINEAR_SOLVERS=ON \ -DAMReX_FORTRAN=ON \ -DAMReX_FORTRAN_INTERFACES=ON \ -DAMReX_EB=ON \ - -DAMReX_PARTICLES=ON - make -j 2 + -DAMReX_PARTICLES=ON \ + -DTUTORIAL_PYTHON=ON + cmake --build build -j 2 + cmake --build build -j 2 --target pyamrex_pip_install + - name: Run Python + run: | + cd GuidedTutorials/MultiFab/ + python main.py # Build all tutorials tutorials_cxx20: @@ -40,9 +44,7 @@ jobs: - name: Build & Install run: | cd ExampleCodes - mkdir build - cd build - cmake .. \ + cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_OMP=ON \ @@ -50,12 +52,18 @@ jobs: -DAMReX_LINEAR_SOLVERS=ON \ -DAMReX_FORTRAN=ON \ -DAMReX_FORTRAN_INTERFACES=ON \ - -DAMReX_EB=ON \ + -DAMReX_EB=ON \ + -DTUTORIAL_PYTHON=ON \ -DCMAKE_CXX_STANDARD=20 \ -DCMAKE_C_COMPILER=$(which gcc-10) \ -DCMAKE_CXX_COMPILER=$(which g++-10) \ -DCMAKE_Fortran_COMPILER=$(which mpif90) - make -j 2 + cmake --build build -j 2 + cmake --build build -j 2 --target pyamrex_pip_install + - name: Run Python + run: | + cd GuidedTutorials/MultiFab/ + python main.py tutorials_clang: name: Clang@6.0 C++14 SP Particles DP Mesh Debug [tutorials] @@ -68,9 +76,7 @@ jobs: - name: Build & Install run: | cd ExampleCodes - mkdir build - cd build - cmake .. \ + cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_MPI=OFF \ @@ -81,11 +87,17 @@ jobs: -DAMReX_EB=ON \ -DAMReX_PRECISION=DOUBLE \ -DAMReX_PARTICLES_PRECISION=SINGLE \ + -DTUTORIAL_PYTHON=ON \ -DCMAKE_CXX_STANDARD=14 \ -DCMAKE_C_COMPILER=$(which clang) \ -DCMAKE_CXX_COMPILER=$(which clang++) \ -DCMAKE_Fortran_COMPILER=$(which gfortran) - make -j 2 + cmake --build build -j 2 + cmake --build build -j 2 --target pyamrex_pip_install + - name: Run Python + run: | + cd GuidedTutorials/MultiFab/ + python main.py # Build all tutorials w/o MPI tutorials-nonmpi: @@ -99,9 +111,7 @@ jobs: - name: Build & Install run: | cd ExampleCodes - mkdir build - cd build - cmake .. \ + cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_MPI=OFF \ @@ -109,8 +119,14 @@ jobs: -DAMReX_FORTRAN=ON \ -DAMReX_FORTRAN_INTERFACES=ON \ -DAMReX_EB=ON \ - -DAMReX_PARTICLES=ON - make -j 2 + -DAMReX_PARTICLES=ON \ + -DTUTORIAL_PYTHON=ON + cmake --build build -j 2 + cmake --build build -j 2 --target pyamrex_pip_install + - name: Run Python + run: | + cd GuidedTutorials/MultiFab/ + python main.py # Build all tutorials tutorials-nofortran: @@ -124,16 +140,20 @@ jobs: - name: Build & Install run: | cd ExampleCodes - mkdir build - cd build - cmake .. \ + cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_LINEAR_SOLVERS=ON \ -DAMReX_EB=ON \ -DAMReX_PARTICLES=ON \ - -DAMReX_FORTRAN=OFF - make -j 2 + -DAMReX_FORTRAN=OFF \ + -DTUTORIAL_PYTHON=ON + cmake --build build -j 2 + cmake --build build -j 2 --target pyamrex_pip_install + - name: Run Python + run: | + cd GuidedTutorials/MultiFab/ + python main.py # Build all tutorials with CUDA 11.0.2 (recent supported) tutorials-cuda11: diff --git a/ExampleCodes/CMakeLists.txt b/ExampleCodes/CMakeLists.txt index 54e5e5de..be729404 100644 --- a/ExampleCodes/CMakeLists.txt +++ b/ExampleCodes/CMakeLists.txt @@ -14,13 +14,22 @@ set( CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake ) # If AMReX_DIR is passed then we link to the library found there. Otherwise # the sources are fetched and compiled from the repo pointed to by AMReX_GIT_REPO # -set ( AMReX_GIT_REPO "https://github.com/AMReX-Codes/amrex.git/" - CACHE STRING "The URL identifying the repo to fetchg AMReX from" ) +set ( AMReX_GIT_REPO "https://github.com/AMReX-Codes/amrex.git" + CACHE STRING "The URL identifying the repo to fetch AMReX from" ) # # Fetch and compile AMReX or link to an existing CMake build install of libamrex. # If AMReX_DIR is passed then we link to the library found there. Otherwise -# the sources are fetched and compiled from the repo pointed to by AMReX_GIT_REPO +# the sources are fetched and compiled from the repo pointed to by pyAMReX_GIT_REPO +# +set ( pyAMReX_GIT_REPO "https://github.com/AMReX-Codes/pyamrex.git" + CACHE STRING "The URL identifying the repo to fetch pyAMReX from" ) +option(TUTORIAL_PYTHON OFF) + +# +# Fetch and compile AMReX or link to an existing CMake build install of libamrex. +# If AMReX_DIR is passed then we link to the library found there. Otherwise +# the sources are fetched and compiled from the repo pointed to by SUNDIALS_GIT_REPO # set ( SUNDIALS_GIT_REPO "https://github.com/LLNL/sundials.git" CACHE STRING "The URL identifying the repo to fetch SUNDIALS from" ) @@ -182,6 +191,33 @@ else() endif() endif() +if(TUTORIAL_PYTHON) + if(DEFINED pyAMReX_DIR) + add_subdirectory(${pyAMReX_DIR}) + else() + # + # Fetch pyAMReX repo + # + message(STATUS "Fetching from ${pyAMReX_GIT_REPO} branch ${pyAMReX_GIT_BRANCH}" ) + + set(pyAMReX_GIT_BRANCH "development" CACHE STRING "The pyAMReX branch to checkout") + set(pyAMReX_INSTALL "NO" CACHE INTERNAL "Disable install target for pyAMReX") + + include(FetchContent) + #set(FETCHCONTENT_QUIET OFF) # Verbose ON + + FetchContent_Declare( fetchedpyamrex + GIT_REPOSITORY ${pyAMReX_GIT_REPO} + GIT_TAG ${pyAMReX_GIT_BRANCH} + ) + + if(NOT fetchedpyamrex_POPULATED) + FetchContent_Populate(fetchedpyamrex) + add_subdirectory(${fetchedpyamrex_SOURCE_DIR} ${fetchedpyamrex_BINARY_DIR}) + endif() + endif() +endif() + # # List of subdirectories to search for CMakeLists. # For now, we do not include MUI, SDC, SWFFT From bc2a906a72392f1dc1106a1a496a0d26f523269d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 13 Aug 2023 19:25:39 -0700 Subject: [PATCH 9/9] CUDA/HIP Support and Transpose Arrays --- GuidedTutorials/MultiFab/main.py | 57 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/GuidedTutorials/MultiFab/main.py b/GuidedTutorials/MultiFab/main.py index dd186069..e4418b98 100644 --- a/GuidedTutorials/MultiFab/main.py +++ b/GuidedTutorials/MultiFab/main.py @@ -1,6 +1,26 @@ -import numpy as np import amrex.space3d as amr +# CPU/GPU logic +if amr.Config.have_gpu: + try: + import cupy as cp + xp = cp + print("Note: found and will use cupy") + except ImportError: + print("Warning: GPU found but cupy not available! Trying managed memory in numpy...") + import numpy as np + xp = np + if amr.Config.gpu_backend == "SYCL": + print("Warning: SYCL GPU backend not yet implemented for Python") + import numpy as np + xp = np + +else: + import numpy as np + xp = np + print("Note: found and will use numpy") + + # Initialize AMReX amr.initialize([]) @@ -57,31 +77,18 @@ mf_val = 0. for mfi in mf: bx = mfi.validbox() - # Preferred way to fill array using NumPy operations: - # - mf_array is indexed in reversed order (n,z,y,x) + # Preferred way to fill array using fast ranged operations: + # - xp.array is indexed in reversed order (n,z,y,x), + # .T creates a view into the AMReX (x,y,z,n) order # - indices are local (range from 0 to box size) - mf_array = np.array(mf.array(mfi), copy=False) - x = (np.arange(bx.small_end[0], bx.big_end[0]+1)+0.5)*dx[0] - y = (np.arange(bx.small_end[1], bx.big_end[1]+1)+0.5)*dx[1] - z = (np.arange(bx.small_end[2], bx.big_end[2]+1)+0.5)*dx[2] - v = (x[np.newaxis,np.newaxis,:] - + y[np.newaxis,:,np.newaxis]*0.1 - + z[:,np.newaxis,np.newaxis]*0.01) - mf_array = 1. + np.exp(-v) - # Alternative way to fill array using standard for loop over box indices: - # - mf_array_ref is indexed in standard order (x,y,z,n) - # - indices are global - mf_array_ref = mf.array(mfi) - for i, j, k in bx: - x = (i+0.5)*dx[0] - y = (j+0.5)*dx[1] - z = (k+0.5)*dx[2] - v = x + y*0.1 + z*0.01 - mf_array_ref[i,j,k,0] = 1. + np.exp(-v) - # Check that mf_array has been filled correctly, compare it with - # mf_array_ref (filled as Array4 and transformed to NumPy array) - mf_array_ref_np = np.array(mf_array_ref, copy=False) - assert np.allclose(mf_array, mf_array_ref_np, rtol=1e-16, atol=1e-16) + mf_array = xp.array(mf.array(mfi), copy=False).T + x = (xp.arange(bx.small_end[0], bx.big_end[0]+1)+0.5)*dx[0] + y = (xp.arange(bx.small_end[1], bx.big_end[1]+1)+0.5)*dx[1] + z = (xp.arange(bx.small_end[2], bx.big_end[2]+1)+0.5)*dx[2] + v = (x[xp.newaxis,xp.newaxis,:] + + y[xp.newaxis,:,xp.newaxis]*0.1 + + z[:,xp.newaxis,xp.newaxis]*0.01) + mf_array = 1. + xp.exp(-v) # Plot MultiFab data plotfile = amr.concatenate(root="plt", num=1, mindigits=3)