diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0dab1ed1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +build/**/* +build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69f94da7..1f545c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,10 @@ jobs: fail-fast: false matrix: include: - - name: ubu22-gcc12-clang15 + - name: ubu22-gcc12-clang16 os: ubuntu-22.04 compiler: gcc-12 - clang-runtime: '15' + clang-runtime: '16' steps: - uses: actions/checkout@v3 with: @@ -79,8 +79,8 @@ jobs: git clone --depth=1 --branch "release/${{ matrix.clang-runtime }}.x" --single-branch https://github.com/llvm/llvm-project.git clang-dev cd clang-dev # Apply patches - echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches" - [ -f ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch ] && git apply ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch + echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" + compgen -G "../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch" > /dev/null && find ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch -printf "%f\n" && git apply ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch # Build clang-dev mkdir build mkdir inst @@ -128,6 +128,9 @@ jobs: if: runner.os == 'Linux' #TODO: exclude on release run: | # Install xeus-clang-repl deps + sudo apt-get update --yes + sudo apt-get install --yes --no-install-recommends ncurses-dev libtinfo-dev libtinfo5 + # conda update --all conda install -y -q -c conda-forge \ 'xeus>=2.0,<3.0' \ @@ -139,17 +142,50 @@ jobs: libuuid \ pytest \ jupyter_kernel_test + - name: Build and Install CppInterOp on Linux + if: runner.os == 'Linux' + run: | + # Build CppInterOp next to cling and llvm-project. + LLVM_DIR="$(realpath clang-dev)" + LLVM_BUILD_DIR="$(realpath clang-dev/build)" + CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include" + git clone https://github.com/compiler-research/CppInterOp.git + export CPPINTEROP_DIR=$PWD/cppyy-backend/python/cppyy_backend/ + cd CppInterOp + mkdir build + cd build + export CPPINTEROP_BUILD_DIR=$PWD + cmake -DCMAKE_BUILD_TYPE=Release \ + -DUSE_CLING=OFF \ + -DUSE_REPL=ON \ + -DLLVM_DIR=$LLVM_BUILD_DIR \ + -DLLVM_USE_LINKER=gold \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ + ../ + cmake --build . --parallel $(nproc --all) + cd ../.. + # We need CPPINTEROP_DIR, LLVM_BUILD_DIR and CPLUS_INCLUDE_PATH later + echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV + echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV + echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV + echo "CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV - name: Build xeus-clang-repl on Linux if: runner.os == 'Linux' #TODO: exclude on release run: | - # Build and Install xeus-clang-repl + ## Build and Install xeus-clang-repl mkdir build cd build PP=$KERNEL_PYTHON_PREFIX if [[ -z "$PP" ]]; then PP=$(conda info --base) fi - cmake -DCMAKE_PREFIX_PATH=$PP -DCMAKE_INSTALL_PREFIX=$PP -DCMAKE_INSTALL_LIBDIR=lib -DLLVM_CONFIG_EXTRA_PATH_HINTS=$PATH_TO_LLVM_BUILD -DLLVM_USE_LINKER=gold -DLLVM_REQUIRED_VERSION=${{ matrix.clang-runtime }} .. + cmake -DCMAKE_PREFIX_PATH=$PP -DCMAKE_INSTALL_PREFIX=$PP \ + -DCMAKE_INSTALL_LIBDIR=lib -DLLVM_DIR=$PATH_TO_LLVM_BUILD \ + -DLLVM_CONFIG_EXTRA_PATH_HINTS=$PATH_TO_LLVM_BUILD \ + -DCPPINTEROP_DIR=$CPPINTEROP_BUILD_DIR \ + -DLLVM_USE_LINKER=gold \ + -DLLVM_REQUIRED_VERSION=${{ matrix.clang-runtime }} .. make install - name: Setup tmate session if: ${{ failure() }} diff --git a/CMakeLists.txt b/CMakeLists.txt index a4a492bc..e6bcf1c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,14 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}") include(GNUInstallDirs) +message(STATUS "Env CPLUS_INCLUDE_PATH=$ENV{CPLUS_INCLUDE_PATH}") +message(STATUS "Env LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}") +message(STATUS "Env PYTHONPATH=$ENV{PYTHONPATH}") + +set(CMAKE_CPLUS_INCLUDE_PATH "$ENV{CPLUS_INCLUDE_PATH}") +set(CMAKE_LD_LIBRARY_PATH "$ENV{LD_LIBRARY_PATH}") +set(CMAKE_PYTHONPATH "$ENV{PYTHONPATH}") + configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp11/kernel.json.in" "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp11/kernel.json" @@ -44,33 +52,29 @@ configure_file ( set(xeus_REQUIRED_VERSION 1.0.0) set(cppzmq_REQUIRED_VERSION 4.3.0) -set(llvm_REQUIRED_VERSION 14) -if (LLVM_REQUIRED_VERSION) - set(llvm_REQUIRED_VERSION ${LLVM_REQUIRED_VERSION}) -endif() find_package(xeus ${xeus_REQUIRED_VERSION} REQUIRED) find_package(cppzmq REQUIRED CONFIG) find_package(pugixml REQUIRED) +#CppInterOp +#TODO: Make CppInterOp cmake package and use find_package +#find_package(CppInterOp REQUIRED CONFIG PATHS "${CPPINTEROP_DIR}") +add_library(clangCppInterOp SHARED IMPORTED) +set_property(TARGET clangCppInterOp PROPERTY IMPORTED_LOCATION "${CPPINTEROP_DIR}/lib/libclangCppInterOp.so") +set_property(TARGET clangCppInterOp PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${CPPINTEROP_DIR}/../include") +# +message(STATUS "CppInterOp Imported location: ${CPPINTEROP_DIR}/lib/libclangCppInterOp.so") +message(STATUS "CppInterOp include location: ${CPPINTEROP_DIR}/../include") -#find_package(LLVM REQUIRED CONFIG PATHS "${LLVM_CONFIG_EXTRA_PATH_HINTS}" NO_DEFAULT_PATH) -#find_package(Clang REQUIRED CONFIG PATHS "${LLVM_CONFIG_EXTRA_PATH_HINTS}/lib/cmake/clang/" NO_DEFAULT_PATH) -find_package(LLVM ${llvm_REQUIRED_VERSION} REQUIRED CONFIG PATHS "${LLVM_CONFIG_EXTRA_PATH_HINTS}") -find_package(Clang REQUIRED CONFIG PATHS "${LLVM_BINARY_DIR}/lib/cmake/clang/" "${LLVM_CONFIG_EXTRA_PATH_HINTS}/lib/cmake/clang/" NO_DEFAULT_PATH) -include_directories(SYSTEM ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) -link_directories(${LLVM_LIBRARY_DIR}) + +### # Before including llvm suppress the option() honors normal variables warning. if(POLICY CMP0077) cmake_policy(SET CMP0077 NEW) endif() -include(AddLLVM) -include(HandleLLVMOptions) - -add_definitions(-DCLANG_RESOURCE_DIR="${LLVM_BINARY_DIR}/lib/clang/${LLVM_PACKAGE_VERSION}") - find_package(cxxopts REQUIRED) find_package(PythonLibs REQUIRED) @@ -100,13 +104,23 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() - CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG) - - if (HAS_CPP14_FLAG) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") - else() - message(FATAL_ERROR "Unsupported compiler -- xeus requires C++14 support!") + ## Clang 13 require c++14 or later, Clang 16 require c++17 or later. + if (LLVM_VERSION_MAJOR GREATER_EQUAL 16) + if (NOT CMAKE_CXX_STANDARD) + set (CMAKE_CXX_STANDARD 17) + endif() + if (CMAKE_CXX_STANDARD LESS 17) + message(fatal "LLVM/CppInterOp requires c++17 or later") + endif() + elseif (LLVM_VERSION_MAJOR GREATER_EQUAL 13) + if (NOT CMAKE_CXX_STANDARD) + set (CMAKE_CXX_STANDARD 14) + endif() + if (CMAKE_CXX_STANDARD LESS 14) + message(fatal "LLVM/CppInterOp requires c++14 or later") + endif() endif() + endif() #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions") @@ -158,13 +172,27 @@ set(XCPP_HEADERS # xeus-clang-repl is the target for the library set(LLVM_REQUIRES_EH ON) set(LLVM_REQUIRES_RTTI ON) -llvm_add_library(xeus-clang-repl SHARED ${XEUS_CLANG_REPL_SRC} ${XEUS_CLANG_REPL_HEADERS}) +add_library(xeus-clang-repl SHARED ${XEUS_CLANG_REPL_SRC} ${XEUS_CLANG_REPL_HEADERS}) # xcpp is the target for the kernel executable +set(CMAKE_CXX_STANDARD 14) # FIXME: Be smarter and check if the standard was defined outside. add_executable(xcpp ${XCPP_SRC}) -set_target_properties(xcpp PROPERTIES ENABLE_EXPORTS 1) +set_target_properties(xcpp PROPERTIES + ENABLE_EXPORTS 1 + CXX_STANDARD ${CMAKE_CXX_STANDARD} +) target_link_libraries(xcpp PUBLIC xeus-clang-repl pthread) +#TODO: We may be need sse RPATH +#set_target_properties(xcpp clangCppInterOp PROPERTIES +# INSTALL_RPATH_USE_LINK_PATH TRUE +#) +if(APPLE) + target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup) +elseif(NOT MSVC) + target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files) +endif() + set(LLVM_NO_DEAD_STRIP 1) if (APPLE) @@ -195,20 +223,25 @@ target_include_directories(xeus-clang-repl SYSTEM PUBLIC $ $) -target_link_libraries(xeus-clang-repl PUBLIC clangInterpreter xeus pugixml cxxopts::cxxopts) +target_link_libraries(xeus-clang-repl PUBLIC clangCppInterOp xeus pugixml cxxopts::cxxopts) + +if(APPLE) + target_link_libraries(xeus-clang-repl PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup) +elseif(NOT MSVC) + target_link_libraries(xeus-clang-repl PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files) +endif() target_include_directories(xeus-clang-repl PUBLIC ${PYTHON_INCLUDE_DIRS}) target_link_libraries(xeus-clang-repl PUBLIC ${PYTHON_LIBRARIES}) target_link_libraries(xeus-clang-repl ${PYTHON_LIBRARIES_Development_Main}) - set_target_properties(xeus-clang-repl PROPERTIES PUBLIC_HEADER "${XEUS_CLANG_REPL_HEADERS}" COMPILE_DEFINITIONS "XEUS_CLANG_REPL_EXPORTS" PREFIX "" VERSION ${${PROJECT_NAME}_VERSION} SOVERSION ${XEUS_CLANG_REPL_VERSION_MAJOR} - OUTPUT_NAME "libxeus-clang-repl") - + OUTPUT_NAME "libxeus-clang-repl" + CXX_STANDARD ${CMAKE_CXX_STANDARD}) ################ # Installation # @@ -279,4 +312,4 @@ install(EXPORT ${PROJECT_NAME}-targets # Documentation if(XEUS_CLANG_REPL_INCLUDE_DOCS) add_subdirectory(docs) -endif() \ No newline at end of file +endif() diff --git a/Dockerfile b/Dockerfile index fcebe48f..335a1a00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV TAG="$BASE_TAG" ENV LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ LANGUAGE=en_US.UTF-8 - + # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) RUN apt-get update --yes && \ @@ -38,6 +38,8 @@ RUN apt-get update --yes && \ git \ nano-tiny \ less \ + gdb valgrind \ + emacs \ && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ @@ -52,6 +54,8 @@ ENV LC_ALL=en_US.UTF-8 \ USER ${NB_UID} +ENV LLVM_REQUIRED_VERSION=16 + # Copy git repository to home directory of container COPY --chown=${NB_UID}:${NB_GID} . "${HOME}"/ @@ -67,7 +71,7 @@ RUN mamba install --quiet --yes -c conda-forge \ # notebook,jpyterhub, jupyterlab are inherited from base-notebook container image # Other "our" conda installs cmake \ - #'clangdev=15' \ + #"clangdev=$LLVM_REQUIRED_VERSION" \ 'xeus>=2.0,<3.0' \ 'nlohmann_json>=3.9.1,<3.10' \ 'cppzmq>=4.6.0,<5' \ @@ -90,7 +94,7 @@ RUN mamba install --quiet --yes -c conda-forge \ EXPOSE 8888 # Configure container startup -CMD ["start-notebook.sh"] +CMD ["start-notebook.sh", "--debug", "&>/home/jovyan/log.txt"] USER root @@ -139,13 +143,15 @@ RUN \ echo "Debug: Repo id: $repository_id" && \ artifacts_info=$(curl -s -H "Accept: application/vnd.github+json" "https://api.github.com/repos/${gh_repo_owner}/${gh_repo_name}/actions/artifacts?per_page=100&name=${artifact_name}") && \ artifact_id=$(echo "$artifacts_info" | jq -r "[.artifacts[] | select(.expired == false and .workflow_run.repository_id == ${repository_id} and (\" \"+.workflow_run.head_branch+\" \" | test(\"${gh_repo_branch_regex}\")))] | sort_by(.updated_at)[-1].id") && \ - download_url="https://nightly.link/${gh_repo_owner}/${gh_repo_name}/actions/artifacts/${artifact_id}.zip" && \ + #download_url="https://nightly.link/${gh_repo_owner}/${gh_repo_name}/actions/artifacts/${artifact_id}.zip" && \ + download_url="https://link-to.alexander-penev.info/${gh_repo_owner}/${gh_repo_name}/actions/artifacts/${artifact_id}.zip" && \ # forked repo f_repository_id=$(curl -s -H "Accept: application/vnd.github+json" "https://api.github.com/repos/${gh_f_repo_owner}/${gh_f_repo_name}" | jq -r ".id") && \ echo "Debug: Forked Repo id: $f_repository_id" && \ f_artifacts_info=$(curl -s -H "Accept: application/vnd.github+json" "https://api.github.com/repos/${gh_f_repo_owner}/${gh_f_repo_name}/actions/artifacts?per_page=100&name=${artifact_name}") && \ f_artifact_id=$(echo "$f_artifacts_info" | jq -r "[.artifacts[] | select(.expired == false and .workflow_run.repository_id == ${f_repository_id} and (\" \"+.workflow_run.head_branch+\" \" | test(\"${gh_repo_branch_regex}\")))] | sort_by(.updated_at)[-1].id") && \ - f_download_url="https://nightly.link/${gh_f_repo_owner}/${gh_f_repo_name}/actions/artifacts/${f_artifact_id}.zip" && \ + #f_download_url="https://nightly.link/${gh_f_repo_owner}/${gh_f_repo_name}/actions/artifacts/${f_artifact_id}.zip" && \ + f_download_url="https://link-to.alexander-penev.info/${gh_f_repo_owner}/${gh_f_repo_name}/actions/artifacts/${f_artifact_id}.zip" && \ # tag for download_tag in $gh_repo_branch; do echo "Debug: try tag $download_tag:"; download_tag_url="https://github.com/${gh_repo_owner}/${gh_repo_name}/releases/download/${download_tag}/${artifact_name}.tar.bz2"; if curl --head --silent --fail -L $download_tag_url 1>/dev/null; then echo "found"; break; fi; done && \ # try to download artifact ot release tag asset @@ -162,27 +168,90 @@ RUN \ # echo "Debug clang path: $PATH_TO_CLANG_DEV" && \ export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/build && \ - export PATH=$PATH_TO_LLVM_BUILD/bin:$PATH && \ + export VENV=/home/jovyan/.venv && \ + echo "export VENV=$VENV" >> ~/.profile && \ + export PATH=$VENV/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \ export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH && \ # - # Build and Install xeus-clang-repl + # Build CppInterOp # + export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/../llvm/include:${PATH_TO_LLVM_BUILD}/../clang/include:${PATH_TO_LLVM_BUILD}/include:${PATH_TO_LLVM_BUILD}/tools/clang/include" && \ + git clone https://github.com/compiler-research/CppInterOp.git && \ + export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \ + export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \ + cd CppInterOp && \ mkdir build && \ cd build && \ - cmake -DLLVM_CMAKE_DIR=$PATH_TO_LLVM_BUILD -DCMAKE_PREFIX_PATH=$KERNEL_PYTHON_PREFIX -DCMAKE_INSTALL_PREFIX=$KERNEL_PYTHON_PREFIX -DCMAKE_INSTALL_LIBDIR=lib -DLLVM_CONFIG_EXTRA_PATH_HINTS=${PATH_TO_LLVM_BUILD}/lib -DLLVM_REQUIRED_VERSION=15 -DLLVM_USE_LINKER=gold .. && \ - make install -j$(nproc --all) && \ + export CPPINTEROP_BUILD_DIR=$PWD && \ + cmake -DCMAKE_BUILD_TYPE=Release -DUSE_CLING=OFF -DUSE_REPL=ON -DLLVM_DIR=$PATH_TO_LLVM_BUILD -DLLVM_USE_LINKER=gold -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. && \ + cmake --build . --parallel $(nproc --all) && \ + #make install -j$(nproc --all) && \ + export CPLUS_INCLUDE_PATH="$CPPINTEROP_DIR/include:$CPLUS_INCLUDE_PATH" && \ + export LD_LIBRARY_PATH="$CPPINTEROP_DIR/lib:$LD_LIBRARY_PATH" && \ + echo "export LD_LIBRARY_PATH=$CPPINTEROP_DIR/lib:$LD_LIBRARY_PATH" >> ~/.profile && \ + cd ../.. && \ + # + # Build and Install cppyy-backend + # + git clone https://github.com/compiler-research/cppyy-backend.git && \ + cd cppyy-backend && \ + mkdir -p $CPPINTEROP_DIR/lib build && cd build && \ + # Install CppInterOp + (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all)) && \ + # Build and Install cppyy-backend + cmake -DCppInterOp_DIR=$CPPINTEROP_DIR .. && \ + cmake --build . --parallel $(nproc --all) && \ + cp libcppyy-backend.so $CPPINTEROP_DIR/lib/ && \ + cd ../.. && \ + # + # Build and Install CPyCppyy + # + # setup virtual environment + python3 -m venv .venv && \ + source $VENV/bin/activate && \ + # Install CPyCppyy + git clone https://github.com/compiler-research/CPyCppyy.git && \ + cd CPyCppyy && \ + mkdir build && cd build && \ + cmake .. && \ + cmake --build . --parallel $(nproc --all) && \ + export CPYCPPYY_DIR=$PWD && \ + cd ../.. && \ + # + # Build and Install cppyy + # + # source virtual environment + source $VENV/bin/activate && \ + # Install cppyy + git clone https://github.com/compiler-research/cppyy.git && \ + cd cppyy && \ + python -m pip install --upgrade . --no-deps && \ cd .. && \ + # Run cppyy + source $VENV/bin/activate && \ + #TODO: Fix cppyy path (/home/jovyan) to path to installed module + export PYTHONPATH=$PYTHONPATH:$CPYCPPYY_DIR:$CB_PYTHON_DIR:/home/jovyan && \ + echo "export PYTHONPATH=$PYTHONPATH" >> ~/.profile && \ + echo "source $VENV/bin/activate" >> ~/.profile && \ + python -c "import cppyy" && \ # - # Build and Install Clad + # Build and Install xeus-clang-repl # - git clone --depth=1 https://github.com/vgvassilev/clad.git && \ - cd clad && \ mkdir build && \ cd build && \ - cmake .. -DClang_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/clang/ -DLLVM_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/llvm/ -DCMAKE_INSTALL_PREFIX=/opt/conda -DLLVM_EXTERNAL_LIT="$(which lit)" && \ - #make -j$(nproc --all) && \ - make && \ - make install && \ - # install clad in all exist kernels - jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' $KERNEL_PYTHON_PREFIX/share/jupyter/kernels/xcpp14/kernel.json > tmp.$$.json && mv tmp.$$.json $KERNEL_PYTHON_PREFIX/share/jupyter/kernels/xcpp14/kernel.json - + cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_CMAKE_DIR=$PATH_TO_LLVM_BUILD -DCMAKE_PREFIX_PATH=$KERNEL_PYTHON_PREFIX -DCMAKE_INSTALL_PREFIX=$KERNEL_PYTHON_PREFIX -DCMAKE_INSTALL_LIBDIR=lib -DLLVM_CONFIG_EXTRA_PATH_HINTS=${PATH_TO_LLVM_BUILD}/lib -DCPPINTEROP_DIR=$CPPINTEROP_BUILD_DIR -DLLVM_REQUIRED_VERSION=$LLVM_REQUIRED_VERSION -DLLVM_USE_LINKER=gold .. && \ + make install -j$(nproc --all) && \ + cd .. + # + # Build and Install Clad + # + #git clone --depth=1 https://github.com/vgvassilev/clad.git && \ + #cd clad && \ + #mkdir build && \ + #cd build && \ + #cmake .. -DClang_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/clang/ -DLLVM_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/llvm/ -DCMAKE_INSTALL_PREFIX=/opt/conda -DLLVM_EXTERNAL_LIT="$(which lit)" && \ + ##make -j$(nproc --all) && \ + #make && \ + #make install && \ + ## install clad in all exist kernels + #for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; done diff --git a/include/xeus-clang-repl/xinterpreter.hpp b/include/xeus-clang-repl/xinterpreter.hpp index 9cfd5c2d..6899dd3a 100644 --- a/include/xeus-clang-repl/xinterpreter.hpp +++ b/include/xeus-clang-repl/xinterpreter.hpp @@ -15,7 +15,7 @@ #ifndef XEUS_CLANG_REPL_INTERPRETER_HPP #define XEUS_CLANG_REPL_INTERPRETER_HPP -#include "clang/Interpreter/Interpreter.h" +#include "clang/Interpreter/CppInterOp.h" #include "nlohmann/json.hpp" @@ -72,7 +72,6 @@ class XEUS_CLANG_REPL_API interpreter : public xeus::xinterpreter { std::string get_stdopt(int argc, const char *const *argv); - std::unique_ptr m_interpreter; std::string m_version; xmagics_manager xmagics; diff --git a/patches/llvm/clang16-D141215-Value.patch b/patches/llvm/clang16-D141215-Value.patch new file mode 100644 index 00000000..76762368 --- /dev/null +++ b/patches/llvm/clang16-D141215-Value.patch @@ -0,0 +1,1838 @@ +diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h +index 863f6ac57f2a..2e343405cfbe 100644 +--- a/clang/include/clang/AST/Decl.h ++++ b/clang/include/clang/AST/Decl.h +@@ -4308,6 +4308,7 @@ class TopLevelStmtDecl : public Decl { + friend class ASTDeclWriter; + + Stmt *Statement = nullptr; ++ bool IsSemiMissing = false; + + TopLevelStmtDecl(DeclContext *DC, SourceLocation L, Stmt *S) + : Decl(TopLevelStmt, DC, L), Statement(S) {} +@@ -4321,6 +4322,12 @@ public: + SourceRange getSourceRange() const override LLVM_READONLY; + Stmt *getStmt() { return Statement; } + const Stmt *getStmt() const { return Statement; } ++ void setStmt(Stmt *S) { ++ assert(IsSemiMissing && "Operation supported for printing values only!"); ++ Statement = S; ++ } ++ bool isValuePrinting() const { return IsSemiMissing; } ++ void setValuePrinting(bool Missing = true) { IsSemiMissing = Missing; } + + static bool classof(const Decl *D) { return classofKind(D->getKind()); } + static bool classofKind(Kind K) { return K == TopLevelStmt; } +diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def +index 96feae991ccb..7526298557a8 100644 +--- a/clang/include/clang/Basic/TokenKinds.def ++++ b/clang/include/clang/Basic/TokenKinds.def +@@ -936,6 +936,9 @@ ANNOTATION(module_end) + // into the name of a header unit. + ANNOTATION(header_unit) + ++// Annotation for end of input in clang-repl. ++ANNOTATION(repl_input_end) ++ + #undef PRAGMA_ANNOTATION + #undef ANNOTATION + #undef TESTING_KEYWORD +diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h +index fd22af976613..020c2a88c185 100644 +--- a/clang/include/clang/Interpreter/Interpreter.h ++++ b/clang/include/clang/Interpreter/Interpreter.h +@@ -14,13 +14,15 @@ + #ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H + #define LLVM_CLANG_INTERPRETER_INTERPRETER_H + +-#include "clang/Interpreter/PartialTranslationUnit.h" +- ++#include "clang/AST/Decl.h" + #include "clang/AST/GlobalDecl.h" ++#include "clang/Interpreter/PartialTranslationUnit.h" ++#include "clang/Interpreter/Value.h" + ++#include "llvm/ADT/DenseMap.h" + #include "llvm/ExecutionEngine/JITSymbol.h" ++#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + #include "llvm/Support/Error.h" +- + #include + #include + +@@ -52,22 +54,24 @@ class Interpreter { + + Interpreter(std::unique_ptr CI, llvm::Error &Err); + ++ llvm::Error CreateExecutor(); ++ unsigned InitPTUSize = 0; ++ ++ Value LastValue; ++ + public: + ~Interpreter(); + static llvm::Expected> + create(std::unique_ptr CI); ++ const ASTContext &getASTContext() const; ++ ASTContext &getASTContext(); + const CompilerInstance *getCompilerInstance() const; +- const llvm::orc::LLJIT *getExecutionEngine() const; ++ llvm::Expected getExecutionEngine(); ++ + llvm::Expected Parse(llvm::StringRef Code); + llvm::Error Execute(PartialTranslationUnit &T); +- llvm::Error ParseAndExecute(llvm::StringRef Code) { +- auto PTU = Parse(Code); +- if (!PTU) +- return PTU.takeError(); +- if (PTU->TheModule) +- return Execute(*PTU); +- return llvm::Error::success(); +- } ++ llvm::Error ParseAndExecute(llvm::StringRef Code, Value *V = nullptr); ++ llvm::Expected CompileDtorCall(CXXRecordDecl *CXXRD); + + /// Undo N previous incremental inputs. + llvm::Error Undo(unsigned N = 1); +@@ -75,16 +79,33 @@ public: + /// \returns the \c JITTargetAddress of a \c GlobalDecl. This interface uses + /// the CodeGenModule's internal mangling cache to avoid recomputing the + /// mangled name. +- llvm::Expected getSymbolAddress(GlobalDecl GD) const; ++ llvm::Expected getSymbolAddress(GlobalDecl GD) const; + + /// \returns the \c JITTargetAddress of a given name as written in the IR. +- llvm::Expected ++ llvm::Expected + getSymbolAddress(llvm::StringRef IRName) const; + + /// \returns the \c JITTargetAddress of a given name as written in the object + /// file. +- llvm::Expected ++ llvm::Expected + getSymbolAddressFromLinkerName(llvm::StringRef LinkerName) const; ++ ++ size_t getEffectivePTUSize() const; ++ ++ enum InterfaceKind { NoAlloc, WithAlloc, CopyArray }; ++ ++ const llvm::SmallVectorImpl &getValuePrintingInfo() const { ++ return ValuePrintingInfo; ++ } ++ ++ Expr *SynthesizeExpr(Expr *E); ++ ++private: ++ bool FindRuntimeInterface(); ++ ++ llvm::DenseMap Dtors; ++ ++ llvm::SmallVector ValuePrintingInfo; + }; + } // namespace clang + +diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h +new file mode 100644 +index 000000000000..f70eb214b3d5 +--- /dev/null ++++ b/clang/include/clang/Interpreter/Value.h +@@ -0,0 +1,174 @@ ++//===--- Value.h - Incremental Compiation and Execution---*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file defines the class that used to represent a value in incremental ++// C++. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_CLANG_INTERPRETER_VALUE_H ++#define LLVM_CLANG_INTERPRETER_VALUE_H ++ ++#include ++ ++namespace llvm { ++class raw_ostream; ++ ++} // namespace llvm ++ ++namespace clang { ++ ++class ASTContext; ++class Interpreter; ++class QualType; ++ ++#if __has_attribute(visibility) && \ ++ (!(defined(_WIN32) || defined(__CYGWIN__)) || \ ++ (defined(__MINGW32__) && defined(__clang__))) ++#if defined(LLVM_BUILD_LLVM_DYLIB) || defined(LLVM_BUILD_SHARED_LIBS) ++#define REPL_EXTERNAL_VISIBILITY __attribute__((visibility("default"))) ++#else ++#define REPL_EXTERNAL_VISIBILITY ++#endif ++#else ++#if defined(_WIN32) ++#define REPL_EXTERNAL_VISIBILITY __declspec(dllexport) ++#endif ++#endif ++ ++#define REPL_BUILTIN_TYPES \ ++ X(bool, Bool) \ ++ X(char, Char_S) \ ++ X(signed char, SChar) \ ++ X(unsigned char, UChar) \ ++ X(short, Short) \ ++ X(unsigned short, UShort) \ ++ X(int, Int) \ ++ X(unsigned int, UInt) \ ++ X(long, Long) \ ++ X(unsigned long, ULong) \ ++ X(long long, LongLong) \ ++ X(unsigned long long, ULongLong) \ ++ X(float, Float) \ ++ X(double, Double) \ ++ X(long double, LongDouble) ++ ++class REPL_EXTERNAL_VISIBILITY Value { ++ union Storage { ++#define X(type, name) type m_##name; ++ REPL_BUILTIN_TYPES ++#undef X ++ void *m_Ptr; ++ }; ++ ++public: ++ enum Kind { ++#define X(type, name) K_##name, ++ REPL_BUILTIN_TYPES ++#undef X ++ ++ K_Void, ++ K_PtrOrObj, ++ K_Unspecified ++ }; ++ ++ Value() = default; ++ Value(Interpreter *In, void *Ty); ++ Value(const Value &RHS); ++ Value(Value &&RHS) noexcept; ++ Value &operator=(const Value &RHS); ++ Value &operator=(Value &&RHS) noexcept; ++ ~Value(); ++ ++ void printType(llvm::raw_ostream &Out) const; ++ void printData(llvm::raw_ostream &Out) const; ++ void print(llvm::raw_ostream &Out) const; ++ void dump() const; ++ void clear(); ++ ++ ASTContext &getASTContext(); ++ const ASTContext &getASTContext() const; ++ Interpreter &getInterpreter(); ++ const Interpreter &getInterpreter() const; ++ QualType getType() const; ++ ++ bool isValid() const { return ValueKind != K_Unspecified; } ++ bool isVoid() const { return ValueKind == K_Void; } ++ bool isManuallyAlloc() const { return IsManuallyAlloc; } ++ Kind getKind() const { return ValueKind; } ++ void setKind(Kind K) { ValueKind = K; } ++ void setOpaqueType(void *Ty) { OpaqueType = Ty; } ++ ++ void *getPtr() const; ++ void setPtr(void *Ptr) { Data.m_Ptr = Ptr; } ++ ++ bool isPointerOrObjectType() const { return ValueKind == K_PtrOrObj; } ++ ++#define X(type, name) \ ++ void set##name(type Val) { Data.m_##name = Val; } \ ++ type get##name() const { return Data.m_##name; } ++ REPL_BUILTIN_TYPES ++#undef X ++ ++ // Allow castAs to be partially specialized. ++ template struct CastFwd { ++ static T cast(const Value &V) { ++ if (V.isPointerOrObjectType()) ++ return (T)(uintptr_t)V.getAs(); ++ if (!V.isValid() || V.isVoid()) { ++ return T(); ++ } ++ return V.getAs(); ++ } ++ }; ++ ++ template struct CastFwd { ++ static T *cast(const Value &V) { ++ if (V.isPointerOrObjectType()) ++ return (T *)(uintptr_t)V.getAs(); ++ return nullptr; ++ } ++ }; ++ ++ /// \brief Get the value with cast. ++ // ++ /// Get the value cast to T. This is similar to reinterpret_cast(value), ++ /// casting the value of builtins (except void), enums and pointers. ++ /// Values referencing an object are treated as pointers to the object. ++ template T castAs() const { return CastFwd::cast(*this); } ++ ++ /// \brief Get to the value with type checking casting the underlying ++ /// stored value to T. ++ template T getAs() const { ++ switch (ValueKind) { ++ default: ++ return T(); ++#define X(type, name) \ ++ case Value::K_##name: \ ++ return (T)Data.m_##name; ++ REPL_BUILTIN_TYPES ++#undef X ++ } ++ } ++ ++private: ++ Interpreter *Interp = nullptr; ++ void *OpaqueType = nullptr; ++ Storage Data; ++ Kind ValueKind = K_Unspecified; ++ bool IsManuallyAlloc = false; ++}; ++ ++template <> inline void *Value::getAs() const { ++ if (isPointerOrObjectType()) ++ return Data.m_Ptr; ++ return (void *)getAs(); ++} ++ ++} // namespace clang ++#endif +diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h +index 6f9581b9ea1f..6b73f43a1fb7 100644 +--- a/clang/include/clang/Parse/Parser.h ++++ b/clang/include/clang/Parse/Parser.h +@@ -18,6 +18,7 @@ + #include "clang/Basic/OpenMPKinds.h" + #include "clang/Basic/OperatorPrecedence.h" + #include "clang/Basic/Specifiers.h" ++#include "clang/Basic/TokenKinds.h" + #include "clang/Lex/CodeCompletionHandler.h" + #include "clang/Lex/Preprocessor.h" + #include "clang/Sema/DeclSpec.h" +@@ -692,7 +693,8 @@ private: + bool isEofOrEom() { + tok::TokenKind Kind = Tok.getKind(); + return Kind == tok::eof || Kind == tok::annot_module_begin || +- Kind == tok::annot_module_end || Kind == tok::annot_module_include; ++ Kind == tok::annot_module_end || Kind == tok::annot_module_include || ++ Kind == tok::annot_repl_input_end; + } + + /// Checks if the \p Level is valid for use in a fold expression. +diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp +index 12d602fed693..8952920a09ad 100644 +--- a/clang/lib/CodeGen/CodeGenModule.cpp ++++ b/clang/lib/CodeGen/CodeGenModule.cpp +@@ -7186,8 +7186,14 @@ void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS, + } + + void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { +- assert(DeferredDeclsToEmit.empty() && +- "Should have emitted all decls deferred to emit."); ++ // FIXME: Re-enable the assertions once we fix regular codegen to not leave ++ // weak references behind. ++ // The code example also leaves entries in WeakRefReferences in regular clang. ++ // #include ++ // auto p = std::make_shared(42); ++ // ++ // assert(DeferredDeclsToEmit.empty() && ++ // "Should have emitted all decls deferred to emit."); + assert(NewBuilder->DeferredDecls.empty() && + "Newly created module should not have deferred decls"); + NewBuilder->DeferredDecls = std::move(DeferredDecls); +diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp +index ffa85e523c03..1b262d9e6f7c 100644 +--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp ++++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp +@@ -663,7 +663,8 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, + // them. + if (Tok.is(tok::eof) || + (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) && +- !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end))) ++ !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) && ++ !Tok.is(tok::annot_repl_input_end))) + return; + + // EmittedDirectiveOnThisLine takes priority over RequireSameLine. +@@ -819,6 +820,9 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, + // -traditional-cpp the lexer keeps /all/ whitespace, including comments. + PP.Lex(Tok); + continue; ++ } else if (Tok.is(tok::annot_repl_input_end)) { ++ PP.Lex(Tok); ++ continue; + } else if (Tok.is(tok::eod)) { + // Don't print end of directive tokens, since they are typically newlines + // that mess up our line tracking. These come from unknown pre-processor +diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt +index c49f22fddd8e..565e824bf0c9 100644 +--- a/clang/lib/Interpreter/CMakeLists.txt ++++ b/clang/lib/Interpreter/CMakeLists.txt +@@ -12,6 +12,8 @@ add_clang_library(clangInterpreter + IncrementalExecutor.cpp + IncrementalParser.cpp + Interpreter.cpp ++ InterpreterUtils.cpp ++ Value.cpp + + DEPENDS + intrinsics_gen +diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp +index 37d230b61f76..b4b6bc8b40fe 100644 +--- a/clang/lib/Interpreter/IncrementalExecutor.cpp ++++ b/clang/lib/Interpreter/IncrementalExecutor.cpp +@@ -86,15 +86,12 @@ llvm::Error IncrementalExecutor::runCtors() const { + return Jit->initialize(Jit->getMainJITDylib()); + } + +-llvm::Expected ++llvm::Expected + IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, + SymbolNameKind NameKind) const { + auto Sym = (NameKind == LinkerName) ? Jit->lookupLinkerMangled(Name) + : Jit->lookup(Name); +- +- if (!Sym) +- return Sym.takeError(); +- return Sym->getValue(); ++ return std::move(Sym); + } + + } // end namespace clang +diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h +index 54d37c76326b..8bead233d448 100644 +--- a/clang/lib/Interpreter/IncrementalExecutor.h ++++ b/clang/lib/Interpreter/IncrementalExecutor.h +@@ -51,9 +51,9 @@ public: + llvm::Error removeModule(PartialTranslationUnit &PTU); + llvm::Error runCtors() const; + llvm::Error cleanUp(); +- llvm::Expected ++ llvm::Expected + getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const; +- llvm::orc::LLJIT *getExecutionEngine() const { return Jit.get(); } ++ llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + }; + + } // end namespace clang +diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp +index 373e2844b4e4..259b9875e6be 100644 +--- a/clang/lib/Interpreter/IncrementalParser.cpp ++++ b/clang/lib/Interpreter/IncrementalParser.cpp +@@ -11,7 +11,6 @@ + //===----------------------------------------------------------------------===// + + #include "IncrementalParser.h" +- + #include "clang/AST/DeclContextInternals.h" + #include "clang/CodeGen/BackendUtil.h" + #include "clang/CodeGen/CodeGenAction.h" +@@ -19,9 +18,9 @@ + #include "clang/Frontend/CompilerInstance.h" + #include "clang/Frontend/FrontendAction.h" + #include "clang/FrontendTool/Utils.h" ++#include "clang/Interpreter/Interpreter.h" + #include "clang/Parse/Parser.h" + #include "clang/Sema/Sema.h" +- + #include "llvm/Option/ArgList.h" + #include "llvm/Support/CrashRecoveryContext.h" + #include "llvm/Support/Error.h" +@@ -31,6 +30,79 @@ + + namespace clang { + ++class IncrementalASTConsumer final : public ASTConsumer { ++ Interpreter &Interp; ++ std::unique_ptr Consumer; ++ ++public: ++ IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr C) ++ : Interp(InterpRef), Consumer(std::move(C)) {} ++ ++ bool HandleTopLevelDecl(DeclGroupRef DGR) override final { ++ if (DGR.isNull()) ++ return true; ++ if (!Consumer) ++ return true; ++ ++ for (Decl *D : DGR) ++ if (auto *TSD = llvm::dyn_cast(D); ++ TSD && TSD->isValuePrinting()) ++ TSD->setStmt(Interp.SynthesizeExpr(cast(TSD->getStmt()))); ++ ++ return Consumer->HandleTopLevelDecl(DGR); ++ } ++ void HandleTranslationUnit(ASTContext &Ctx) override final { ++ Consumer->HandleTranslationUnit(Ctx); ++ } ++ void HandleInlineFunctionDefinition(FunctionDecl *D) override final { ++ Consumer->HandleInlineFunctionDefinition(D); ++ } ++ void HandleInterestingDecl(DeclGroupRef D) override final { ++ Consumer->HandleInterestingDecl(D); ++ } ++ void HandleTagDeclDefinition(TagDecl *D) override final { ++ Consumer->HandleTagDeclDefinition(D); ++ } ++ void HandleTagDeclRequiredDefinition(const TagDecl *D) override final { ++ Consumer->HandleTagDeclRequiredDefinition(D); ++ } ++ void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final { ++ Consumer->HandleCXXImplicitFunctionInstantiation(D); ++ } ++ void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final { ++ Consumer->HandleTopLevelDeclInObjCContainer(D); ++ } ++ void HandleImplicitImportDecl(ImportDecl *D) override final { ++ Consumer->HandleImplicitImportDecl(D); ++ } ++ void CompleteTentativeDefinition(VarDecl *D) override final { ++ Consumer->CompleteTentativeDefinition(D); ++ } ++ void CompleteExternalDeclaration(VarDecl *D) override final { ++ Consumer->CompleteExternalDeclaration(D); ++ } ++ void AssignInheritanceModel(CXXRecordDecl *RD) override final { ++ Consumer->AssignInheritanceModel(RD); ++ } ++ void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final { ++ Consumer->HandleCXXStaticMemberVarInstantiation(D); ++ } ++ void HandleVTable(CXXRecordDecl *RD) override final { ++ Consumer->HandleVTable(RD); ++ } ++ ASTMutationListener *GetASTMutationListener() override final { ++ return Consumer->GetASTMutationListener(); ++ } ++ ASTDeserializationListener *GetASTDeserializationListener() override final { ++ return Consumer->GetASTDeserializationListener(); ++ } ++ void PrintStats() override final { Consumer->PrintStats(); } ++ bool shouldSkipFunctionBody(Decl *D) override final { ++ return Consumer->shouldSkipFunctionBody(D); ++ } ++ static bool classof(const clang::ASTConsumer *) { return true; } ++}; ++ + /// A custom action enabling the incremental processing functionality. + /// + /// The usual \p FrontendAction expects one call to ExecuteAction and once it +@@ -122,7 +194,8 @@ public: + } + }; + +-IncrementalParser::IncrementalParser(std::unique_ptr Instance, ++IncrementalParser::IncrementalParser(Interpreter &Interp, ++ std::unique_ptr Instance, + llvm::LLVMContext &LLVMCtx, + llvm::Error &Err) + : CI(std::move(Instance)) { +@@ -131,6 +204,9 @@ IncrementalParser::IncrementalParser(std::unique_ptr Instance, + if (Err) + return; + CI->ExecuteAction(*Act); ++ std::unique_ptr IncrConsumer = ++ std::make_unique(Interp, CI->takeASTConsumer()); ++ CI->setASTConsumer(std::move(IncrConsumer)); + Consumer = &CI->getASTConsumer(); + P.reset( + new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false)); +@@ -158,8 +234,8 @@ IncrementalParser::ParseOrWrapTopLevelDecl() { + LastPTU.TUPart = C.getTranslationUnitDecl(); + + // Skip previous eof due to last incremental input. +- if (P->getCurToken().is(tok::eof)) { +- P->ConsumeToken(); ++ if (P->getCurToken().is(tok::annot_repl_input_end)) { ++ P->ConsumeAnyToken(); + // FIXME: Clang does not call ExitScope on finalizing the regular TU, we + // might want to do that around HandleEndOfTranslationUnit. + P->ExitScope(); +@@ -259,25 +335,30 @@ IncrementalParser::Parse(llvm::StringRef input) { + Token Tok; + do { + PP.Lex(Tok); +- } while (Tok.isNot(tok::eof)); ++ } while (Tok.isNot(tok::annot_repl_input_end)); + } + + Token AssertTok; + PP.Lex(AssertTok); +- assert(AssertTok.is(tok::eof) && ++ assert(AssertTok.is(tok::annot_repl_input_end) && + "Lexer must be EOF when starting incremental parse!"); + +- if (CodeGenerator *CG = getCodeGen(Act.get())) { +- std::unique_ptr M(CG->ReleaseModule()); +- CG->StartModule("incr_module_" + std::to_string(PTUs.size()), +- M->getContext()); +- ++ if (std::unique_ptr M = GenModule()) + PTU->TheModule = std::move(M); +- } + + return PTU; + } + ++std::unique_ptr IncrementalParser::GenModule() { ++ static unsigned ID = 0; ++ if (CodeGenerator *CG = getCodeGen(Act.get())) { ++ std::unique_ptr M(CG->ReleaseModule()); ++ CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext()); ++ return M; ++ } ++ return nullptr; ++} ++ + void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) { + TranslationUnitDecl *MostRecentTU = PTU.TUPart; + TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl(); +diff --git a/clang/lib/Interpreter/IncrementalParser.h b/clang/lib/Interpreter/IncrementalParser.h +index 8e45d6b5931b..99e37588df9d 100644 +--- a/clang/lib/Interpreter/IncrementalParser.h ++++ b/clang/lib/Interpreter/IncrementalParser.h +@@ -16,7 +16,6 @@ + #include "clang/Interpreter/PartialTranslationUnit.h" + + #include "clang/AST/GlobalDecl.h" +- + #include "llvm/ADT/ArrayRef.h" + #include "llvm/ADT/StringRef.h" + #include "llvm/Support/Error.h" +@@ -31,8 +30,8 @@ namespace clang { + class ASTConsumer; + class CompilerInstance; + class IncrementalAction; ++class Interpreter; + class Parser; +- + /// Provides support for incremental compilation. Keeps track of the state + /// changes between the subsequent incremental input. + /// +@@ -57,7 +56,8 @@ class IncrementalParser { + std::list PTUs; + + public: +- IncrementalParser(std::unique_ptr Instance, ++ IncrementalParser(Interpreter &Interp, ++ std::unique_ptr Instance, + llvm::LLVMContext &LLVMCtx, llvm::Error &Err); + ~IncrementalParser(); + +@@ -76,6 +76,8 @@ public: + + std::list &getPTUs() { return PTUs; } + ++ std::unique_ptr GenModule(); ++ + private: + llvm::Expected ParseOrWrapTopLevelDecl(); + }; +diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp +index a6f5fdc6eefc..e8a07eceb678 100644 +--- a/clang/lib/Interpreter/Interpreter.cpp ++++ b/clang/lib/Interpreter/Interpreter.cpp +@@ -16,7 +16,11 @@ + #include "IncrementalExecutor.h" + #include "IncrementalParser.h" + ++#include "InterpreterUtils.h" + #include "clang/AST/ASTContext.h" ++#include "clang/AST/Mangle.h" ++#include "clang/AST/TypeVisitor.h" ++#include "clang/Basic/DiagnosticSema.h" + #include "clang/Basic/TargetInfo.h" + #include "clang/CodeGen/ModuleBuilder.h" + #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +@@ -27,12 +31,15 @@ + #include "clang/Driver/Tool.h" + #include "clang/Frontend/CompilerInstance.h" + #include "clang/Frontend/TextDiagnosticBuffer.h" ++#include "clang/Interpreter/Value.h" + #include "clang/Lex/PreprocessorOptions.h" +- ++#include "clang/Sema/Lookup.h" ++#include "llvm/ExecutionEngine/JITSymbol.h" ++#include "llvm/ExecutionEngine/Orc/LLJIT.h" + #include "llvm/IR/Module.h" + #include "llvm/Support/Errc.h" +-#include "llvm/Support/Host.h" +- ++#include "llvm/Support/raw_ostream.h" ++#include "llvm/TargetParser/Host.h" + using namespace clang; + + // FIXME: Figure out how to unify with namespace init_convenience from +@@ -176,7 +183,7 @@ Interpreter::Interpreter(std::unique_ptr CI, + llvm::ErrorAsOutParameter EAO(&Err); + auto LLVMCtx = std::make_unique(); + TSCtx = std::make_unique(std::move(LLVMCtx)); +- IncrParser = std::make_unique(std::move(CI), ++ IncrParser = std::make_unique(*this, std::move(CI), + *TSCtx->getContext(), Err); + } + +@@ -189,6 +196,28 @@ Interpreter::~Interpreter() { + } + } + ++// These better to put in a runtime header but we can't. This is because we ++// can't find the precise resource directory in unittests so we have to hard ++// code them. ++const char *const Runtimes = R"( ++ void* operator new(__SIZE_TYPE__, void* __p) noexcept; ++ void *__clang_Interpreter_SetValueWithAlloc(void*, void*, void*); ++ void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, void*); ++ void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, float); ++ void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, double); ++ void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, long double); ++ void __clang_Interpreter_SetValueNoAlloc(void*,void*,void*,unsigned long long); ++ template ++ void __clang_Interpreter_SetValueCopyArr(T* Src, void* Placement, unsigned long Size) { ++ for (auto Idx = 0; Idx < Size; ++Idx) ++ new ((void*)(((T*)Placement) + Idx)) T(Src[Idx]); ++ } ++ template ++ void __clang_Interpreter_SetValueCopyArr(const T (*Src)[N], void* Placement, unsigned long Size) { ++ __clang_Interpreter_SetValueCopyArr(Src[0], Placement, Size); ++ } ++)"; ++ + llvm::Expected> + Interpreter::create(std::unique_ptr CI) { + llvm::Error Err = llvm::Error::success(); +@@ -196,6 +225,14 @@ Interpreter::create(std::unique_ptr CI) { + std::unique_ptr(new Interpreter(std::move(CI), Err)); + if (Err) + return std::move(Err); ++ if (llvm::Error Err = Interp->ParseAndExecute(Runtimes)) ++ return std::move(Err); ++ ++ Interp->ValuePrintingInfo.resize(3); ++ // FIXME: This is a ugly hack. Undo command checks its availability by looking ++ // at the size of the PTU list. However we have parsed something in the ++ // beginning of the REPL so we have to mark them as 'Irrevocable'. ++ Interp->InitPTUSize = Interp->IncrParser->getPTUs().size(); + return std::move(Interp); + } + +@@ -203,19 +240,50 @@ const CompilerInstance *Interpreter::getCompilerInstance() const { + return IncrParser->getCI(); + } + +-const llvm::orc::LLJIT *Interpreter::getExecutionEngine() const { +- if (IncrExecutor) +- return IncrExecutor->getExecutionEngine(); +- return nullptr; ++llvm::Expected Interpreter::getExecutionEngine() { ++ if (!IncrExecutor) { ++ if (auto Err = CreateExecutor()) ++ return std::move(Err); ++ } ++ ++ return IncrExecutor->GetExecutionEngine(); ++} ++ ++ASTContext &Interpreter::getASTContext() { ++ return getCompilerInstance()->getASTContext(); ++} ++ ++const ASTContext &Interpreter::getASTContext() const { ++ return getCompilerInstance()->getASTContext(); ++} ++ ++size_t Interpreter::getEffectivePTUSize() const { ++ std::list &PTUs = IncrParser->getPTUs(); ++ assert(PTUs.size() >= InitPTUSize && "empty PTU list?"); ++ return PTUs.size() - InitPTUSize; + } + + llvm::Expected + Interpreter::Parse(llvm::StringRef Code) { ++ // Tell the interpreter sliently ignore unused expressions since value ++ // printing could cause it. ++ getCompilerInstance()->getDiagnostics().setSeverity( ++ clang::diag::warn_unused_expr, diag::Severity::Ignored, SourceLocation()); + return IncrParser->Parse(Code); + } + ++llvm::Error Interpreter::CreateExecutor() { ++ const clang::TargetInfo &TI = ++ getCompilerInstance()->getASTContext().getTargetInfo(); ++ llvm::Error Err = llvm::Error::success(); ++ auto Executor = std::make_unique(*TSCtx, Err, TI); ++ if (!Err) ++ IncrExecutor = std::move(Executor); ++ ++ return Err; ++} ++ + llvm::Error Interpreter::Execute(PartialTranslationUnit &T) { +- assert(T.TheModule); + if (!IncrExecutor) { + const clang::TargetInfo &TI = + getCompilerInstance()->getASTContext().getTargetInfo(); +@@ -235,7 +303,26 @@ llvm::Error Interpreter::Execute(PartialTranslationUnit &T) { + return llvm::Error::success(); + } + +-llvm::Expected ++llvm::Error Interpreter::ParseAndExecute(llvm::StringRef Code, Value *V) { ++ ++ auto PTU = Parse(Code); ++ if (!PTU) ++ return PTU.takeError(); ++ if (PTU->TheModule) ++ if (llvm::Error Err = Execute(*PTU)) ++ return Err; ++ ++ if (LastValue.isValid()) { ++ if (!V) { ++ LastValue.dump(); ++ LastValue.clear(); ++ } else ++ *V = std::move(LastValue); ++ } ++ return llvm::Error::success(); ++} ++ ++llvm::Expected + Interpreter::getSymbolAddress(GlobalDecl GD) const { + if (!IncrExecutor) + return llvm::make_error("Operation failed. " +@@ -245,7 +332,7 @@ Interpreter::getSymbolAddress(GlobalDecl GD) const { + return getSymbolAddress(MangledName); + } + +-llvm::Expected ++llvm::Expected + Interpreter::getSymbolAddress(llvm::StringRef IRName) const { + if (!IncrExecutor) + return llvm::make_error("Operation failed. " +@@ -255,7 +342,7 @@ Interpreter::getSymbolAddress(llvm::StringRef IRName) const { + return IncrExecutor->getSymbolAddress(IRName, IncrementalExecutor::IRName); + } + +-llvm::Expected ++llvm::Expected + Interpreter::getSymbolAddressFromLinkerName(llvm::StringRef Name) const { + if (!IncrExecutor) + return llvm::make_error("Operation failed. " +@@ -268,7 +355,7 @@ Interpreter::getSymbolAddressFromLinkerName(llvm::StringRef Name) const { + llvm::Error Interpreter::Undo(unsigned N) { + + std::list &PTUs = IncrParser->getPTUs(); +- if (N > PTUs.size()) ++ if (N > getEffectivePTUSize()) + return llvm::make_error("Operation failed. " + "Too many undos", + std::error_code()); +@@ -283,3 +370,305 @@ llvm::Error Interpreter::Undo(unsigned N) { + } + return llvm::Error::success(); + } ++ ++llvm::Expected ++Interpreter::CompileDtorCall(CXXRecordDecl *CXXRD) { ++ assert(CXXRD && "Cannot compile a destructor for a nullptr"); ++ if (auto Dtor = Dtors.find(CXXRD); Dtor != Dtors.end()) ++ return Dtor->getSecond(); ++ ++ if (CXXRD->hasIrrelevantDestructor()) ++ return llvm::orc::ExecutorAddr{}; ++ ++ CXXDestructorDecl *DtorRD = ++ getCompilerInstance()->getSema().LookupDestructor(CXXRD); ++ ++ llvm::StringRef Name = ++ IncrParser->GetMangledName(GlobalDecl(DtorRD, Dtor_Base)); ++ auto AddrOrErr = getSymbolAddress(Name); ++ if (!AddrOrErr) ++ return AddrOrErr.takeError(); ++ ++ Dtors[CXXRD] = *AddrOrErr; ++ return AddrOrErr; ++} ++ ++static constexpr llvm::StringRef MagicRuntimeInterface[] = { ++ "__clang_Interpreter_SetValueNoAlloc", ++ "__clang_Interpreter_SetValueWithAlloc", ++ "__clang_Interpreter_SetValueCopyArr"}; ++ ++bool Interpreter::FindRuntimeInterface() { ++ if (llvm::all_of(ValuePrintingInfo, [](Expr *E) { return E != nullptr; })) ++ return true; ++ ++ Sema &S = getCompilerInstance()->getSema(); ++ ASTContext &Ctx = S.getASTContext(); ++ ++ auto LookupInterface = [&](Expr *&Interface, llvm::StringRef Name) { ++ LookupResult R(S, &Ctx.Idents.get(Name), SourceLocation(), ++ Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); ++ S.LookupQualifiedName(R, Ctx.getTranslationUnitDecl()); ++ if (R.empty()) ++ return false; ++ ++ CXXScopeSpec CSS; ++ Interface = S.BuildDeclarationNameExpr(CSS, R, /*ADL=*/false).get(); ++ return true; ++ }; ++ ++ if (!LookupInterface(ValuePrintingInfo[NoAlloc], ++ MagicRuntimeInterface[NoAlloc])) ++ return false; ++ if (!LookupInterface(ValuePrintingInfo[WithAlloc], ++ MagicRuntimeInterface[WithAlloc])) ++ return false; ++ if (!LookupInterface(ValuePrintingInfo[CopyArray], ++ MagicRuntimeInterface[CopyArray])) ++ return false; ++ return true; ++} ++ ++namespace { ++ ++class RuntimeInterfaceBuilder ++ : public TypeVisitor { ++ clang::Interpreter &Interp; ++ ASTContext &Ctx; ++ Sema &S; ++ Expr *E; ++ llvm::SmallVector Args; ++ ++public: ++ RuntimeInterfaceBuilder(clang::Interpreter &In, ASTContext &C, Sema &SemaRef, ++ Expr *VE, ArrayRef FixedArgs) ++ : Interp(In), Ctx(C), S(SemaRef), E(VE) { ++ // The Interpreter* parameter and the out parameter `OutVal`. ++ for (Expr *E : FixedArgs) ++ Args.push_back(E); ++ ++ // Get rid of ExprWithCleanups. ++ if (auto *EWC = llvm::dyn_cast_if_present(E)) ++ E = EWC->getSubExpr(); ++ } ++ ++ExprResult getCall() { ++ QualType Ty = E->getType(); ++ QualType DesugaredTy = Ty.getDesugaredType(Ctx); ++ ++ // For lvalue struct, we treat it as a reference. ++ if (DesugaredTy->isRecordType() && E->isLValue()) { ++ DesugaredTy = Ctx.getLValueReferenceType(DesugaredTy); ++ Ty = Ctx.getLValueReferenceType(Ty); ++ } ++ ++ Expr *TypeArg = ++ CStyleCastPtrExpr(S, Ctx.VoidPtrTy, (uintptr_t)Ty.getAsOpaquePtr()); ++ // The QualType parameter `OpaqueType`, represented as `void*`. ++ Args.push_back(TypeArg); ++ ++ // We push the last parameter based on the type of the Expr. Note we need ++ // special care for rvalue struct. ++ Interpreter::InterfaceKind Kind = Visit(&*DesugaredTy); ++ switch (Kind) { ++ case Interpreter::InterfaceKind::WithAlloc: ++ case Interpreter::InterfaceKind::CopyArray: { ++ // __clang_Interpreter_SetValueWithAlloc. ++ ExprResult AllocCall = S.ActOnCallExpr( ++ /*Scope=*/nullptr, ++ Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::WithAlloc], ++ E->getBeginLoc(), Args, E->getEndLoc()); ++ assert(!AllocCall.isInvalid() && "Can't create runtime interface call!"); ++ ++ TypeSourceInfo *TSI = Ctx.getTrivialTypeSourceInfo(Ty, SourceLocation()); ++ ++ // Force CodeGen to emit destructor. ++ if (auto *RD = Ty->getAsCXXRecordDecl()) { ++ auto *Dtor = S.LookupDestructor(RD); ++ Dtor->addAttr(UsedAttr::CreateImplicit(Ctx)); ++ Interp.getCompilerInstance()->getASTConsumer().HandleTopLevelDecl( ++ DeclGroupRef(Dtor)); ++ } ++ ++ // __clang_Interpreter_SetValueCopyArr. ++ if (Kind == Interpreter::InterfaceKind::CopyArray) { ++ const auto *ConstantArrTy = ++ cast(DesugaredTy.getTypePtr()); ++ size_t ArrSize = Ctx.getConstantArrayElementCount(ConstantArrTy); ++ Expr *ArrSizeExpr = IntegerLiteralExpr(Ctx, ArrSize); ++ Expr *Args[] = {E, AllocCall.get(), ArrSizeExpr}; ++ return S.ActOnCallExpr( ++ /*Scope *=*/nullptr, ++ Interp ++ .getValuePrintingInfo()[Interpreter::InterfaceKind::CopyArray], ++ SourceLocation(), Args, SourceLocation()); ++ } ++ Expr *Args[] = {AllocCall.get()}; ++ ExprResult CXXNewCall = S.BuildCXXNew( ++ E->getSourceRange(), ++ /*UseGlobal=*/true, /*PlacementLParen=*/SourceLocation(), Args, ++ /*PlacementRParen=*/SourceLocation(), ++ /*TypeIdParens=*/SourceRange(), TSI->getType(), TSI, std::nullopt, ++ E->getSourceRange(), E); ++ ++ assert(!CXXNewCall.isInvalid() && ++ "Can't create runtime placement new call!"); ++ ++ return S.ActOnFinishFullExpr(CXXNewCall.get(), ++ /*DiscardedValue=*/false); ++ } ++ // __clang_Interpreter_SetValueNoAlloc. ++ case Interpreter::InterfaceKind::NoAlloc: { ++ return S.ActOnCallExpr( ++ /*Scope=*/nullptr, ++ Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::NoAlloc], ++ E->getBeginLoc(), Args, E->getEndLoc()); ++ } ++ default: llvm_unreachable("Unknown InterfaceKind."); ++ } ++ } ++ ++ Interpreter::InterfaceKind VisitRecordType(const RecordType *Ty) { ++ return Interpreter::InterfaceKind::WithAlloc; ++ } ++ ++ Interpreter::InterfaceKind ++ VisitMemberPointerType(const MemberPointerType *Ty) { ++ llvm_unreachable("Not implemented yet"); ++ } ++ Interpreter::InterfaceKind ++ VisitConstantArrayType(const ConstantArrayType *Ty) { ++ return Interpreter::InterfaceKind::CopyArray; ++ } ++ ++ Interpreter::InterfaceKind VisitPointerType(const PointerType *Ty) { ++ TypeSourceInfo *TSI = Ctx.getTrivialTypeSourceInfo(Ctx.VoidPtrTy); ++ ExprResult CastedExpr = ++ S.BuildCStyleCastExpr(SourceLocation(), TSI, SourceLocation(), E); ++ assert(!CastedExpr.isInvalid() && "Can not create cstyle cast expression"); ++ Args.push_back(CastedExpr.get()); ++ return Interpreter::InterfaceKind::NoAlloc; ++ } ++ ++ Interpreter::InterfaceKind VisitReferenceType(const ReferenceType *Ty) { ++ ExprResult AddrOfE = S.CreateBuiltinUnaryOp(SourceLocation(), UO_AddrOf, E); ++ assert(!AddrOfE.isInvalid() && "Can not create unary expression"); ++ Args.push_back(AddrOfE.get()); ++ return Interpreter::InterfaceKind::NoAlloc; ++ } ++ ++ Interpreter::InterfaceKind VisitBuiltinType(const BuiltinType *Ty) { ++ if (Ty->isNullPtrType()) ++ Args.push_back(E); ++ else if (Ty->isFloatingType()) ++ Args.push_back(E); ++ else if (Ty->isVoidType()) ++ Args.push_back(E); ++ else if (Ty->isIntegralOrEnumerationType()) ++ HandleIntegralOrEnumType(Ty); ++ ++ return Interpreter::InterfaceKind::NoAlloc; ++ } ++ ++ Interpreter::InterfaceKind VisitEnumType(const EnumType *Ty) { ++ HandleIntegralOrEnumType(Ty); ++ return Interpreter::InterfaceKind::NoAlloc; ++ } ++ ++private: ++ // Force cast these types to uint64 to reduce the number of overloads of ++ // `__clang_Interpreter_SetValueNoAlloc`. ++ void HandleIntegralOrEnumType(const Type *Ty) { ++ TypeSourceInfo *TSI = Ctx.getTrivialTypeSourceInfo(Ctx.UnsignedLongLongTy); ++ ExprResult CastedExpr = ++ S.BuildCStyleCastExpr(SourceLocation(), TSI, SourceLocation(), E); ++ assert(!CastedExpr.isInvalid() && "Cannot create cstyle cast expr"); ++ Args.push_back(CastedExpr.get()); ++ } ++}; ++} // namespace ++ ++// This synthesizes a call expression to a speciall ++// function that is responsible for generating the Value. ++// In general, we transform: ++// clang-repl> x ++// To: ++// // 1. If x is a built-in type like int, float. ++// __clang_Interpreter_SetValueNoAlloc(ThisInterp, OpaqueValue, xQualType, x); ++// // 2. If x is a struct, and a lvalue. ++// __clang_Interpreter_SetValueNoAlloc(ThisInterp, OpaqueValue, xQualType, ++// &x); ++// // 3. If x is a struct, but a rvalue. ++// new (__clang_Interpreter_SetValueWithAlloc(ThisInterp, OpaqueValue, ++// xQualType)) (x); ++ ++Expr *Interpreter::SynthesizeExpr(Expr *E) { ++ Sema &S = getCompilerInstance()->getSema(); ++ ASTContext &Ctx = S.getASTContext(); ++ ++ if (!FindRuntimeInterface()) ++ llvm_unreachable("We can't find the runtime iterface for pretty print!"); ++ ++ // Create parameter `ThisInterp`. ++ auto *ThisInterp = CStyleCastPtrExpr(S, Ctx.VoidPtrTy, (uintptr_t)this); ++ ++ // Create parameter `OutVal`. ++ auto *OutValue = CStyleCastPtrExpr(S, Ctx.VoidPtrTy, (uintptr_t)&LastValue); ++ ++ // Build `__clang_Interpreter_SetValue*` call. ++ RuntimeInterfaceBuilder Builder(*this, Ctx, S, E, {ThisInterp, OutValue}); ++ ++ ExprResult Result = Builder.getCall(); ++ assert(!Result.isInvalid() && "Failed to generate the CallExpr!"); ++ return Result.get(); ++} ++ ++// Temporary rvalue struct that need special care. ++REPL_EXTERNAL_VISIBILITY void * ++__clang_Interpreter_SetValueWithAlloc(void *This, void *OutVal, ++ void *OpaqueType) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ return VRef.getPtr(); ++} ++ ++// Pointers, lvalue struct that can take as a reference. ++REPL_EXTERNAL_VISIBILITY void ++__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ++ void *Val) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ VRef.setPtr(Val); ++} ++ ++REPL_EXTERNAL_VISIBILITY void ++__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ++ unsigned long long Val) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ VRef.setULongLong(Val); ++} ++ ++REPL_EXTERNAL_VISIBILITY void ++__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ++ float Val) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ VRef.setFloat(Val); ++} ++ ++REPL_EXTERNAL_VISIBILITY void ++__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ++ double Val) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ VRef.setDouble(Val); ++} ++ ++REPL_EXTERNAL_VISIBILITY void ++__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ++ long double Val) { ++ Value &VRef = *(Value *)OutVal; ++ VRef = Value(static_cast(This), OpaqueType); ++ VRef.setLongDouble(Val); ++} +diff --git a/clang/lib/Interpreter/InterpreterUtils.cpp b/clang/lib/Interpreter/InterpreterUtils.cpp +new file mode 100644 +index 000000000000..1ec2bf54987b +--- /dev/null ++++ b/clang/lib/Interpreter/InterpreterUtils.cpp +@@ -0,0 +1,111 @@ ++//===--- InterpreterUtils.cpp - Incremental Utils --------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements some common utils used in the incremental library. ++// ++//===----------------------------------------------------------------------===// ++ ++#include "InterpreterUtils.h" ++ ++namespace clang { ++ ++IntegerLiteral *IntegerLiteralExpr(ASTContext &C, uintptr_t Ptr) { ++ const llvm::APInt Addr(8 * sizeof(void *), Ptr); ++ return IntegerLiteral::Create(C, Addr, C.getUIntPtrType(), SourceLocation()); ++} ++ ++Expr *CStyleCastPtrExpr(Sema &S, QualType Ty, Expr *E) { ++ ASTContext &Ctx = S.getASTContext(); ++ if (!Ty->isPointerType()) ++ Ty = Ctx.getPointerType(Ty); ++ ++ TypeSourceInfo *TSI = Ctx.getTrivialTypeSourceInfo(Ty, SourceLocation()); ++ Expr *Result = ++ S.BuildCStyleCastExpr(SourceLocation(), TSI, SourceLocation(), E).get(); ++ assert(Result && "Cannot create CStyleCastPtrExpr"); ++ return Result; ++} ++ ++Expr *CStyleCastPtrExpr(Sema &S, QualType Ty, uintptr_t Ptr) { ++ ASTContext &Ctx = S.getASTContext(); ++ return CStyleCastPtrExpr(S, Ty, IntegerLiteralExpr(Ctx, Ptr)); ++} ++ ++Sema::DeclGroupPtrTy CreateDGPtrFrom(Sema &S, Decl *D) { ++ SmallVector DeclsInGroup; ++ DeclsInGroup.push_back(D); ++ Sema::DeclGroupPtrTy DeclGroupPtr = S.BuildDeclaratorGroup(DeclsInGroup); ++ return DeclGroupPtr; ++} ++ ++NamespaceDecl *LookupNamespace(Sema &S, llvm::StringRef Name, ++ const DeclContext *Within) { ++ DeclarationName DName = &S.Context.Idents.get(Name); ++ LookupResult R(S, DName, SourceLocation(), ++ Sema::LookupNestedNameSpecifierName); ++ R.suppressDiagnostics(); ++ if (!Within) ++ S.LookupName(R, S.TUScope); ++ else { ++ if (const clang::TagDecl *TD = dyn_cast(Within)) ++ if (!TD->getDefinition()) ++ // No definition, no lookup result. ++ return nullptr; ++ ++ S.LookupQualifiedName(R, const_cast(Within)); ++ } ++ ++ if (R.empty()) ++ return nullptr; ++ ++ R.resolveKind(); ++ ++ return dyn_cast(R.getFoundDecl()); ++} ++ ++NamedDecl *LookupNamed(Sema &S, llvm::StringRef Name, ++ const DeclContext *Within) { ++ DeclarationName DName = &S.Context.Idents.get(Name); ++ LookupResult R(S, DName, SourceLocation(), Sema::LookupOrdinaryName, ++ Sema::ForVisibleRedeclaration); ++ ++ R.suppressDiagnostics(); ++ ++ if (!Within) ++ S.LookupName(R, S.TUScope); ++ else { ++ const DeclContext *PrimaryWithin = nullptr; ++ if (const auto *TD = dyn_cast(Within)) ++ PrimaryWithin = llvm::dyn_cast_or_null(TD->getDefinition()); ++ else ++ PrimaryWithin = Within->getPrimaryContext(); ++ ++ // No definition, no lookup result. ++ if (!PrimaryWithin) ++ return nullptr; ++ ++ S.LookupQualifiedName(R, const_cast(PrimaryWithin)); ++ } ++ ++ if (R.empty()) ++ return nullptr; ++ R.resolveKind(); ++ ++ if (R.isSingleResult()) ++ return llvm::dyn_cast(R.getFoundDecl()); ++ ++ return nullptr; ++} ++ ++std::string GetFullTypeName(ASTContext &Ctx, QualType QT) { ++ PrintingPolicy Policy(Ctx.getPrintingPolicy()); ++ Policy.SuppressScope = false; ++ Policy.AnonymousTagLocations = false; ++ return QT.getAsString(Policy); ++} ++} // namespace clang +diff --git a/clang/lib/Interpreter/InterpreterUtils.h b/clang/lib/Interpreter/InterpreterUtils.h +new file mode 100644 +index 000000000000..61f4cf7e1239 +--- /dev/null ++++ b/clang/lib/Interpreter/InterpreterUtils.h +@@ -0,0 +1,55 @@ ++//===--- InterpreterUtils.h - Incremental Utils --------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements some common utils used in the incremental library. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_CLANG_INTERPRETER_UTILS_H ++#define LLVM_CLANG_INTERPRETER_UTILS_H ++ ++#include "clang/AST/ASTContext.h" ++#include "clang/AST/Mangle.h" ++#include "clang/AST/TypeVisitor.h" ++#include "clang/Basic/TargetInfo.h" ++#include "clang/CodeGen/ModuleBuilder.h" ++#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" ++#include "clang/Driver/Compilation.h" ++#include "clang/Driver/Driver.h" ++#include "clang/Driver/Job.h" ++#include "clang/Driver/Options.h" ++#include "clang/Driver/Tool.h" ++#include "clang/Frontend/CompilerInstance.h" ++#include "clang/Frontend/TextDiagnosticBuffer.h" ++#include "clang/Lex/PreprocessorOptions.h" ++ ++#include "clang/Sema/Lookup.h" ++#include "llvm/IR/Module.h" ++#include "llvm/Support/Errc.h" ++#include "llvm/TargetParser/Host.h" ++ ++// TODO: create a sub namespace `repl`. ++namespace clang { ++IntegerLiteral *IntegerLiteralExpr(ASTContext &C, uintptr_t Ptr); ++ ++Expr *CStyleCastPtrExpr(Sema &S, QualType Ty, Expr *E); ++ ++Expr *CStyleCastPtrExpr(Sema &S, QualType Ty, uintptr_t Ptr); ++ ++Sema::DeclGroupPtrTy CreateDGPtrFrom(Sema &S, Decl *D); ++ ++NamespaceDecl *LookupNamespace(Sema &S, llvm::StringRef Name, ++ const DeclContext *Within = nullptr); ++ ++NamedDecl *LookupNamed(Sema &S, llvm::StringRef Name, ++ const DeclContext *Within); ++ ++std::string GetFullTypeName(ASTContext &Ctx, QualType QT); ++} // namespace clang ++ ++#endif +diff --git a/clang/lib/Interpreter/Value.cpp b/clang/lib/Interpreter/Value.cpp +new file mode 100644 +index 000000000000..34a77344bc86 +--- /dev/null ++++ b/clang/lib/Interpreter/Value.cpp +@@ -0,0 +1,260 @@ ++//===--- Interpreter.h - Incremental Compiation and Execution---*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file defines the class that used to represent a value in incremental ++// C++. ++// ++//===----------------------------------------------------------------------===// ++ ++#include "clang/Interpreter/Value.h" ++#include "clang/AST/ASTContext.h" ++#include "clang/AST/Type.h" ++#include "clang/Interpreter/Interpreter.h" ++#include "llvm/ADT/StringExtras.h" ++#include "llvm/Support/ErrorHandling.h" ++#include "llvm/Support/raw_os_ostream.h" ++#include ++#include ++#include ++ ++using namespace clang; ++ ++namespace { ++ ++// This is internal buffer maintained by Value, used to hold temporaries. ++class ValueStorage { ++public: ++ using DtorFunc = void (*)(void *); ++ ++ static unsigned char *CreatePayload(void *DtorF, size_t AllocSize, ++ size_t ElementsSize) { ++ if (AllocSize < sizeof(Canary)) ++ AllocSize = sizeof(Canary); ++ unsigned char *Buf = ++ new unsigned char[ValueStorage::getPayloadOffset() + AllocSize]; ++ ValueStorage *VS = new (Buf) ValueStorage(DtorF, AllocSize, ElementsSize); ++ std::memcpy(VS->getPayload(), Canary, sizeof(Canary)); ++ return VS->getPayload(); ++ } ++ ++ unsigned char *getPayload() { return Storage; } ++ const unsigned char *getPayload() const { return Storage; } ++ ++ static unsigned getPayloadOffset() { ++ static ValueStorage Dummy(nullptr, 0, 0); ++ return Dummy.getPayload() - reinterpret_cast(&Dummy); ++ } ++ ++ static ValueStorage *getFromPayload(void *Payload) { ++ ValueStorage *R = reinterpret_cast( ++ (unsigned char *)Payload - getPayloadOffset()); ++ return R; ++ } ++ ++ void Retain() { ++RefCnt; } ++ ++ void Release() { ++ assert(RefCnt > 0 && "Can't release if reference count is already zero"); ++ if (--RefCnt == 0) { ++ // We hace a non-trivial dtor. ++ if (Dtor && IsAlive()) { ++ assert(Elements && "We at least should have 1 element in Value"); ++ size_t Stride = AllocSize / Elements; ++ for (size_t Idx = 0; Idx < Elements; ++Idx) ++ (*Dtor)(getPayload() + Idx * Stride); ++ } ++ delete[] reinterpret_cast(this); ++ } ++ } ++ ++ bool IsAlive() const { ++ return std::memcmp(getPayload(), Canary, sizeof(Canary)) != 0; ++ } ++ ++private: ++ ValueStorage(void *DtorF, size_t AllocSize, size_t ElementsNum) ++ : RefCnt(1), Dtor(reinterpret_cast(DtorF)), ++ AllocSize(AllocSize), Elements(ElementsNum) {} ++ ++ mutable unsigned RefCnt; ++ DtorFunc Dtor = nullptr; ++ size_t AllocSize = 0; ++ size_t Elements = 0; ++ unsigned char Storage[1]; ++ ++ static constexpr unsigned char Canary[8] = {0x4c, 0x37, 0xad, 0x8f, ++ 0x2d, 0x23, 0x95, 0x91}; ++}; ++} // namespace ++ ++static Value::Kind ConvertQualTypeToKind(const ASTContext &Ctx, QualType QT) { ++ if (Ctx.hasSameType(QT, Ctx.VoidTy)) ++ return Value::K_Void; ++ ++ if (const auto *ET = dyn_cast(QT.getTypePtr())) ++ QT = ET->getDecl()->getIntegerType(); ++ ++ if (!QT->isBuiltinType() || QT->castAs()->isNullPtrType()) ++ return Value::K_PtrOrObj; ++ ++ switch (QT->getAs()->getKind()) { ++ default: ++ assert(false && "Type not supported"); ++ return Value::K_Unspecified; ++#define X(type, name) \ ++ case BuiltinType::name: \ ++ return Value::K_##name; ++ REPL_BUILTIN_TYPES ++#undef X ++ } ++} ++ ++Value::Value(Interpreter *In, void *Ty) : Interp(In), OpaqueType(Ty) { ++ setKind(ConvertQualTypeToKind(getASTContext(), getType())); ++ if (ValueKind == K_PtrOrObj) { ++ QualType Canon = getType().getCanonicalType(); ++ if ((Canon->isPointerType() || Canon->isObjectType() || ++ Canon->isReferenceType()) && ++ (Canon->isRecordType() || Canon->isConstantArrayType() || ++ Canon->isMemberPointerType())) { ++ IsManuallyAlloc = true; ++ // Compile dtor function. ++ Interpreter &Interp = getInterpreter(); ++ void *DtorF = nullptr; ++ size_t ElementsSize = 1; ++ QualType DtorTy = getType(); ++ ++ if (const auto *ArrTy = ++ llvm::dyn_cast(DtorTy.getTypePtr())) { ++ DtorTy = ArrTy->getElementType(); ++ llvm::APInt ArrSize(sizeof(size_t) * 8, 1); ++ do { ++ ArrSize *= ArrTy->getSize(); ++ ArrTy = llvm::dyn_cast( ++ ArrTy->getElementType().getTypePtr()); ++ } while (ArrTy); ++ ElementsSize = static_cast(ArrSize.getZExtValue()); ++ } ++ if (const auto *RT = DtorTy->getAs()) { ++ if (CXXRecordDecl *CXXRD = ++ llvm::dyn_cast(RT->getDecl())) { ++ if (llvm::Expected Addr = ++ Interp.CompileDtorCall(CXXRD)) ++ DtorF = reinterpret_cast(Addr->getValue()); ++ else ++ llvm::logAllUnhandledErrors(Addr.takeError(), llvm::errs()); ++ } ++ } ++ ++ size_t AllocSize = ++ getASTContext().getTypeSizeInChars(getType()).getQuantity(); ++ unsigned char *Payload = ++ ValueStorage::CreatePayload(DtorF, AllocSize, ElementsSize); ++ setPtr((void *)Payload); ++ } ++ } ++} ++ ++Value::Value(const Value &RHS) ++ : Interp(RHS.Interp), OpaqueType(RHS.OpaqueType), Data(RHS.Data), ++ ValueKind(RHS.ValueKind), IsManuallyAlloc(RHS.IsManuallyAlloc) { ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Retain(); ++} ++ ++Value::Value(Value &&RHS) noexcept { ++ Interp = std::exchange(RHS.Interp, nullptr); ++ OpaqueType = std::exchange(RHS.OpaqueType, nullptr); ++ Data = RHS.Data; ++ ValueKind = std::exchange(RHS.ValueKind, K_Unspecified); ++ IsManuallyAlloc = std::exchange(RHS.IsManuallyAlloc, false); ++ ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Release(); ++} ++ ++Value &Value::operator=(const Value &RHS) { ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Release(); ++ ++ Interp = RHS.Interp; ++ OpaqueType = RHS.OpaqueType; ++ Data = RHS.Data; ++ ValueKind = RHS.ValueKind; ++ IsManuallyAlloc = RHS.IsManuallyAlloc; ++ ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Retain(); ++ ++ return *this; ++} ++ ++Value &Value::operator=(Value &&RHS) noexcept { ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Release(); ++ ++ Interp = std::exchange(RHS.Interp, nullptr); ++ OpaqueType = std::exchange(RHS.OpaqueType, nullptr); ++ ValueKind = std::exchange(RHS.ValueKind, K_Unspecified); ++ IsManuallyAlloc = std::exchange(RHS.IsManuallyAlloc, false); ++ ++ Data = RHS.Data; ++ ++ return *this; ++} ++ ++void Value::clear() { ++ if (IsManuallyAlloc) ++ ValueStorage::getFromPayload(getPtr())->Release(); ++ ValueKind = K_Unspecified; ++ OpaqueType = nullptr; ++ Interp = nullptr; ++ IsManuallyAlloc = false; ++} ++ ++Value::~Value() { clear(); } ++ ++void *Value::getPtr() const { ++ assert(ValueKind == K_PtrOrObj); ++ return Data.m_Ptr; ++} ++ ++QualType Value::getType() const { ++ return QualType::getFromOpaquePtr(OpaqueType); ++} ++ ++Interpreter &Value::getInterpreter() { ++ assert(Interp != nullptr && ++ "Can't get interpreter from a default constructed value"); ++ return *Interp; ++} ++ ++const Interpreter &Value::getInterpreter() const { ++ assert(Interp != nullptr && ++ "Can't get interpreter from a default constructed value"); ++ return *Interp; ++} ++ ++ASTContext &Value::getASTContext() { return getInterpreter().getASTContext(); } ++ ++const ASTContext &Value::getASTContext() const { ++ return getInterpreter().getASTContext(); ++} ++ ++void Value::dump() const { print(llvm::outs()); } ++ ++void Value::printType(llvm::raw_ostream &Out) const { ++ Out << "Not implement yet.\n"; ++} ++void Value::printData(llvm::raw_ostream &Out) const { ++ Out << "Not implement yet.\n"; ++} ++void Value::print(llvm::raw_ostream &Out) const { ++ assert(OpaqueType != nullptr && "Can't print default Value"); ++ Out << "Not implement yet.\n"; ++} +diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp +index 66168467ecf5..0822f83b58df 100644 +--- a/clang/lib/Lex/PPLexerChange.cpp ++++ b/clang/lib/Lex/PPLexerChange.cpp +@@ -526,13 +526,19 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { + return LeavingSubmodule; + } + } +- + // If this is the end of the main file, form an EOF token. + assert(CurLexer && "Got EOF but no current lexer set!"); + const char *EndPos = getCurLexerEndPos(); + Result.startToken(); + CurLexer->BufferPtr = EndPos; +- CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); ++ ++ if (isIncrementalProcessingEnabled()) { ++ CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end); ++ Result.setAnnotationEndLoc(Result.getLocation()); ++ Result.setAnnotationValue(nullptr); ++ } else { ++ CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); ++ } + + if (isCodeCompletionEnabled()) { + // Inserting the code-completion point increases the source buffer by 1, +diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp +index 3a7f5426d4a7..57a3dfba4f1d 100644 +--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp ++++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp +@@ -836,6 +836,7 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2, + case tok::annot_module_begin: + case tok::annot_module_end: + case tok::annot_module_include: ++ case tok::annot_repl_input_end: + // Ran out of tokens. + return false; + +@@ -1242,6 +1243,7 @@ bool Parser::ConsumeAndStoreInitializer(CachedTokens &Toks, + case tok::annot_module_begin: + case tok::annot_module_end: + case tok::annot_module_include: ++ case tok::annot_repl_input_end: + // Ran out of tokens. + return false; + +diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp +index e6812ac72c88..3e64e6f074fc 100644 +--- a/clang/lib/Parse/ParseDecl.cpp ++++ b/clang/lib/Parse/ParseDecl.cpp +@@ -2030,6 +2030,7 @@ void Parser::SkipMalformedDecl() { + case tok::annot_module_begin: + case tok::annot_module_end: + case tok::annot_module_include: ++ case tok::annot_repl_input_end: + return; + + default: +@@ -5394,6 +5395,13 @@ Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() { + + SmallVector DeclsInGroup; + DeclsInGroup.push_back(Actions.ActOnTopLevelStmtDecl(R.get())); ++ ++ if (Tok.is(tok::annot_repl_input_end) && ++ Tok.getAnnotationValue() != nullptr) { ++ ConsumeAnnotationToken(); ++ cast(DeclsInGroup.back())->setValuePrinting(); ++ } ++ + // Currently happens for things like -fms-extensions and use `__if_exists`. + for (Stmt *S : Stmts) + DeclsInGroup.push_back(Actions.ActOnTopLevelStmtDecl(S)); +diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp +index 1c8441fafc48..446fc7ea6385 100644 +--- a/clang/lib/Parse/ParseStmt.cpp ++++ b/clang/lib/Parse/ParseStmt.cpp +@@ -543,9 +543,22 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) { + return ParseCaseStatement(StmtCtx, /*MissingCase=*/true, Expr); + } + +- // Otherwise, eat the semicolon. +- ExpectAndConsumeSemi(diag::err_expected_semi_after_expr); +- return handleExprStmt(Expr, StmtCtx); ++ Token *CurTok = nullptr; ++ // If we're parsing an ExprStmt and the last semicolon is missing and the ++ // incremental externsion is enabled and we're reaching the end, consider we ++ // want to do value printing. Note we shouldn't eat the token since the ++ // callback need it. ++ if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::annot_repl_input_end)) ++ CurTok = &Tok; ++ else ++ // Otherwise, eat the semicolon. ++ ExpectAndConsumeSemi(diag::err_expected_semi_after_expr); ++ ++ StmtResult R = handleExprStmt(Expr, StmtCtx); ++ if (!R.isInvalid() && CurTok) ++ CurTok->setAnnotationValue(R.get()); ++ ++ return R; + } + + /// ParseSEHTryBlockCommon +diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp +index 6db3dc3156fd..9682f12ed211 100644 +--- a/clang/lib/Parse/Parser.cpp ++++ b/clang/lib/Parse/Parser.cpp +@@ -319,6 +319,7 @@ bool Parser::SkipUntil(ArrayRef Toks, SkipUntilFlags Flags) { + case tok::annot_module_begin: + case tok::annot_module_end: + case tok::annot_module_include: ++ case tok::annot_repl_input_end: + // Stop before we change submodules. They generally indicate a "good" + // place to pick up parsing again (except in the special case where + // we're trying to skip to EOF). +@@ -614,8 +615,8 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, + + // Skip over the EOF token, flagging end of previous input for incremental + // processing +- if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof)) +- ConsumeToken(); ++ if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::annot_repl_input_end)) ++ ConsumeAnnotationToken(); + + Result = nullptr; + switch (Tok.getKind()) { +@@ -695,6 +696,7 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, + return false; + + case tok::eof: ++ case tok::annot_repl_input_end: + // Check whether -fmax-tokens= was reached. + if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) { + PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total) +diff --git a/clang/tools/clang-repl/CMakeLists.txt b/clang/tools/clang-repl/CMakeLists.txt +index b51a18c10cdc..15d7f9439ff5 100644 +--- a/clang/tools/clang-repl/CMakeLists.txt ++++ b/clang/tools/clang-repl/CMakeLists.txt +@@ -12,6 +12,7 @@ add_clang_tool(clang-repl + ) + + clang_target_link_libraries(clang-repl PRIVATE ++ clangAST + clangBasic + clangFrontend + clangInterpreter +diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt +index 1a099dbbfe59..698494b9897f 100644 +--- a/clang/unittests/Interpreter/CMakeLists.txt ++++ b/clang/unittests/Interpreter/CMakeLists.txt +@@ -22,3 +22,5 @@ target_link_libraries(ClangReplInterpreterTests PUBLIC + if(NOT WIN32) + add_subdirectory(ExceptionTests) + endif() ++ ++export_executable_symbols(ClangReplInterpreterTests) +diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp +index d4900a0e4de8..4e737a0b313a 100644 +--- a/clang/unittests/Interpreter/InterpreterTest.cpp ++++ b/clang/unittests/Interpreter/InterpreterTest.cpp +@@ -17,6 +17,7 @@ + #include "clang/AST/Mangle.h" + #include "clang/Frontend/CompilerInstance.h" + #include "clang/Frontend/TextDiagnosticPrinter.h" ++#include "clang/Interpreter/Value.h" + #include "clang/Sema/Lookup.h" + #include "clang/Sema/Sema.h" + +@@ -33,6 +34,10 @@ using namespace clang; + #define CLANG_INTERPRETER_NO_SUPPORT_EXEC + #endif + ++int Global = 42; ++int getGlobal() { return Global; } ++void setGlobal(int val) { Global = val; } ++ + namespace { + using Args = std::vector; + static std::unique_ptr +@@ -276,8 +281,7 @@ TEST(IncrementalProcessing, InstantiateTemplate) { + std::vector Args = {"-fno-delayed-template-parsing"}; + std::unique_ptr Interp = createInterpreter(Args); + +- llvm::cantFail(Interp->Parse("void* operator new(__SIZE_TYPE__, void* __p);" +- "extern \"C\" int printf(const char*,...);" ++ llvm::cantFail(Interp->Parse("extern \"C\" int printf(const char*,...);" + "class A {};" + "struct B {" + " template" +@@ -314,4 +318,55 @@ TEST(IncrementalProcessing, InstantiateTemplate) { + free(NewA); + } + ++TEST(InterpreterTest, Value) { ++ std::unique_ptr Interp = createInterpreter(); ++ ++ Value V1; ++ llvm::cantFail(Interp->ParseAndExecute("int x = 42;")); ++ llvm::cantFail(Interp->ParseAndExecute("x", &V1)); ++ EXPECT_TRUE(V1.isValid()); ++ EXPECT_EQ(V1.getInt(), 42); ++ EXPECT_TRUE(V1.getType()->isIntegerType()); ++ EXPECT_EQ(V1.getKind(), Value::K_Int); ++ EXPECT_FALSE(V1.isManuallyAlloc()); ++ EXPECT_FALSE(V1.isPointerOrObjectType()); ++ ++ Value V2; ++ llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;")); ++ llvm::cantFail(Interp->ParseAndExecute("y", &V2)); ++ EXPECT_TRUE(V2.isValid()); ++ EXPECT_EQ(V2.getDouble(), 3.14); ++ EXPECT_TRUE(V2.getType()->isFloatingType()); ++ EXPECT_EQ(V2.getKind(), Value::K_Double); ++ EXPECT_FALSE(V2.isManuallyAlloc()); ++ EXPECT_FALSE(V2.isPointerOrObjectType()); ++ ++ Value V3; ++ llvm::cantFail(Interp->ParseAndExecute( ++ "struct S { int* p; S() { p = new int(42); } ~S() { delete p; }};")); ++ llvm::cantFail(Interp->ParseAndExecute("S{}", &V3)); ++ EXPECT_TRUE(V3.isValid()); ++ EXPECT_TRUE(V3.getType()->isRecordType()); ++ EXPECT_EQ(V3.getKind(), Value::K_PtrOrObj); ++ EXPECT_TRUE(V3.isManuallyAlloc()); ++ EXPECT_TRUE(V3.isPointerOrObjectType()); ++ ++ Value V4; ++ llvm::cantFail(Interp->ParseAndExecute("int getGlobal();")); ++ llvm::cantFail(Interp->ParseAndExecute("void setGlobal(int);")); ++ llvm::cantFail(Interp->ParseAndExecute("getGlobal()", &V4)); ++ EXPECT_EQ(V4.getInt(), 42); ++ EXPECT_TRUE(V4.getType()->isIntegerType()); ++ ++ Value V5; ++ // Change the global from the compiled code. ++ setGlobal(43); ++ llvm::cantFail(Interp->ParseAndExecute("getGlobal()", &V5)); ++ EXPECT_EQ(V5.getInt(), 43); ++ EXPECT_TRUE(V5.getType()->isIntegerType()); ++ ++ // Change the global from the interpreted code. ++ llvm::cantFail(Interp->ParseAndExecute("setGlobal(44);")); ++ EXPECT_EQ(getGlobal(), 44); ++} + } // end anonymous namespace diff --git a/patches/llvm/clang16-D148435-WeakRefs.patch b/patches/llvm/clang16-D148435-WeakRefs.patch new file mode 100644 index 00000000..e04f43c2 --- /dev/null +++ b/patches/llvm/clang16-D148435-WeakRefs.patch @@ -0,0 +1,33 @@ +diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp +index 12d602fed693..e73102544361 100644 +--- a/clang/lib/CodeGen/CodeGenModule.cpp ++++ b/clang/lib/CodeGen/CodeGenModule.cpp +@@ -7202,7 +7202,6 @@ void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { + "Newly created module should not have manglings"); + NewBuilder->Manglings = std::move(Manglings); + +- assert(WeakRefReferences.empty() && "Not all WeakRefRefs have been applied"); + NewBuilder->WeakRefReferences = std::move(WeakRefReferences); + + NewBuilder->TBAA = std::move(TBAA); +diff --git a/clang/test/Interpreter/execute-weak.cpp b/clang/test/Interpreter/execute-weak.cpp +index 5b343512c545..66f2214ab03c 100644 +--- a/clang/test/Interpreter/execute-weak.cpp ++++ b/clang/test/Interpreter/execute-weak.cpp +@@ -2,11 +2,16 @@ + // RUN: clang-repl "int i = 10;" 'extern "C" int printf(const char*,...);' \ + // RUN: 'auto r1 = printf("i = %d\n", i);' | FileCheck --check-prefix=CHECK-DRIVER %s + // CHECK-DRIVER: i = 10 ++// + // UNSUPPORTED: system-aix, system-windows + // RUN: cat %s | clang-repl | FileCheck %s ++ + extern "C" int printf(const char *, ...); + int __attribute__((weak)) bar() { return 42; } + auto r4 = printf("bar() = %d\n", bar()); + // CHECK: bar() = 42 + ++int a = 12; ++static __typeof(a) b __attribute__((__weakref__("a"))); ++int c = b; + %quit diff --git a/share/jupyter/kernels/xcpp11/kernel.json.in b/share/jupyter/kernels/xcpp11/kernel.json.in index 17097f66..d3274587 100644 --- a/share/jupyter/kernels/xcpp11/kernel.json.in +++ b/share/jupyter/kernels/xcpp11/kernel.json.in @@ -1,5 +1,10 @@ { "display_name": "C++11", + "env": { + "CPLUS_INCLUDE_PATH":"@CMAKE_CPLUS_INCLUDE_PATH@", + "LD_LIBRARY_PATH":"@CMAKE_LD_LIBRARY_PATH@", + "PYTHONPATH":"@CMAKE_PYTHONPATH@" + }, "argv": [ "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp", "-f", diff --git a/share/jupyter/kernels/xcpp14/kernel.json.in b/share/jupyter/kernels/xcpp14/kernel.json.in index a9fa64eb..bf7f60ff 100644 --- a/share/jupyter/kernels/xcpp14/kernel.json.in +++ b/share/jupyter/kernels/xcpp14/kernel.json.in @@ -1,5 +1,10 @@ { "display_name": "C++14", + "env": { + "CPLUS_INCLUDE_PATH":"@CMAKE_CPLUS_INCLUDE_PATH@", + "LD_LIBRARY_PATH":"@CMAKE_LD_LIBRARY_PATH@", + "PYTHONPATH":"@CMAKE_PYTHONPATH@" + }, "argv": [ "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp", "-f", diff --git a/share/jupyter/kernels/xcpp17/kernel.json.in b/share/jupyter/kernels/xcpp17/kernel.json.in index 99c2f951..82e3d0e5 100644 --- a/share/jupyter/kernels/xcpp17/kernel.json.in +++ b/share/jupyter/kernels/xcpp17/kernel.json.in @@ -1,5 +1,10 @@ { "display_name": "C++17", + "env": { + "CPLUS_INCLUDE_PATH":"@CMAKE_CPLUS_INCLUDE_PATH@", + "LD_LIBRARY_PATH":"@CMAKE_LD_LIBRARY_PATH@", + "PYTHONPATH":"@CMAKE_PYTHONPATH@" + }, "argv": [ "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp", "-f", diff --git a/src/main.cpp b/src/main.cpp index 9f4a740e..8ace7d7d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -57,9 +57,9 @@ interpreter_ptr build_interpreter(int argc, char **argv) { for (int i = 1; i < argc; i++) { interpreter_argv[i] = argv[i]; } - std::string resource_dir = - "-resource-dir=" + std::string(CLANG_RESOURCE_DIR); - interpreter_argv[interpreter_argc - 1] = resource_dir.c_str(); + // std::string resource_dir = + // "-resource-dir=" + std::string(CLANG_RESOURCE_DIR); + // interpreter_argv[interpreter_argc - 1] = resource_dir.c_str(); interpreter_ptr interp_ptr = interpreter_ptr( new xcpp::interpreter(interpreter_argc, interpreter_argv)); diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 3049a58d..92d487ac 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -22,12 +22,6 @@ #include #include "Python.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "llvm/ExecutionEngine/Orc/LLJIT.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/TargetSelect.h" #include "xeus-clang-repl/xbuffer.hpp" #include "xeus-clang-repl/xeus_clang-repl_config.hpp" @@ -41,105 +35,29 @@ #include "xmagics/pythonexec.hpp" using namespace std::placeholders; -std::string DiagnosticOutput; -llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); -auto DiagPrinter = std::make_unique( - DiagnosticsOS, new clang::DiagnosticOptions()); - -///\returns true on error. -static bool ProcessCode(clang::Interpreter &Interp, const std::string &code, - llvm::raw_string_ostream &error_stream) { - if (xcpp::pythonexec::python_check_for_initialisation()) { - std::string newPythonInts = - xcpp::pythonexec::transfer_python_ints_utility(); - llvm::cantFail(Interp.ParseAndExecute(newPythonInts)); - std::string newPythonLists = - xcpp::pythonexec::transfer_python_lists_utility(); - llvm::cantFail(Interp.ParseAndExecute(newPythonLists)); - } - - auto PTU = Interp.Parse(code); - if (!PTU) { - auto Err = PTU.takeError(); - error_stream << DiagnosticsOS.str(); - // avoid printing the "Parsing failed error" - // llvm::logAllUnhandledErrors(std::move(Err), error_stream, "error: "); - return true; - } - if (PTU->TheModule) { - llvm::Error ex = Interp.Execute(*PTU); - error_stream << DiagnosticsOS.str(); - if (code.substr(0, 3) == "int") { - for (clang::Decl *D : PTU->TUPart->decls()) { - if (clang::VarDecl *VD = llvm::dyn_cast(D)) { - auto Name = VD->getNameAsString(); - auto Addr = Interp.getSymbolAddress(clang::GlobalDecl(VD)); - if (!Addr) { - llvm::logAllUnhandledErrors(std::move(Addr.takeError()), - error_stream, "error: "); - return true; - } - void *AddrVP = (void *)*Addr; - // printf("Value at '%p' is:'%d'\n", AddrVP, *(int*)AddrVP); - xcpp::pythonexec::update_python_dict_var(Name.c_str(), - *(int *)AddrVP); - } - } - } - - else if (code.substr(0, 16) == "std::vector") { - for (clang::Decl *D : PTU->TUPart->decls()) { - if (clang::VarDecl *VD = llvm::dyn_cast(D)) { - auto Name = VD->getNameAsString(); - auto Addr = Interp.getSymbolAddress(clang::GlobalDecl(VD)); - if (!Addr) { - llvm::logAllUnhandledErrors(std::move(Addr.takeError()), - error_stream, "error: "); - return true; - } - void *AddrVP = (void *)*Addr; - xcpp::pythonexec::update_python_dict_var_vector( - Name.c_str(), *(std::vector *)AddrVP); - } - } - } - - llvm::logAllUnhandledErrors(std::move(ex), error_stream, "error: "); - return false; - } - return false; -} using Args = std::vector; -static std::unique_ptr -createInterpreter(const Args &ExtraArgs = {}, - clang::DiagnosticConsumer *Client = nullptr) { - llvm::InitializeNativeTarget(); - llvm::InitializeNativeTargetAsmPrinter(); - +void* createInterpreter(const Args &ExtraArgs = {}) { Args ClangArgs = {"-Xclang", "-emit-llvm-only", "-Xclang", "-diagnostic-log-file", "-Xclang", "-", "-xc++"}; ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end()); - auto CI = cantFail(clang::IncrementalCompilerBuilder::create(ClangArgs)); - if (Client) - CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false); - return cantFail(clang::Interpreter::create(std::move(CI))); + //return InterOp::CreateInterpreter(ClangArgs); + return Cpp::CreateInterpreter(); } namespace xcpp { void interpreter::configure_impl() {} interpreter::interpreter(int argc, const char *const *argv) - : m_interpreter(std::move(createInterpreter(Args(argv + 1, argv + argc), - DiagPrinter.get()))), - // m_input_validator(), + : // m_input_validator(), m_version(get_stdopt(argc, argv)), // Extract C++ language standard // version from command-line option xmagics(), p_cout_strbuf(nullptr), p_cerr_strbuf(nullptr), m_cout_buffer(std::bind(&interpreter::publish_stdout, this, _1)), m_cerr_buffer(std::bind(&interpreter::publish_stderr, this, _1)) { + createInterpreter(Args(argv + 1, argv + argc)/*, DiagPrinter.get()*/); // Bootstrap the execution engine redirect_output(); init_preamble(); @@ -190,9 +108,9 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/, for (const auto &block : blocks) { // Attempt normal evaluation std::string error_message; - llvm::raw_string_ostream error_stream(error_message); + std::stringstream error_stream(error_message); try { - compilation_result = ProcessCode(*m_interpreter, block, error_stream); + compilation_result = Cpp::Process(block.c_str()); redirect_output(); } @@ -216,7 +134,7 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/, // If an error was encountered, don't attempt further execution if (errorlevel) { error_stream.str().clear(); - DiagnosticsOS.str().clear(); + //DiagnosticsOS.str().clear(); break; } } @@ -447,53 +365,52 @@ static int fprintf_jit(std::FILE *stream, const char *format, ...) { return ret; } - -static void injectSymbol(llvm::StringRef LinkerMangledName, - llvm::JITTargetAddress KnownAddr, - clang::Interpreter &Interp) { - using namespace llvm; - using namespace llvm::orc; - - auto Symbol = - Interp.getSymbolAddress(LinkerMangledName); //, /*IncludeFromHost=*/true); - if (Error Err = Symbol.takeError()) { - logAllUnhandledErrors(std::move(Err), errs(), - "[IncrementalJIT] define() failed1: "); - return; - } - - // Nothing to define, we are redefining the same function. FIXME: Diagnose. - if (*Symbol && (JITTargetAddress)*Symbol == KnownAddr) - return; - - // Let's inject it - bool Inserted; - SymbolMap::iterator It; - static llvm::orc::SymbolMap m_InjectedSymbols; - - llvm::orc::LLJIT *Jit = - const_cast(Interp.getExecutionEngine()); - JITDylib &DyLib = Jit->getMainJITDylib(); - - std::tie(It, Inserted) = m_InjectedSymbols.try_emplace( - Jit->getExecutionSession().intern(LinkerMangledName), - JITEvaluatedSymbol(KnownAddr, JITSymbolFlags::Exported)); - assert(Inserted && "Why wasn't this found in the initial Jit lookup?"); - - // We want to replace a symbol with a custom provided one. - if (Symbol && KnownAddr) { - // The symbol be in the DyLib or in-process. - if (auto Err = DyLib.remove({It->first})) { - logAllUnhandledErrors(std::move(Err), errs(), - "[IncrementalJIT] define() failed2: "); - return; - } - } - - if (Error Err = DyLib.define(absoluteSymbols({*It}))) - logAllUnhandledErrors(std::move(Err), errs(), - "[IncrementalJIT] define() failed3: "); -} +// static void injectSymbol(llvm::StringRef LinkerMangledName, +// llvm::JITTargetAddress KnownAddr, +// clang::Interpreter &Interp) { +// using namespace llvm; +// using namespace llvm::orc; + +// auto Symbol = +// Interp.getSymbolAddress(LinkerMangledName); //, /*IncludeFromHost=*/true); +// if (Error Err = Symbol.takeError()) { +// logAllUnhandledErrors(std::move(Err), errs(), +// "[IncrementalJIT] define() failed1: "); +// return; +// } + +// // Nothing to define, we are redefining the same function. FIXME: Diagnose. +// // if (*Symbol && (JITTargetAddress)*Symbol == KnownAddr) +// if (*Symbol && (*Symbol).getValue() == KnownAddr) +// return; + +// // Let's inject it +// bool Inserted; +// SymbolMap::iterator It; +// static llvm::orc::SymbolMap m_InjectedSymbols; + +// llvm::orc::LLJIT &Jit = *Interp.getExecutionEngine(); +// JITDylib &DyLib = Jit.getMainJITDylib(); + +// std::tie(It, Inserted) = m_InjectedSymbols.try_emplace( +// Jit.getExecutionSession().intern(LinkerMangledName), +// JITEvaluatedSymbol(KnownAddr, JITSymbolFlags::Exported)); +// assert(Inserted && "Why wasn't this found in the initial Jit lookup?"); + +// // We want to replace a symbol with a custom provided one. +// if (Symbol && KnownAddr) { +// // The symbol be in the DyLib or in-process. +// if (auto Err = DyLib.remove({It->first})) { +// logAllUnhandledErrors(std::move(Err), errs(), +// "[IncrementalJIT] define() failed2: "); +// return; +// } +// } + +// if (Error Err = DyLib.define(absoluteSymbols({*It}))) +// logAllUnhandledErrors(std::move(Err), errs(), +// "[IncrementalJIT] define() failed3: "); +// } void interpreter::redirect_output() { p_cout_strbuf = std::cout.rdbuf(); @@ -504,10 +421,11 @@ void interpreter::redirect_output() { // Inject versions of printf and fprintf that output to std::cout // and std::cerr (see implementation above). - injectSymbol("printf", llvm::pointerToJITTargetAddress(printf_jit), - *m_interpreter); - injectSymbol("fprintf", llvm::pointerToJITTargetAddress(fprintf_jit), - *m_interpreter); + // FIXME: Move this facility to CppInterOp. + // injectSymbol("printf", llvm::pointerToJITTargetAddress(printf_jit), + // m_interpreter); + // injectSymbol("fprintf", llvm::pointerToJITTargetAddress(fprintf_jit), + // m_interpreter); // llvm::sys::DynamicLibrary::AddSymbol("printf", (void*) &printf_jit); // llvm::sys::DynamicLibrary::AddSymbol("fprintf", (void*) &fprintf_jit); } diff --git a/src/xmagics/pythonexec.cpp b/src/xmagics/pythonexec.cpp index 56672cf2..46f9b214 100644 --- a/src/xmagics/pythonexec.cpp +++ b/src/xmagics/pythonexec.cpp @@ -20,13 +20,6 @@ #include "xmagics/pythonexec.hpp" -#include "clang/Frontend/CompilerInstance.h" - -#include "llvm/ExecutionEngine/Orc/LLJIT.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/TargetSelect.h" - #include #include #include @@ -36,12 +29,37 @@ namespace xcpp { static PyObject *gMainDict = 0; + void pythonexec::startup() { - Py_Initialize(); - gMainDict = PyModule_GetDict(PyImport_AddModule(("__main__"))); - Py_INCREF(gMainDict); - if (!gMainDict) - printf("Could not add module __main__"); + Py_Initialize(); + gMainDict = PyModule_GetDict(PyImport_AddModule(("__main__"))); + Py_INCREF(gMainDict); + if (!gMainDict) + printf("Could not add module __main__"); + + PyRun_SimpleString("import sys\nprint(sys.path)"); + + // Import cppyy module + PyObject* cppyyModule = PyImport_ImportModule("cppyy"); + if (!cppyyModule) { + PyErr_Print(); + Py_Finalize(); + return; // Handle import error as needed + } + + // Retrieve the dictionary of cppyy module + PyObject* cppyyDict = PyModule_GetDict(cppyyModule); + Py_DECREF(cppyyModule); + if (!cppyyDict) { + PyErr_Print(); + Py_Finalize(); + return; // Handle retrieval error as needed + } + + // Add cppyyDict to gMainDict (if needed for further usage) + PyDict_Update(gMainDict, cppyyDict); + + Py_DECREF(cppyyDict); } xoptions pythonexec::get_options() {