From fa5e46d33ba7fcd9c3841fb812d87f4e4fe4c706 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Thu, 10 Aug 2023 04:18:36 +0000
Subject: [PATCH 01/22] Add OpenMP support

---
 Dockerfile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 0c55f360..e00a8b0f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update --yes && \
     unzip \
     curl \
     jq \
+    libomp-dev \
     # Other "our" apt installs (development and testing)
     build-essential \
     git \
@@ -256,3 +257,7 @@ RUN \
     make install && \
     # install clad in all exist kernels
     for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
+    #
+    # Add OpenMP to all kernels
+    #
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do jq '.argv += ["-fopenmp", "-lgomp"] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; done

From 7f5c07797c7d9a1d187bc0d5df52b080a9d37d18 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Thu, 10 Aug 2023 04:57:37 +0000
Subject: [PATCH 02/22] Fix docker build script

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e00a8b0f..db884bd4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -260,4 +260,4 @@ RUN \
     #
     # Add OpenMP to all kernels
     #
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do jq '.argv += ["-fopenmp", "-lgomp"] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; done
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fopenmp", "-llibomp5"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done

From 46cc31f73384565ba3508276af2b7c1ab761d41c Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Fri, 11 Aug 2023 05:08:42 +0000
Subject: [PATCH 03/22] Fix OpenMP

---
 CMakeLists.txt | 11 +++++++++++
 Dockerfile     |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 596dcc50..aa03585c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -21,6 +21,17 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 # Configuration
 # =============
 
+# OpenMP config
+find_package(OpenMP)
+if (OpenMP_CXX_FOUND)
+    message(STATUS "OpenMP::OpenMP_CXX=${OpenMP::OpenMP_CXX}")
+    message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
+    message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
+    message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")
+    message(STATUS "OpenMP_pthread_LIBRARY=${OpenMP_pthread_LIBRARY}")
+endif()
+
+
 include(GNUInstallDirs)
 
 message(STATUS "Env CPLUS_INCLUDE_PATH=$ENV{CPLUS_INCLUDE_PATH}")
diff --git a/Dockerfile b/Dockerfile
index db884bd4..33cd0f7c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -260,4 +260,4 @@ RUN \
     #
     # Add OpenMP to all kernels
     #
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fopenmp", "-llibomp5"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fopenmp", "-llibomp5"] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done

From 711d2e55d0f29af53145a861fb919499a79a48bc Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Fri, 11 Aug 2023 05:23:22 +0000
Subject: [PATCH 04/22] Fix OpenMP debug print

---
 CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aa03585c..2692d3d7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,6 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 # OpenMP config
 find_package(OpenMP)
 if (OpenMP_CXX_FOUND)
-    message(STATUS "OpenMP::OpenMP_CXX=${OpenMP::OpenMP_CXX}")
     message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
     message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
     message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")

From af6f79e8d699b5234210d5847e9bcecc6aa99837 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Fri, 11 Aug 2023 06:34:33 +0000
Subject: [PATCH 05/22] Fix OpenMP 2

---
 CMakeLists.txt                              | 6 ++++++
 Dockerfile                                  | 2 +-
 share/jupyter/kernels/xcpp11/kernel.json.in | 2 +-
 share/jupyter/kernels/xcpp14/kernel.json.in | 2 +-
 share/jupyter/kernels/xcpp17/kernel.json.in | 2 +-
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2692d3d7..31068951 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,12 +22,18 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 # =============
 
 # OpenMP config
+set(CMAKE_OMP "")
 find_package(OpenMP)
 if (OpenMP_CXX_FOUND)
     message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
     message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
+    set(omp_libs OpenMP_CXX_LIB_NAMES)
+    list(TRANSFORM omp_libs PREPEND "-l")
+    list(JOIN omp_libs " " INCLUDE_STRING)
+    message(STATUS "OpenMP_CXX_LIB_NAMES(params)=${omp_libs}")
     message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")
     message(STATUS "OpenMP_pthread_LIBRARY=${OpenMP_pthread_LIBRARY}")
+    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\", \"-l${OpenMP_gomp_LIBRARY}\", \"-l${OpenMP_pthread_LIBRARY}\"")
 endif()
 
 
diff --git a/Dockerfile b/Dockerfile
index 33cd0f7c..477e91e7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -260,4 +260,4 @@ RUN \
     #
     # Add OpenMP to all kernels
     #
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fopenmp", "-llibomp5"] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += [] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
diff --git a/share/jupyter/kernels/xcpp11/kernel.json.in b/share/jupyter/kernels/xcpp11/kernel.json.in
index d3274587..4e7f5ebb 100644
--- a/share/jupyter/kernels/xcpp11/kernel.json.in
+++ b/share/jupyter/kernels/xcpp11/kernel.json.in
@@ -9,7 +9,7 @@
       "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp",
       "-f",
       "{connection_file}",
-      "-std=c++11"
+      "-std=c++11"@CMAKE_OMP@
   ],
   "language": "C++11"
 }
diff --git a/share/jupyter/kernels/xcpp14/kernel.json.in b/share/jupyter/kernels/xcpp14/kernel.json.in
index bf7f60ff..27e4d2ad 100644
--- a/share/jupyter/kernels/xcpp14/kernel.json.in
+++ b/share/jupyter/kernels/xcpp14/kernel.json.in
@@ -13,7 +13,7 @@
       "-fno-exceptions",
       "-fopenmp=libiomp5",
       "-O2",
-      "-v"
+      "-v"@CMAKE_OMP@
   ],
   "language": "C++14"
 }
diff --git a/share/jupyter/kernels/xcpp17/kernel.json.in b/share/jupyter/kernels/xcpp17/kernel.json.in
index 82e3d0e5..a524a6d1 100644
--- a/share/jupyter/kernels/xcpp17/kernel.json.in
+++ b/share/jupyter/kernels/xcpp17/kernel.json.in
@@ -9,7 +9,7 @@
       "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp",
       "-f",
       "{connection_file}",
-      "-std=c++17"
+      "-std=c++17"@CMAKE_OMP@
   ],
   "language": "C++17"
 }

From 5066562f3d63ec4e47a34c3dcfe07b8583e7e4bd Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 12 Aug 2023 15:53:52 +0000
Subject: [PATCH 06/22] Fix system includes

---
 Dockerfile                                  |  4 +++-
 run-docker.sh                               | 10 ++++++++++
 share/jupyter/kernels/xcpp14/kernel.json.in |  1 -
 3 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100755 run-docker.sh

diff --git a/Dockerfile b/Dockerfile
index 477e91e7..a5ab74eb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -176,7 +176,9 @@ RUN \
     #
     # Build CppInterOp
     #
-    export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/../llvm/include:${PATH_TO_LLVM_BUILD}/../clang/include:${PATH_TO_LLVM_BUILD}/include:${PATH_TO_LLVM_BUILD}/tools/clang/include" && \
+    sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \
+    export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/../llvm/include:${PATH_TO_LLVM_BUILD}/../clang/include:${PATH_TO_LLVM_BUILD}/include:${PATH_TO_LLVM_BUILD}/tools/clang/include:${sys_incs%:}" && \
+    echo $CPLUS_INCLUDE_PATH && \
     git clone https://github.com/compiler-research/CppInterOp.git && \
     export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \
     export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \
diff --git a/run-docker.sh b/run-docker.sh
new file mode 100755
index 00000000..eee2baf9
--- /dev/null
+++ b/run-docker.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+jupyter-repo2docker \
+    --user-name=jovyan \
+    --image-name xeus-clang-repl \
+    --publish 8888 \
+    .
+
+#    --editable \
+#    --ref InterOpIntegration \
+#    https://github.com/alexander-penev/xeus-clang-repl.git \
diff --git a/share/jupyter/kernels/xcpp14/kernel.json.in b/share/jupyter/kernels/xcpp14/kernel.json.in
index 27e4d2ad..45d1dad5 100644
--- a/share/jupyter/kernels/xcpp14/kernel.json.in
+++ b/share/jupyter/kernels/xcpp14/kernel.json.in
@@ -11,7 +11,6 @@
       "{connection_file}",
       "-std=c++14",
       "-fno-exceptions",
-      "-fopenmp=libiomp5",
       "-O2",
       "-v"@CMAKE_OMP@
   ],

From dc44cd848eb5351ebd76ae7e0e1de946b810ac2a Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 12 Aug 2023 19:09:05 +0000
Subject: [PATCH 07/22] Pass the xcpp arguments to the interpreter.

---
 src/xinterpreter.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
index f964be71..ac69010b 100644
--- a/src/xinterpreter.cpp
+++ b/src/xinterpreter.cpp
@@ -38,13 +38,11 @@ using namespace std::placeholders;
 
 using Args = std::vector<const char *>;
 void* createInterpreter(const Args &ExtraArgs = {}) {
-  Args ClangArgs = {"-Xclang", "-emit-llvm-only",
-                    "-Xclang", "-diagnostic-log-file",
+  Args ClangArgs = {"-Xclang", "-diagnostic-log-file",
                     "-Xclang", "-",
                     "-xc++"};
   ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end());
-  //return InterOp::CreateInterpreter(ClangArgs);
-  return Cpp::CreateInterpreter();
+  return Cpp::CreateInterpreter(ClangArgs);
 }
 
 namespace xcpp {
@@ -57,7 +55,7 @@ interpreter::interpreter(int argc, const char *const *argv)
       xmagics(), p_cout_strbuf(nullptr), p_cerr_strbuf(nullptr),
       m_cout_buffer(std::bind(&interpreter::publish_stdout, this, _1)),
       m_cerr_buffer(std::bind(&interpreter::publish_stderr, this, _1)) {
-  createInterpreter(Args(argv + 1, argv + argc)/*, DiagPrinter.get()*/);
+  createInterpreter(Args(argv + 1, argv + argc - 2));
   // Bootstrap the execution engine
   redirect_output();
   init_preamble();

From 42b48e50d14a2de19e2f7b42b484840d2a5bf484 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sun, 13 Aug 2023 14:49:27 +0000
Subject: [PATCH 08/22] Add OpenMP build in LLVM/clang

---
 .github/workflows/ci.yml    |   2 +-
 CMakeLists.txt              |   9 ++-
 notebooks/openmp-demo.ipynb |  69 +++++++++++++++++++++
 src/xinterpreter.cpp        | 117 +++++++++++++++++++++++++++++-------
 4 files changed, 170 insertions(+), 27 deletions(-)
 create mode 100644 notebooks/openmp-demo.ipynb

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1f545c62..97c67822 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -85,7 +85,7 @@ jobs:
         mkdir build
         mkdir inst
         cd build
-        cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_INSTALL_PREFIX=../inst ../llvm
+        cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_INSTALL_PREFIX=../inst ../llvm
         make ClangDriverOptions clang-repl -j$(nproc --all)
         #make install -j$(nproc --all)
         cd ../..
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 31068951..6552a127 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,14 +39,17 @@ endif()
 
 include(GNUInstallDirs)
 
-message(STATUS "Env CPLUS_INCLUDE_PATH=$ENV{CPLUS_INCLUDE_PATH}")
-message(STATUS "Env LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}")
-message(STATUS "Env PYTHONPATH=$ENV{PYTHONPATH}")
 
 set(CMAKE_CPLUS_INCLUDE_PATH "$ENV{CPLUS_INCLUDE_PATH}")
 set(CMAKE_LD_LIBRARY_PATH "$ENV{LD_LIBRARY_PATH}")
 set(CMAKE_PYTHONPATH "$ENV{PYTHONPATH}")
 
+message(STATUS "Debug: Replace in kernels")
+message(STATUS "Debug: CMAKE_CPLUS_INCLUDE_PATH=${CMAKE_CPLUS_INCLUDE_PATH}")
+message(STATUS "Debug: CMAKE_LD_LIBRARY_PATH=${CMAKE_LD_LIBRARY_PATH}")
+message(STATUS "Debug: CMAKE_PYTHONPATH=${CMAKE_PYTHONPATH}")
+message(STATUS "Debug: CMAKE_OMP=${CMAKE_OMP}")
+
 configure_file (
     "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp11/kernel.json.in"
     "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp11/kernel.json"
diff --git a/notebooks/openmp-demo.ipynb b/notebooks/openmp-demo.ipynb
new file mode 100644
index 00000000..d5ee4b3f
--- /dev/null
+++ b/notebooks/openmp-demo.ipynb
@@ -0,0 +1,69 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0c15570-ee24-42ed-b61f-11a3fc858b2d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#include <iostream>\n",
+    "#include <omp.h>\n",
+    "#include <dlfcn.h>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b4dac8e-3ad2-46eb-b801-ba717e664b93",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "auto h = dlopen(\"libgomp.so\", RTLD_LAZY);\n",
+    "if (!h) printf(\"%s\\n\", dlerror());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0803d343-f636-4bc9-9968-8b49023c4440",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "//#pragma omp parallel num_threads(4)\n",
+    "//#pragma omp critical\n",
+    "std::cout << \"tid = \" << omp_get_thread_num() << std::endl;"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e25ba227-2995-4041-a2a9-258ac975c3e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#pragma omp parallel for\n",
+    "for(int i = 1; i < 100; ++i)\n",
+    "{\n",
+    "   std::cout << i << std::endl;\n",
+    "}"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "C++11 (with OpenMP)",
+   "language": "C++11",
+   "name": "xcpp11"
+  },
+  "language_info": {
+   "codemirror_mode": "text/x-c++src",
+   "file_extension": ".cpp",
+   "mimetype": "text/x-c++src",
+   "name": "c++",
+   "version": "11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
index ac69010b..fae919a1 100644
--- a/src/xinterpreter.cpp
+++ b/src/xinterpreter.cpp
@@ -38,9 +38,7 @@ using namespace std::placeholders;
 
 using Args = std::vector<const char *>;
 void* createInterpreter(const Args &ExtraArgs = {}) {
-  Args ClangArgs = {"-Xclang", "-diagnostic-log-file",
-                    "-Xclang", "-",
-                    "-xc++"};
+  Args ClangArgs = {"-xc++"};
   ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end());
   return Cpp::CreateInterpreter(ClangArgs);
 }
@@ -56,14 +54,90 @@ interpreter::interpreter(int argc, const char *const *argv)
       m_cout_buffer(std::bind(&interpreter::publish_stdout, this, _1)),
       m_cerr_buffer(std::bind(&interpreter::publish_stderr, this, _1)) {
   createInterpreter(Args(argv + 1, argv + argc - 2));
-  // Bootstrap the execution engine
   redirect_output();
+  // Bootstrap the execution engine
   init_preamble();
   init_magic();
 }
 
 interpreter::~interpreter() { restore_output(); }
 
+size_t GetFileSize(FILE* file) {
+    fseek(file, 0, SEEK_END);
+    return static_cast<size_t>(ftell(file));
+  }
+
+std::string ReadEntireFile(FILE* file) {
+    const size_t file_size = GetFileSize(file);
+    char* const buffer = new char[file_size];
+  
+    size_t bytes_last_read = 0;  // # of bytes read in the last fread()
+    size_t bytes_read = 0;       // # of bytes read so far
+  
+    fseek(file, 0, SEEK_SET);
+  
+    // Keeps reading the file until we cannot read further or the
+    // pre-determined file size is reached.
+    do {
+      bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
+      bytes_read += bytes_last_read;
+    } while (bytes_last_read > 0 && bytes_read < file_size);
+  
+    const std::string content(buffer, bytes_read);
+    delete[] buffer;
+  
+    return content;
+  }
+  
+// Object that captures an output stream (stdout/stderr).
+class CapturedStream {
+public:
+  // The ctor redirects the stream to a temporary file.
+  explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) {
+    char name_template[] = "/tmp/captured_stream.XXXXXX";
+    const int captured_fd = mkstemp(name_template);
+    if (captured_fd == -1) {
+      // GTEST_LOG_(WARNING)
+      // 	<< "Failed to create tmp file " << name_template
+      // 	<< " for test; does the test have access to the /tmp directory?";
+    }
+    filename_ = name_template;
+    fflush(nullptr);
+    dup2(captured_fd, fd_);
+    close(captured_fd);
+  }
+  
+  ~CapturedStream() {
+    remove(filename_.c_str());
+  }
+  
+  std::string GetCapturedString() {
+    if (uncaptured_fd_ != -1) {
+      // Restores the original stream.
+      fflush(nullptr);
+      dup2(uncaptured_fd_, fd_);
+      close(uncaptured_fd_);
+      uncaptured_fd_ = -1;
+    }
+  
+    FILE* const file = fopen(filename_.c_str(), "r");
+    if (file == nullptr) {
+      // GTEST_LOG_(FATAL) << "Failed to open tmp file " << filename_
+      // 			<< " for capturing stream.";
+    }
+    const std::string content = ReadEntireFile(file);
+    fclose(file);
+    return content;
+  }
+private:
+  const int fd_;  // A stream to capture.
+  int uncaptured_fd_;
+  // Name of the temporary file holding the stderr output.
+  ::std::string filename_;
+  
+};
+
+
 nl::json interpreter::execute_request_impl(int /*execution_counter*/,
                                            const std::string &code, bool silent,
                                            bool /*store_history*/,
@@ -91,24 +165,25 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
   // If silent is set to true, temporarily dismiss all std::cerr and
   // std::cout outputs resulting from `m_interpreter.process`.
 
-  auto cout_strbuf = std::cout.rdbuf();
-  auto cerr_strbuf = std::cerr.rdbuf();
+  // auto cout_strbuf = std::cout.rdbuf();
+  // auto cerr_strbuf = std::cerr.rdbuf();
 
-  if (silent) {
-    auto null = xnull();
-    std::cout.rdbuf(&null);
-    std::cerr.rdbuf(&null);
-  }
+  // if (silent) {
+  //   auto null = xnull();
+  //   std::cout.rdbuf(&null);
+  //   std::cerr.rdbuf(&null);
+  // }
 
   // Scope guard performing the temporary redirection of input requests.
   auto input_guard = input_redirection(allow_stdin);
 
   for (const auto &block : blocks) {
     // Attempt normal evaluation
-    std::string error_message;
-    std::stringstream error_stream(error_message);
+    std::string err;
     try {
+      CapturedStream s = CapturedStream(STDERR_FILENO);
       compilation_result = Cpp::Process(block.c_str());
+      err = s.GetCapturedString();
     }
 
     catch (std::exception &e) {
@@ -125,13 +200,9 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
       errorlevel = 1;
       // send the errors directly to std::cerr
       ename = "";
-      std::cerr << error_stream.str();
-    }
+      std::cerr << "TODO:ERROR HERE:" << err;
 
-    // If an error was encountered, don't attempt further execution
-    if (errorlevel) {
-      error_stream.str().clear();
-      //DiagnosticsOS.str().clear();
+      // If an error was encountered, don't attempt further execution
       break;
     }
   }
@@ -141,10 +212,10 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
   std::cerr << std::flush;
 
   // Reset non-silent output buffers
-  if (silent) {
-    std::cout.rdbuf(cout_strbuf);
-    std::cerr.rdbuf(cerr_strbuf);
-  }
+  // if (silent) {
+  //   std::cout.rdbuf(cout_strbuf);
+  //   std::cerr.rdbuf(cerr_strbuf);
+  // }
 
   // Depending of error level, publish execution result or execution
   // error, and compose execute_reply message.

From c5e60daabcef5a8586c560ad273d38ea3dac349a Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Tue, 22 Aug 2023 14:07:52 +0000
Subject: [PATCH 09/22] Fix openmp flags

---
 CMakeLists.txt | 6 +++++-
 Dockerfile     | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6552a127..5a28cfcd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,9 +25,13 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 set(CMAKE_OMP "")
 find_package(OpenMP)
 if (OpenMP_CXX_FOUND)
+    message(STATUS "OpenMP_CXX_VERSION=${OpenMP_CXX_VERSION}")
+    message(STATUS "OpenMP_CXX_SPEC_DATE=${OpenMP_CXX_SPEC_DATE}")
     message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
+    message(STATUS "OpenMP_CXX_INCLUDE_DIRS=${OpenMP_CXX_INCLUDE_DIRS}")
     message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
-    set(omp_libs OpenMP_CXX_LIB_NAMES)
+    message(STATUS "OpenMP_CXX_LIBRARIES=${OpenMP_CXX_LIBRARIES}")
+    set(omp_libs "${OpenMP_CXX_LIB_NAMES}")
     list(TRANSFORM omp_libs PREPEND "-l")
     list(JOIN omp_libs " " INCLUDE_STRING)
     message(STATUS "OpenMP_CXX_LIB_NAMES(params)=${omp_libs}")
diff --git a/Dockerfile b/Dockerfile
index a5ab74eb..3c330cc5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -33,7 +33,7 @@ RUN apt-get update --yes && \
     unzip \
     curl \
     jq \
-    libomp-dev \
+    ###libomp-dev \
     # Other "our" apt installs (development and testing)
     build-essential \
     git \

From 0f99d6e92b077c84be2dbb23b94bd0d599c5ce32 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Thu, 24 Aug 2023 09:36:25 +0000
Subject: [PATCH 10/22] Add openmp examples

---
 notebooks/openmp-demo.ipynb | 121 +++++++++++++++++++++++++++++++-----
 1 file changed, 107 insertions(+), 14 deletions(-)

diff --git a/notebooks/openmp-demo.ipynb b/notebooks/openmp-demo.ipynb
index d5ee4b3f..d108640c 100644
--- a/notebooks/openmp-demo.ipynb
+++ b/notebooks/openmp-demo.ipynb
@@ -8,8 +8,8 @@
    "outputs": [],
    "source": [
     "#include <iostream>\n",
-    "#include <omp.h>\n",
-    "#include <dlfcn.h>"
+    "#include \"/usr/lib/gcc/x86_64-linux-gnu/11/include/omp.h\"\n",
+    "#include </home/jovyan/CppInterOp/include/clang/Interpreter/CppInterOp.h>"
    ]
   },
   {
@@ -19,34 +19,127 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "auto h = dlopen(\"libgomp.so\", RTLD_LAZY);\n",
-    "if (!h) printf(\"%s\\n\", dlerror());"
+    "Cpp::LoadLibrary(\"/usr/lib/gcc/x86_64-linux-gnu/11/libgomp.so\");"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "0803d343-f636-4bc9-9968-8b49023c4440",
+   "id": "5001e441-1fa5-4bdc-9fa5-2ca103ae484f",
    "metadata": {},
    "outputs": [],
    "source": [
-    "//#pragma omp parallel num_threads(4)\n",
-    "//#pragma omp critical\n",
-    "std::cout << \"tid = \" << omp_get_thread_num() << std::endl;"
+    "void example1() {\n",
+    "  std::cout << \"Hello World!\" << std::endl;\n",
+    "}\n",
+    "example1();"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "e25ba227-2995-4041-a2a9-258ac975c3e4",
+   "id": "53fb7656-b72e-42bc-ade7-2ae2077142da",
    "metadata": {},
    "outputs": [],
    "source": [
-    "#pragma omp parallel for\n",
-    "for(int i = 1; i < 100; ++i)\n",
-    "{\n",
-    "   std::cout << i << std::endl;\n",
-    "}"
+    "void example2() {\n",
+    "    #pragma omp parallel num_threads(4)\n",
+    "    {\n",
+    "    std::cout << \"Hello World!\" << std::endl;\n",
+    "    }\n",
+    "}\n",
+    "example2();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "efcdfdb6-a60b-46af-8194-75ef9cc0e27f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "void example3() {\n",
+    "    #pragma omp parallel num_threads(4)\n",
+    "    {\n",
+    "    std::cout << \"Hello World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
+    "    }\n",
+    "}\n",
+    "example3();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d86a9efa-ba28-4cb6-bbfc-abc00ee63506",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "void example4() {\n",
+    "    #pragma omp parallel num_threads(4)\n",
+    "    {\n",
+    "    std::cout << \"Hello World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
+    "    }\n",
+    "\n",
+    "    std::cout << \"This is another message! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
+    "\n",
+    "    #pragma omp parallel num_threads(3)\n",
+    "    {\n",
+    "    std::cout << \"Goodbye World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
+    "    }\n",
+    "}\n",
+    "example4();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5557e01a-7c7d-4b54-8545-962ad11027df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "void example5() {\n",
+    "    double start_time = omp_get_wtime();\n",
+    "    double start_loop;\n",
+    "    \n",
+    "    const int N = 1000000000;\n",
+    "    int* a = new int[N];\n",
+    "    int* b = new int[N];\n",
+    "    \n",
+    "    start_loop = omp_get_wtime();\n",
+    "    #pragma\tomp parallel for\n",
+    "    for (int i=0; i<N; i++) {\n",
+    "        a[i] = 1.0;\n",
+    "    }\n",
+    "    std::cout << \"Initialize a[] time: \" << omp_get_wtime()-start_loop << std::endl;\n",
+    "\n",
+    "    start_loop = omp_get_wtime();\n",
+    "    #pragma\tomp parallel for\n",
+    "    for (int i=0; i<N; i++) {\n",
+    "        b[i] = 1.0 + double(i);\n",
+    "    }\n",
+    "    std::cout << \"Initialize b[] time: \" << omp_get_wtime()-start_loop << std::endl;\n",
+    "\n",
+    "    start_loop = omp_get_wtime();\n",
+    "    #pragma omp parallel for\n",
+    "    for (int i=0; i<N; i++) {\n",
+    "        a[i] = a[i] + b[i];\n",
+    "    }\n",
+    "    std::cout << \"Add arrays time: \" << omp_get_wtime()-start_loop << std::endl;\n",
+    "    \n",
+    "    start_loop = omp_get_wtime();\n",
+    "    double average = 0.0;\n",
+    "    #pragma omp parallel for reduction(+:average)\n",
+    "    for (int i=0; i<N; i++) {\n",
+    "        average += a[i];\n",
+    "    }\n",
+    "    average = average/double(N);\n",
+    "    std::cout << \"Average result time: \" << omp_get_wtime()-start_loop << std::endl;\n",
+    "    \n",
+    "    std::cout << \"Average: \" << average << std::endl;\n",
+    "\n",
+    "    std::cout << \"Total time: \" << omp_get_wtime()-start_time << std::endl;\n",
+    "}\n",
+    "example5();"
    ]
   }
  ],

From effd6b083f429107fa884b0f010d0882f0bc84b3 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Thu, 24 Aug 2023 19:50:46 +0000
Subject: [PATCH 11/22] Add omp targets to CI build

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 97c67822..2aa99145 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,7 +86,7 @@ jobs:
         mkdir inst
         cd build
         cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_INSTALL_PREFIX=../inst ../llvm
-        make ClangDriverOptions clang-repl -j$(nproc --all)
+        make ClangDriverOptions clang-repl omp openmp-resource-headers -j$(nproc --all)
         #make install -j$(nproc --all)
         cd ../..
     - name: Save Cache LLVM/Clang runtime build directory

From f9bfef652dde88d0ec44542c9453df8eb6e682fe Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Fri, 25 Aug 2023 07:54:03 +0000
Subject: [PATCH 12/22] Try install clang-dev

---
 .github/workflows/ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2aa99145..30249e32 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -85,9 +85,10 @@ jobs:
         mkdir build
         mkdir inst
         cd build
-        cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_INSTALL_PREFIX=../inst ../llvm
+        cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_RUNTIMES="openmp" -DCMAKE_INSTALL_PREFIX=../inst ../llvm
         make ClangDriverOptions clang-repl omp openmp-resource-headers -j$(nproc --all)
-        #make install -j$(nproc --all)
+        make install -j$(nproc --all)
+        make clean -j$(nproc --all)
         cd ../..
     - name: Save Cache LLVM/Clang runtime build directory
       uses: actions/cache/save@v3

From 847dbed7d02a28105b08287d5ea182e163875510 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Fri, 25 Aug 2023 21:13:30 +0000
Subject: [PATCH 13/22] Try install clang-dev

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 30249e32..a06b55e4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,7 +86,7 @@ jobs:
         mkdir inst
         cd build
         cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_OPTIMIZED_TABLEGEN=On -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_RUNTIMES="openmp" -DCMAKE_INSTALL_PREFIX=../inst ../llvm
-        make ClangDriverOptions clang-repl omp openmp-resource-headers -j$(nproc --all)
+        make ClangDriverOptions clang-repl openmp openmp-resource-headers -j$(nproc --all)
         make install -j$(nproc --all)
         make clean -j$(nproc --all)
         cd ../..

From 3cf159d25b7a1e93817aa12528be7062a1cf3488 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 04:36:47 +0000
Subject: [PATCH 14/22] Fix ci build to new clang-dev

---
 .github/workflows/ci.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a06b55e4..7c2ec525 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -98,7 +98,7 @@ jobs:
         key: ${{ steps.cache.outputs.cache-primary-key }}
     - name: Set PATH_TO_LLVM_BUILD etc to Env
       run: |
-        export PATH_TO_LLVM_BUILD="$(realpath clang-dev/build)"
+        export PATH_TO_LLVM_BUILD="$(realpath clang-dev/inst)"
         export PATH=$PATH_TO_LLVM_BUILD/bin:$PATH
         export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH
         echo "PATH_TO_LLVM_BUILD=$PATH_TO_LLVM_BUILD" >> $GITHUB_ENV
@@ -148,8 +148,9 @@ jobs:
       run: |
         # Build CppInterOp next to cling and llvm-project.
         LLVM_DIR="$(realpath clang-dev)"
-        LLVM_BUILD_DIR="$(realpath clang-dev/build)"
-        CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include"
+        LLVM_BUILD_DIR="$(realpath clang-dev/inst)"
+        #CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include"
+        CPLUS_INCLUDE_PATH="${LLVM_BUILD_DIR}/include/llvm:${LLVM_BUILD_DIR}/include/clang"
         git clone https://github.com/compiler-research/CppInterOp.git
         export CPPINTEROP_DIR=$PWD/cppyy-backend/python/cppyy_backend/
         cd CppInterOp

From 19233e7a1a2c88f1af9fe13d3514bc888b3ecffd Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 07:46:55 +0000
Subject: [PATCH 15/22] Properly extract the interpreter-related args

---
 src/main.cpp         | 18 +++++++-----------
 src/xinterpreter.cpp |  2 +-
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/src/main.cpp b/src/main.cpp
index 8ace7d7d..d3a72b87 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -50,20 +50,16 @@ std::string extract_filename(int& argc, char *argv[]) {
 
 using interpreter_ptr = std::unique_ptr<xcpp::interpreter>;
 interpreter_ptr build_interpreter(int argc, char **argv) {
-  int interpreter_argc = argc + 1;
-  const char **interpreter_argv = new const char *[interpreter_argc];
-  interpreter_argv[0] = "xeus-clang-repl";
-  // Copy all arguments in the new array excepting the process name.
+  std::vector<const char*> interpreter_args;
   for (int i = 1; i < argc; i++) {
-    interpreter_argv[i] = argv[i];
+    if (argv[i] == "-f") {
+      i++; // skip the value of -f which is a json file.
+      continue;
+    }
+    interpreter_args.push_back(argv[i]);
   }
-  // std::string resource_dir =
-  //     "-resource-dir=" + std::string(CLANG_RESOURCE_DIR);
-  // interpreter_argv[interpreter_argc - 1] = resource_dir.c_str();
-
   interpreter_ptr interp_ptr = interpreter_ptr(
-      new xcpp::interpreter(interpreter_argc, interpreter_argv));
-  delete[] interpreter_argv;
+       new xcpp::interpreter(interpreter_args.size(), interpreter_args.data()));
   return interp_ptr;
 }
 
diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
index fae919a1..3fa501cf 100644
--- a/src/xinterpreter.cpp
+++ b/src/xinterpreter.cpp
@@ -53,7 +53,7 @@ interpreter::interpreter(int argc, const char *const *argv)
       xmagics(), p_cout_strbuf(nullptr), p_cerr_strbuf(nullptr),
       m_cout_buffer(std::bind(&interpreter::publish_stdout, this, _1)),
       m_cerr_buffer(std::bind(&interpreter::publish_stderr, this, _1)) {
-  createInterpreter(Args(argv + 1, argv + argc - 2));
+  createInterpreter(Args(argv, argv + argc));
   redirect_output();
   // Bootstrap the execution engine
   init_preamble();

From 3e02301eb861d846a26e8f34088ae8ab5c8da736 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 08:08:31 +0000
Subject: [PATCH 16/22] Fix OpenMP params

---
 CMakeLists.txt | 34 +++++++++++++++++++---------------
 Dockerfile     |  7 ++++---
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5a28cfcd..64b945c5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,22 +22,26 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 # =============
 
 # OpenMP config
-set(CMAKE_OMP "")
-find_package(OpenMP)
+find_package(OpenMP PATHS "${LLVM_CMAKE_DIR}/lib/cmake/openmp")
 if (OpenMP_CXX_FOUND)
-    message(STATUS "OpenMP_CXX_VERSION=${OpenMP_CXX_VERSION}")
-    message(STATUS "OpenMP_CXX_SPEC_DATE=${OpenMP_CXX_SPEC_DATE}")
-    message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
-    message(STATUS "OpenMP_CXX_INCLUDE_DIRS=${OpenMP_CXX_INCLUDE_DIRS}")
-    message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
-    message(STATUS "OpenMP_CXX_LIBRARIES=${OpenMP_CXX_LIBRARIES}")
-    set(omp_libs "${OpenMP_CXX_LIB_NAMES}")
-    list(TRANSFORM omp_libs PREPEND "-l")
-    list(JOIN omp_libs " " INCLUDE_STRING)
-    message(STATUS "OpenMP_CXX_LIB_NAMES(params)=${omp_libs}")
-    message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")
-    message(STATUS "OpenMP_pthread_LIBRARY=${OpenMP_pthread_LIBRARY}")
-    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\", \"-l${OpenMP_gomp_LIBRARY}\", \"-l${OpenMP_pthread_LIBRARY}\"")
+    message(STATUS "OpenMP: Found.")
+#    message(STATUS "OpenMP_CXX_VERSION=${OpenMP_CXX_VERSION}")
+#    message(STATUS "OpenMP_CXX_SPEC_DATE=${OpenMP_CXX_SPEC_DATE}")
+#    message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
+#    message(STATUS "OpenMP_CXX_INCLUDE_DIRS=${OpenMP_CXX_INCLUDE_DIRS}")
+#    message(STATUS "OpenMP_CXX_LIB_NAMES=${OpenMP_CXX_LIB_NAMES}")
+#    message(STATUS "OpenMP_CXX_LIBRARIES=${OpenMP_CXX_LIBRARIES}")
+#    set(omp_libs "${OpenMP_CXX_LIB_NAMES}")
+#    list(TRANSFORM omp_libs PREPEND "-l")
+#    list(JOIN omp_libs " " INCLUDE_STRING)
+#    message(STATUS "OpenMP_CXX_LIB_NAMES(params)=${omp_libs}")
+#    message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")
+#    message(STATUS "OpenMP_pthread_LIBRARY=${OpenMP_pthread_LIBRARY}")
+#    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\", \"-l${OpenMP_gomp_LIBRARY}\", \"-l${OpenMP_pthread_LIBRARY}\"")
+    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\"")
+else()
+    message(STATUS "OpenMP: Not Found.")
+    set(CMAKE_OMP ",\n\"-fopenmp\"")
 endif()
 
 
diff --git a/Dockerfile b/Dockerfile
index 3c330cc5..675cbf87 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -168,7 +168,7 @@ RUN \
     popd && \
     #
     echo "Debug clang path: $PATH_TO_CLANG_DEV" && \
-    export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/build && \
+    export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/inst && \
     export VENV=/home/jovyan/.venv && \
     echo "export VENV=$VENV" >> ~/.profile && \
     export PATH=$VENV/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \
@@ -177,7 +177,8 @@ RUN \
     # Build CppInterOp
     #
     sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \
-    export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/../llvm/include:${PATH_TO_LLVM_BUILD}/../clang/include:${PATH_TO_LLVM_BUILD}/include:${PATH_TO_LLVM_BUILD}/tools/clang/include:${sys_incs%:}" && \
+    #export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/../llvm/include:${PATH_TO_LLVM_BUILD}/../clang/include:${PATH_TO_LLVM_BUILD}/include:${PATH_TO_LLVM_BUILD}/tools/clang/include:${sys_incs%:}"
+    export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:${sys_incs%:}" && \
     echo $CPLUS_INCLUDE_PATH && \
     git clone https://github.com/compiler-research/CppInterOp.git && \
     export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \
@@ -258,7 +259,7 @@ RUN \
     make && \
     make install && \
     # install clad in all exist kernels
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done && \
     #
     # Add OpenMP to all kernels
     #

From 16148089195836c284b9adcc8fd2f52fbaa9c16e Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 08:55:25 +0000
Subject: [PATCH 17/22] Add clad demo

---
 Dockerfile                                    |   2 +-
 notebooks/clad-demo.ipynb                     |  87 ++++++++++++++++++
 notebooks/openmp-demo.ipynb                   |  16 ++--
 .../kernels/clad-xcpp17/kernel.json.in        |  15 +++
 .../kernels/clad-xcpp17/logo-32x32.png        | Bin 0 -> 1520 bytes
 .../kernels/clad-xcpp17/logo-64x64.png        | Bin 0 -> 3113 bytes
 6 files changed, 111 insertions(+), 9 deletions(-)
 create mode 100644 notebooks/clad-demo.ipynb
 create mode 100644 share/jupyter/kernels/clad-xcpp17/kernel.json.in
 create mode 100644 share/jupyter/kernels/clad-xcpp17/logo-32x32.png
 create mode 100644 share/jupyter/kernels/clad-xcpp17/logo-64x64.png

diff --git a/Dockerfile b/Dockerfile
index 675cbf87..3d205a94 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -259,7 +259,7 @@ RUN \
     make && \
     make install && \
     # install clad in all exist kernels
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done && \
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/clad-xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done && \
     #
     # Add OpenMP to all kernels
     #
diff --git a/notebooks/clad-demo.ipynb b/notebooks/clad-demo.ipynb
new file mode 100644
index 00000000..7ef88c8b
--- /dev/null
+++ b/notebooks/clad-demo.ipynb
@@ -0,0 +1,87 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "74f52cc2-bfda-4fee-9cd9-94d683a8b7e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#include \"clad/Differentiator/Differentiator.h\"\n",
+    "#include <chrono>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "55453d37-e8af-4cb7-8f82-550e9cf88b3c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "// Rosenbrock function declaration\n",
+    "double rosenbrock_func(double x, double y) {\n",
+    "return (x - 1) * (x - 1) + 100 * (y - x * x) * (y - x * x);\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9098b6e4-87a7-497d-9a1b-dddeb813dc56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "double rosenbrock_forward(double x[], int size) {\n",
+    "    double sum = 0;\n",
+    "    auto rosenbrockX = clad::differentiate(rosenbrock_func, 0);\n",
+    "    auto rosenbrockY = clad::differentiate(rosenbrock_func, 1);\n",
+    "    for (int i = 0; i < size-1; i++) {\n",
+    "        double one = rosenbrockX.execute(x[i], x[i + 1]);\n",
+    "        double two = rosenbrockY.execute(x[i], x[i + 1]);\n",
+    "            sum = sum + one + two;\n",
+    "    }\n",
+    "    return sum;\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e771bbdc-3979-43df-9ed9-3429ee713b5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "const int size = 100000000;\n",
+    "double Xarray[size];\n",
+    "for(int i=0;i<size;i++)\n",
+    "  Xarray[i]=((double)rand()/RAND_MAX);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e74f85c2-a3bc-4562-a9ae-321eb306e8ff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "double forward_time = timeForwardMode();"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "C++17 (with clad)",
+   "language": "C++17",
+   "name": "clad-xcpp17"
+  },
+  "language_info": {
+   "codemirror_mode": "text/x-c++src",
+   "file_extension": ".cpp",
+   "mimetype": "text/x-c++src",
+   "name": "c++",
+   "version": "17"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
\ No newline at end of file
diff --git a/notebooks/openmp-demo.ipynb b/notebooks/openmp-demo.ipynb
index d108640c..a9ac4a2a 100644
--- a/notebooks/openmp-demo.ipynb
+++ b/notebooks/openmp-demo.ipynb
@@ -8,7 +8,7 @@
    "outputs": [],
    "source": [
     "#include <iostream>\n",
-    "#include \"/usr/lib/gcc/x86_64-linux-gnu/11/include/omp.h\"\n",
+    "#include \"omp.h\"\n",
     "#include </home/jovyan/CppInterOp/include/clang/Interpreter/CppInterOp.h>"
    ]
   },
@@ -19,7 +19,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "Cpp::LoadLibrary(\"/usr/lib/gcc/x86_64-linux-gnu/11/libgomp.so\");"
+    "Cpp::LoadLibrary(\"libomp\");"
    ]
   },
   {
@@ -43,7 +43,7 @@
    "outputs": [],
    "source": [
     "void example2() {\n",
-    "    #pragma omp parallel num_threads(4)\n",
+    "    #pragma omp parallel\n",
     "    {\n",
     "    std::cout << \"Hello World!\" << std::endl;\n",
     "    }\n",
@@ -59,7 +59,7 @@
    "outputs": [],
    "source": [
     "void example3() {\n",
-    "    #pragma omp parallel num_threads(4)\n",
+    "    #pragma omp parallel\n",
     "    {\n",
     "    std::cout << \"Hello World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
     "    }\n",
@@ -75,14 +75,14 @@
    "outputs": [],
    "source": [
     "void example4() {\n",
-    "    #pragma omp parallel num_threads(4)\n",
+    "    #pragma omp parallel\n",
     "    {\n",
     "    std::cout << \"Hello World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
     "    }\n",
     "\n",
     "    std::cout << \"This is another message! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
     "\n",
-    "    #pragma omp parallel num_threads(3)\n",
+    "    #pragma omp parallel num_threads(2)\n",
     "    {\n",
     "    std::cout << \"Goodbye World! (\" << omp_get_thread_num() << \")\" << std::endl;\n",
     "    }\n",
@@ -106,14 +106,14 @@
     "    int* b = new int[N];\n",
     "    \n",
     "    start_loop = omp_get_wtime();\n",
-    "    #pragma\tomp parallel for\n",
+    "    #pragma omp parallel for\n",
     "    for (int i=0; i<N; i++) {\n",
     "        a[i] = 1.0;\n",
     "    }\n",
     "    std::cout << \"Initialize a[] time: \" << omp_get_wtime()-start_loop << std::endl;\n",
     "\n",
     "    start_loop = omp_get_wtime();\n",
-    "    #pragma\tomp parallel for\n",
+    "    #pragma omp parallel for\n",
     "    for (int i=0; i<N; i++) {\n",
     "        b[i] = 1.0 + double(i);\n",
     "    }\n",
diff --git a/share/jupyter/kernels/clad-xcpp17/kernel.json.in b/share/jupyter/kernels/clad-xcpp17/kernel.json.in
new file mode 100644
index 00000000..a524a6d1
--- /dev/null
+++ b/share/jupyter/kernels/clad-xcpp17/kernel.json.in
@@ -0,0 +1,15 @@
+{
+  "display_name": "C++17",
+  "env": {
+      "CPLUS_INCLUDE_PATH":"@CMAKE_CPLUS_INCLUDE_PATH@",
+      "LD_LIBRARY_PATH":"@CMAKE_LD_LIBRARY_PATH@",
+      "PYTHONPATH":"@CMAKE_PYTHONPATH@"
+  },
+  "argv": [
+      "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/xcpp",
+      "-f",
+      "{connection_file}",
+      "-std=c++17"@CMAKE_OMP@
+  ],
+  "language": "C++17"
+}
diff --git a/share/jupyter/kernels/clad-xcpp17/logo-32x32.png b/share/jupyter/kernels/clad-xcpp17/logo-32x32.png
new file mode 100644
index 0000000000000000000000000000000000000000..c09c458544bb83b9e3ff4d38f5758bda10ceebe8
GIT binary patch
literal 1520
zcmV<M1rPd(P)<h;3K|Lk000e1NJLTq001BW001Be1^@s6b9#F800004b3#c}2nYxW
zd<bNS00009a7bBm0003=0003=0Yp65&j0`b8FWQhbW?9;ba!ELWdL_~cP?peYja~^
zaAhuUa%Y?FJQ@H11#L-0K~z|Ut(R+1lw}mhf6u%763Z$#l~4*&L9|8_oH9c))2YeC
zsWF{ux|k+4))>`fj+1!di&IS(aK;ELYw|L~OnvC0l61ks3o346fp`H$NMM0omj!m;
z_kH@H6w9)^K>k0T=Y7uqf1dN4=RHR-3_SCt%^Kb<)&MU7;Zlg5y2-oY^X2I+L-i$w
zx?#n~wk$=V07O#1r4C?Ac(>)lg4HQKH(Wq=p)Fks<s*=@L%6I3UN6d?`rQo?kX?9e
zw4tc$g%mFXrolM9jssxi6lG68JuCs)g~wDW)Kwrh1Cc`@?hCIJa+_v0Htcvltz*yv
zuH{*L08AVvaeuUd+}sgev$c5XQazx6<)7M8O}cm=SP;xxq{Ym{n<I#^s+ddyhg+kr
zrHf`qsJkamg2*Y#p7yJsfR&&8u?#2z%z?H~9yfxl^f;2^t%w26uA!})5?c#L&UPS!
z$q?jiUpX!JUjfTLzYwl^uU;WQu2qxZ>C6OXrNjh{uNT#gE{aMoakX3XTk{(FnV(Oe
z@2s~w9%t9`^9n$i$CVI>+-wrO`tbNclf$)>dTY@HMq2(mOewM3E;b(gqU`v@`quU|
z&?f^Z0J=#DSPf5QBrs{Lb<pI!h>N<3m*yuT1l|^h>)_7v6XNK(hD<51m$Gp!i2?Hs
zFyZD1GE)5`@AeoRtZC!y-<<#?jkYj5HI~Q-Uw67;ILoSQ?bVI#nSnTh3V0+f&aX}z
z+PcZ#S4T&u_g`COos*JNt-LrdktqoyXmNVH?|fTz?!E6TCllfg2`C^uOkv8n5k5AL
zX0Ua?FL^KAJv!TdtaGQ2w_aYgqb#wjN1GgsIHZ82Q5MW5-}n!o?V!!&PYxJuo$f<>
zZ8eh$O1vq(#>ehTqQ`6S!^ukkJbCX#nw?#gRyBqe5M@>UY!{l{eztCpX4`+NHoVz+
zHKVWnvAdF3yXa0lngKwhMP*IaRN7oU2;jh}#^3@%K}wpARsDJXWX;)VB4aevn}+2l
zpa91Jd7cm#!S6Ng?hBV1Dt@V`yj#=sYf*RD4b1BAM2(t+qKpCfrlKCeCj~+XzZ58e
z1k|^7q3OO4>OINP#*q^h&I5;z&CxX7(q~CQ^Jc{Kv|Ha^kXjq2Dm}d-ddUHHm!I<?
z4=SMBYw*`)myb;~E6P`E7RKK)#=o*+M_MS(T@sP^^z4*Bw>+vmF=K3XAm4o<09I^2
zovavoZU4pOcuU)x3zK8}|9GytJ#61w$!}E`0hlpqEXyC7PJFCy*_NHU=v<QjlfAi9
zAMeL3S;(Fjz7KQ<yp1ibhramwNLq~PK~^qE_gNen5yoqaXY$(OnV|xVY^`sIz+^s1
z)Tld9l-T|sm;p?>A?5WbU2gZGAIi!T98TAL0IwA6BY9jbbEn@Hip&RR8tgp2sTe~_
zGnzLO`;~SgtmRlEX96Y^DdGAZ@Pv)Htm)c`((;O<yT2=$<#4(ZdmVW+orl-$;<J4x
z2TlIt@$+Ql?4ZTb_uY^&xLx<s+H{egE*qGtp>IbtZ_{%Kv7I2LIcHiD8=sv+=F~(6
z`hlLPZQ{M+!|eH`3K_H`VKb$<8+-)-%*{*1Bty1janV+0-F^$Vj)^2pHPK{uakS<#
z=Nj9CZ3C!8QF17GyR6qRa33%~Z$8L;U`i;DgM=L+@`z5|N^y{;)CV&PRbrc0A?5#S
z%MolthUIc#tz)1MLQh8bZyb%PtVc5R!9*vApL2&dnE2s=jp-=jBM>1*GtMBz8<eiU
zF{7Cu9^90LF7grF)^DW)A+}(S`jEY=Z~SoTE8<-%de<5RF9P8pccH2q*t>4X<M}_8
W!Ww75y$Xi_0000<MNUMnLSTZoM9_Tz

literal 0
HcmV?d00001

diff --git a/share/jupyter/kernels/clad-xcpp17/logo-64x64.png b/share/jupyter/kernels/clad-xcpp17/logo-64x64.png
new file mode 100644
index 0000000000000000000000000000000000000000..396c24463156c7b2c5cb48a9e1b8919c4a808607
GIT binary patch
literal 3113
zcmV+^4A%3BP)<h;3K|Lk000e1NJLTq002M$002M;1^@s6s%dfF00004b3#c}2nYxW
zd<bNS00009a7bBm0007$0007$0l(mG*8l(j8FWQhbW?9;ba!ELWdL_~cP?peYja~^
zaAhuUa%Y?FJQ@H13#Lg#K~#90)mwX5RAn0feP;#+W`F@yV312_DydkcY>Aq=sq3n3
z^=Q_!)g~c!+q~quu9-OAo?0!V)>cdGqFI(tTeg~MXdzxOyl`DEa#8LB3@`(8pL4$b
z13|_am@{Vvvfp1b=X<~Rd(ZED=l$OIJKsz2D`4pd1@kDuE0oYlghEOvN&xIPd;9PD
zBq_!?n6{9?9PGKhd1FK@*#Tf~A0oAs&~@ocChP|xbTDih6arrScYcsnj5PpO0`TV1
zC6nQp<sZB-K5u|c8(0FAO3Ks@&wrXwvK>H(i`_YZ9-xF-ot_e>8VLJ<0VH7Q2Zb{z
z;nDz1a8p$~fH!@NzG?qjHp=7y(`?{z0+K!|h!k77tpIt(!>Y>wvTD!LxFeoa>M;VI
z*_S8tHDEO*WDNixPw91l41im)XX!mfo>1mK0hE%Z9~LYG@HQogbl)UiuuwvGS;gFj
zPm<!>-BIKg0Vy9ACIDO-fJArnxPexHZfXxK-FaxyB36h3K?FRv_j0({D&9Z|mH=Rc
z=<qCPE<jf7Svn!p)t)X9uyV&a-%dZjWdPOz@OO2vfd@T836`a&++D@D2X6wBcjrCK
zFyvhTF@kIxJeVn=yE#Vk`bWvJ9nOTE5|Esp8&4>cMkpTejD3TKdP=Y{qAo67sU%!q
zLVXhO^3LKRCW$$fQ1UVWk-OH&rDEI_?uR?WWC#nCAjr=PGVd<6Ix~w-qZRdSCR8^Y
zP*UHChIWs|JULzdA?}=`Mu!BXd{{6ap!)#?x-&8%z`ZeYJP;d<*l<4(7d%dFs}bj~
zw&8SfD@>L?A0S_F0JaaWk9+Ob*<feVjiv1j_$4iUVnUQZ7T&Kwm|WtH8gv@1IG*2v
zjDi-hoO=l(B}AE?61TDUH3tObX9KuT=pa&w2+vF#j)^gHp(-4rMs38$nKh_Y3)?O*
zvuyFn!g5n>d;7yUkW$-y-R9k1vL`QwfNU3Pq#{Uyzt10mk)bl7D*A+6Dn{byV5l@E
zG_>*e50pZ6@mg)(@%*xQt-+)KNHq$+yU^V(?k!9x87b7j=wZHCF((ExAGiHwpYRe9
zEd6~HzVxlfsUo#AabvYwlYOeFJi)@UQ+lp)xU;vA4FL>C#T?^$a2OUS!SXqG4Cu{w
zgAm|}#Bg+)So~7%I1SJlO*u#NO2agI<AV;GZHyE<5O4`TUJPEI9SuJpw^i{50kC*_
z1X{FaR5s~*uCpvxnORcRR?(zR;H%5+IR1&maD=+)>4iYd5F|Yq0WUEFP_z}*HJ<)B
z>tb|8lX{{{dfYK$ltPBY(LpZv(-~Qmt8{49Szxwuko$TeYN!vwgIwE280znhQ6Z9|
z*AAUk7|d2dk*Zq+EJz4*DVwTn(&M{JDwNdez)|}(Jyb5itO-LgbxaT#Cs9*LeWx}h
z{dD`8l7?|Enr#EY1dLY5Fd|gSOP1p(zRGUEPep3<7s{wKCVZAvhqI+^cz$LSf@NNI
zGR<1EW!rxXE8aSKB^E${FzatZun(9T8_Y|NqZGR{s&UH6&G$xClOAs#u7JA3+;h!n
z;pi7<t1F_GeZk$hd5=KQ3=oS5;_u)`{<k?Q<X+dhJVv|Dg7;5c$J$3m;$l^s>d!k*
ziLW(l1;;6UK@b5WLZpz0oll;|b`wq%v<Mjj08rVaEBf8~BjVia&0_>BaROlk@O<}=
z<u-#8kibxihKtu~D|4^cCxSZmvtkj0QDI%Pu(-}XAoB4NAtuBh7AuExRo^pE5CIAv
z>psg-<W;*o_4PIC_MDSN6_aejR|g0Q@WKDKF8}~APwmC6i^}-OAmnU$1dZ(mC|>%!
zPd$PN2z20`Z$+cV1igv(MlIDCjoHU?OQSjrrs@3&GK9d-M+`ron>P60_}yW&K*TVR
zdW#^Fbln;uK!9&oyVhW_(<g`kiI?;Dzg=fPm?KzOwm74(+)~r3naoFiV$4wdvgJ=U
z*ED<o0sx5o>p@J68H#UL&FpClmHYOzi=O((F8&y>0G>DrktnDtujFdk#iEiS)Gr9|
z9{g|`5J&)@Eh@RJPt}(C=9cq6W?hIkTP@Q702(zQ66HWXBpf2~@c#5(s%n8ec^|?8
zr6_%SVb@s6`v7#k0*x92<jMOGf5#B~ynY^38UseZdI$g<MxTNRXyY-x6@e1Sd_>S0
zZ~6h9-jMVC>2sm&+7A1;on;fy+N6iKBohI_lR${WT@{W}Xbl$l`P{Uj!)WPof>xG;
z*1$n$vH}1ob^PvkLl6OtJa%juLNFmJ0O!lvz;ayW>72{0<=1PSrKwVwP1B)nY=G21
z7gE1O^s#Z<q%~sf>aPL7ZUq2<!kRXWUHvsI*8XR!AOb3S<>38cUVI4q$??nQ&Roo&
z%uzam_W+bCpwlX#?<i74M~EAAtX&{vWjTlIe>aR4w%_({2+9Sjn)JN<KuCbMXv&>_
z(>O{65q)<|$heA~PmlTPr75bg0EyE*Fd+~`fR&}Va*bb*vn6Q;9-HEN?h-L90Eag{
z0bhw2k57#aE#2{W@YYAiRd|URkDd|{oDF7`w(=W*5FAQfh?NWO<NaY|^4JKRe>Vxy
zp*OeI1Edo2+M@eM7XR~6=E;eQ5+OD_v`=Wqs|`AotN8f>5ks)!h1tk{FBx+ZMnWud
zJnY!$VMu><KC<3fjEG^5v(1r;Ag?c0&5g}mJ4-uW;a4rx{%+VjljnEm&EXhwjAvgO
z6(lNLx8SZg;>tHK0IeMcoGYqDb(02qlLd-kKTI60Ky2htm-JYy9FpEST6Ex2qXG=W
zZ-atj7Y;6Ysr#=W=VY<4ML(X)rgf_3IE8FJlK&X*QvYCoU(COMl#roqP}$Ii|6~@9
zgNbc}RC*dRc_LiACzpgdN*gZ~7FXn!l=Y;kD|Vg0l<|=+KRtJ0vaon|+hG{G0-hk~
zbq_#qs6>Fg6%yY`eD(140fw4}#>{=+9QDsFE}POjNAEP4@kr`dXx8xa2c7_>z~b$P
zaiOH{mKJUVv~82ns>%Utso|r`h5(Ks)?3Hw=<Liocr-)(Lss^5i`D8dJT0qLW6s*o
z(J}}maFk-%`#<2|PyL*dW7!0>Hits1J`YNDx3>2FV?zKXw4(dBg|(Lakaa%);P)BR
zI&_^8&J4SJwE@#seu|3vHeNR(1O}58i#C6aoriPyh<n3inv9kP9dz1k;KrUTgYw=&
zcJ4cA_m!iCWfhW(`Na=XP?taU$z>AkUOo>`&TwnH-VT?qHem7g?@&<HBuoKeN)Qle
z2ID^$ho0?uckI`%Sz9K7a%bV@0f{*YBk}h0voJP#m^*6FtkGlJm*?>AHy4~HHcmmX
z0Y9d0vc2R8GdIoy$Y*f#hzvsr9-A7Al)3SkdG82_T=?>QrMwZJp2$P`u`AFS`Ry+C
z0}jA8Oy$-=sqFuVOb`B`3<k$+0EI^mNreQ;@KAgVro}}eHgX7}LgWbY_W>aUty%+G
zw0aa)HR00L2AsTd4P~_+%YAY{u^cB;&pT>#{zzsi?*V1f0Q9?Ac^jf0z*=Oi-w*U%
zeZqfE&ffSixZlJ<NdrLNgTZGj&aQF)prj2RS9+se^dczTjN9_4ERKL7%Wz`7=a0(T
zV3sl*l-U5>E{COY1q`zSCtg48YDf3#1s>Xz0EVW4l3(k%yc&dTM&Ov8I5ddI<!wO;
zW^GysO5Oqx<-R^%uz+9}sCOzd{-JS4k>_fLW^b2))2{{wp=5@|3jCCMo0QDlirHHt
z!O^V%^?cD3C9809{q0#aWrv40&H!Q30P23(93gK)EKkF+Ww(FXoDGyp24|S30SGLd
zBnQB-f5Qo-zzoPa2B&s--Wx$Mnb!b>Z|UsM5Mc$5r~bA}XYDX!^B9O|-^J4ewV+t%
zEPVAO2A6<tbY+{b0GI?o3Xoy|pTWxR!q3V<Eua5C%uRiK?&r{e00000NkvXXu0mjf
D0zkP<

literal 0
HcmV?d00001


From 310b2f4c284e358710474d11fc4b52d3c34a946b Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 09:22:10 +0000
Subject: [PATCH 18/22] Configure clad kernel

---
 CMakeLists.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 64b945c5..26fd578e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,6 +73,11 @@ configure_file (
     "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp17/kernel.json"
 )
 
+configure_file (
+    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/clad-xcpp17/kernel.json.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/clad-xcpp17/kernel.json"
+)
+
 ################
 # Dependencies #
 ################

From 5fe96213c7627e8335a34e5adb81827950476095 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 09:54:47 +0000
Subject: [PATCH 19/22] Fix clad and openmp params

---
 CMakeLists.txt | 18 +++++++++---------
 Dockerfile     |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 26fd578e..32d3c518 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,9 +22,9 @@ message(STATUS "Building xeus-clang-repl v${${PROJECT_NAME}_VERSION}")
 # =============
 
 # OpenMP config
-find_package(OpenMP PATHS "${LLVM_CMAKE_DIR}/lib/cmake/openmp")
-if (OpenMP_CXX_FOUND)
-    message(STATUS "OpenMP: Found.")
+#find_package(OpenMP PATHS "${LLVM_CMAKE_DIR}/lib/cmake/openmp")
+#if (OpenMP_CXX_FOUND)
+#    message(STATUS "OpenMP: Found.")
 #    message(STATUS "OpenMP_CXX_VERSION=${OpenMP_CXX_VERSION}")
 #    message(STATUS "OpenMP_CXX_SPEC_DATE=${OpenMP_CXX_SPEC_DATE}")
 #    message(STATUS "OpenMP_CXX_FLAGS=${OpenMP_CXX_FLAGS}")
@@ -38,11 +38,11 @@ if (OpenMP_CXX_FOUND)
 #    message(STATUS "OpenMP_gomp_LIBRARY=${OpenMP_gomp_LIBRARY}")
 #    message(STATUS "OpenMP_pthread_LIBRARY=${OpenMP_pthread_LIBRARY}")
 #    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\", \"-l${OpenMP_gomp_LIBRARY}\", \"-l${OpenMP_pthread_LIBRARY}\"")
-    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\"")
-else()
-    message(STATUS "OpenMP: Not Found.")
-    set(CMAKE_OMP ",\n\"-fopenmp\"")
-endif()
+#    set(CMAKE_OMP ",\n\"${OpenMP_CXX_FLAGS}\"")
+#else()
+#    message(STATUS "OpenMP: Not Found.")
+#    set(CMAKE_OMP ",\n\"-fopenmp\"")
+#endif()
 
 
 include(GNUInstallDirs)
@@ -56,7 +56,7 @@ message(STATUS "Debug: Replace in kernels")
 message(STATUS "Debug: CMAKE_CPLUS_INCLUDE_PATH=${CMAKE_CPLUS_INCLUDE_PATH}")
 message(STATUS "Debug: CMAKE_LD_LIBRARY_PATH=${CMAKE_LD_LIBRARY_PATH}")
 message(STATUS "Debug: CMAKE_PYTHONPATH=${CMAKE_PYTHONPATH}")
-message(STATUS "Debug: CMAKE_OMP=${CMAKE_OMP}")
+#message(STATUS "Debug: CMAKE_OMP=${CMAKE_OMP}")
 
 configure_file (
     "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp11/kernel.json.in"
diff --git a/Dockerfile b/Dockerfile
index 3d205a94..31357ff3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -263,4 +263,4 @@ RUN \
     #
     # Add OpenMP to all kernels
     #
-    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += [] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done
+    for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/xcpp.* ]]; then jq '.argv += ["-fopenmp"] | .display_name += " (with OpenMP)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done

From 2b5d3f5fe0de56790d8383b2a78c167ca7b4e120 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 13:46:51 +0000
Subject: [PATCH 20/22] Use the new stream redirection in CppInterOp.

---
 src/xinterpreter.cpp | 103 ++++++-------------------------------------
 1 file changed, 14 insertions(+), 89 deletions(-)

diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
index 3fa501cf..bcd7672d 100644
--- a/src/xinterpreter.cpp
+++ b/src/xinterpreter.cpp
@@ -62,82 +62,6 @@ interpreter::interpreter(int argc, const char *const *argv)
 
 interpreter::~interpreter() { restore_output(); }
 
-size_t GetFileSize(FILE* file) {
-    fseek(file, 0, SEEK_END);
-    return static_cast<size_t>(ftell(file));
-  }
-
-std::string ReadEntireFile(FILE* file) {
-    const size_t file_size = GetFileSize(file);
-    char* const buffer = new char[file_size];
-  
-    size_t bytes_last_read = 0;  // # of bytes read in the last fread()
-    size_t bytes_read = 0;       // # of bytes read so far
-  
-    fseek(file, 0, SEEK_SET);
-  
-    // Keeps reading the file until we cannot read further or the
-    // pre-determined file size is reached.
-    do {
-      bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
-      bytes_read += bytes_last_read;
-    } while (bytes_last_read > 0 && bytes_read < file_size);
-  
-    const std::string content(buffer, bytes_read);
-    delete[] buffer;
-  
-    return content;
-  }
-  
-// Object that captures an output stream (stdout/stderr).
-class CapturedStream {
-public:
-  // The ctor redirects the stream to a temporary file.
-  explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) {
-    char name_template[] = "/tmp/captured_stream.XXXXXX";
-    const int captured_fd = mkstemp(name_template);
-    if (captured_fd == -1) {
-      // GTEST_LOG_(WARNING)
-      // 	<< "Failed to create tmp file " << name_template
-      // 	<< " for test; does the test have access to the /tmp directory?";
-    }
-    filename_ = name_template;
-    fflush(nullptr);
-    dup2(captured_fd, fd_);
-    close(captured_fd);
-  }
-  
-  ~CapturedStream() {
-    remove(filename_.c_str());
-  }
-  
-  std::string GetCapturedString() {
-    if (uncaptured_fd_ != -1) {
-      // Restores the original stream.
-      fflush(nullptr);
-      dup2(uncaptured_fd_, fd_);
-      close(uncaptured_fd_);
-      uncaptured_fd_ = -1;
-    }
-  
-    FILE* const file = fopen(filename_.c_str(), "r");
-    if (file == nullptr) {
-      // GTEST_LOG_(FATAL) << "Failed to open tmp file " << filename_
-      // 			<< " for capturing stream.";
-    }
-    const std::string content = ReadEntireFile(file);
-    fclose(file);
-    return content;
-  }
-private:
-  const int fd_;  // A stream to capture.
-  int uncaptured_fd_;
-  // Name of the temporary file holding the stderr output.
-  ::std::string filename_;
-  
-};
-
-
 nl::json interpreter::execute_request_impl(int /*execution_counter*/,
                                            const std::string &code, bool silent,
                                            bool /*store_history*/,
@@ -165,14 +89,14 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
   // If silent is set to true, temporarily dismiss all std::cerr and
   // std::cout outputs resulting from `m_interpreter.process`.
 
-  // auto cout_strbuf = std::cout.rdbuf();
-  // auto cerr_strbuf = std::cerr.rdbuf();
+  auto cout_strbuf = std::cout.rdbuf();
+  auto cerr_strbuf = std::cerr.rdbuf();
 
-  // if (silent) {
-  //   auto null = xnull();
-  //   std::cout.rdbuf(&null);
-  //   std::cerr.rdbuf(&null);
-  // }
+  if (silent) {
+    auto null = xnull();
+    std::cout.rdbuf(&null);
+    std::cerr.rdbuf(&null);
+  }
 
   // Scope guard performing the temporary redirection of input requests.
   auto input_guard = input_redirection(allow_stdin);
@@ -182,8 +106,9 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
     std::string err;
     try {
       CapturedStream s = CapturedStream(STDERR_FILENO);
+      Cpp::BeginStdStreamCapture(Cpp::kStdErr);
       compilation_result = Cpp::Process(block.c_str());
-      err = s.GetCapturedString();
+      err = Cpp.EndStdStreamCapture();
     }
 
     catch (std::exception &e) {
@@ -200,7 +125,7 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
       errorlevel = 1;
       // send the errors directly to std::cerr
       ename = "";
-      std::cerr << "TODO:ERROR HERE:" << err;
+      std::cerr << err;
 
       // If an error was encountered, don't attempt further execution
       break;
@@ -212,10 +137,10 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
   std::cerr << std::flush;
 
   // Reset non-silent output buffers
-  // if (silent) {
-  //   std::cout.rdbuf(cout_strbuf);
-  //   std::cerr.rdbuf(cerr_strbuf);
-  // }
+  if (silent) {
+    std::cout.rdbuf(cout_strbuf);
+    std::cerr.rdbuf(cerr_strbuf);
+  }
 
   // Depending of error level, publish execution result or execution
   // error, and compose execute_reply message.

From 26c52643fd4176e90163b6a47a84438ded7ec806 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sat, 26 Aug 2023 18:23:37 +0000
Subject: [PATCH 21/22] Clean clad-dev after build

---
 .github/workflows/ci.yml  | 4 +++-
 notebooks/clad-demo.ipynb | 2 +-
 src/xinterpreter.cpp      | 3 +--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7c2ec525..46c5e1f6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -89,7 +89,9 @@ jobs:
         make ClangDriverOptions clang-repl openmp openmp-resource-headers -j$(nproc --all)
         make install -j$(nproc --all)
         make clean -j$(nproc --all)
-        cd ../..
+        cd ..
+        rm -rf $(find . -maxdepth 1 ! -name "inst" ! -name ".")
+        cd ..
     - name: Save Cache LLVM/Clang runtime build directory
       uses: actions/cache/save@v3
       if: ${{ steps.cache.outputs.cache-hit != 'true' }}
diff --git a/notebooks/clad-demo.ipynb b/notebooks/clad-demo.ipynb
index 7ef88c8b..09af6ee0 100644
--- a/notebooks/clad-demo.ipynb
+++ b/notebooks/clad-demo.ipynb
@@ -64,7 +64,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "double forward_time = timeForwardMode();"
+    "double r = rosenbrock_forward(Xarray, size);"
    ]
   }
  ],
diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
index bcd7672d..e742972a 100644
--- a/src/xinterpreter.cpp
+++ b/src/xinterpreter.cpp
@@ -105,10 +105,9 @@ nl::json interpreter::execute_request_impl(int /*execution_counter*/,
     // Attempt normal evaluation
     std::string err;
     try {
-      CapturedStream s = CapturedStream(STDERR_FILENO);
       Cpp::BeginStdStreamCapture(Cpp::kStdErr);
       compilation_result = Cpp::Process(block.c_str());
-      err = Cpp.EndStdStreamCapture();
+      err = Cpp::EndStdStreamCapture();
     }
 
     catch (std::exception &e) {

From f3e380cf204206e8f13ff5421ce503016f5e5109 Mon Sep 17 00:00:00 2001
From: Alexander Penev <alexander.p.penev@gmail.com>
Date: Sun, 27 Aug 2023 05:28:15 +0000
Subject: [PATCH 22/22] Fix install images in kernels spec

---
 CMakeLists.txt | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 32d3c518..35a61ae6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -58,25 +58,25 @@ message(STATUS "Debug: CMAKE_LD_LIBRARY_PATH=${CMAKE_LD_LIBRARY_PATH}")
 message(STATUS "Debug: CMAKE_PYTHONPATH=${CMAKE_PYTHONPATH}")
 #message(STATUS "Debug: CMAKE_OMP=${CMAKE_OMP}")
 
-configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp11/kernel.json.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp11/kernel.json"
-)
-
-configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp14/kernel.json.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp14/kernel.json"
-)
-
-configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp17/kernel.json.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/xcpp17/kernel.json"
-)
-
-configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/clad-xcpp17/kernel.json.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/clad-xcpp17/kernel.json"
-)
+function(configure_kernel kernel)
+    configure_file (
+    "${CMAKE_CURRENT_SOURCE_DIR}/${kernel}/kernel.json.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${kernel}/kernel.json"
+    )
+    file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/${kernel}/*.png")
+    foreach(file ${files})
+        configure_file(
+            "${file}"
+            "${CMAKE_CURRENT_BINARY_DIR}/${kernel}/"
+            COPYONLY
+       )
+    endforeach()
+endfunction()
+
+configure_kernel("share/jupyter/kernels/xcpp11")
+configure_kernel("share/jupyter/kernels/xcpp14")
+configure_kernel("share/jupyter/kernels/xcpp17")
+configure_kernel("share/jupyter/kernels/clad-xcpp17")
 
 ################
 # Dependencies #
@@ -305,12 +305,18 @@ set(XCPP_KERNELSPEC_DIR ${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels)
 install(DIRECTORY ${XCPP_KERNELSPEC_DIR}
         DESTINATION ${XJUPYTER_DATA_DIR}
         PATTERN "*.in" EXCLUDE)
+install(DIRECTORY ${XCPP_KERNELSPEC_DIR}
+        DESTINATION ${XJUPYTER_DATA_DIR}
+        FILES_MATCHING PATTERN "*.png")
 
 # Extra path for installing Jupyter kernelspec
 if (XEXTRA_JUPYTER_DATA_DIR)
     install(DIRECTORY ${XCPP_KERNELSPEC_DIR}
             DESTINATION ${XEXTRA_JUPYTER_DATA_DIR}
             PATTERN "*.in" EXCLUDE)
+    install(DIRECTORY ${XCPP_KERNELSPEC_DIR}
+            DESTINATION ${XEXTRA_JUPYTER_DATA_DIR}
+            FILES_MATCHING PATTERN "*.png")
 endif(XEXTRA_JUPYTER_DATA_DIR)
 
 # Makes the project importable from the build directory