From 920873eec30e971cb76233715a6577565a8eb845 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:42:56 +0800 Subject: [PATCH 01/10] Use console scripts for cross-platform compatibility --- interpreter/cling/tools/Jupyter/kernel/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/interpreter/cling/tools/Jupyter/kernel/setup.py b/interpreter/cling/tools/Jupyter/kernel/setup.py index 503d5f61186b5..8e0d6df99ca14 100644 --- a/interpreter/cling/tools/Jupyter/kernel/setup.py +++ b/interpreter/cling/tools/Jupyter/kernel/setup.py @@ -31,7 +31,6 @@ #----------------------------------------------------------------------------- import os -from glob import glob from distutils.core import setup @@ -44,7 +43,11 @@ name = name, version = '0.0.2', py_modules = ['clingkernel'], - scripts = glob(pjoin('scripts', '*')), + entry_points = { + 'console_scripts': [ + 'jupyter-cling-kernel=clingkernel:main' + ], + }, description = "C++ Kernel for Jupyter with Cling", author = 'Min RK, Axel Naumann', author_email = 'cling-dev@cern.ch', From 1780b609fc7e3bad49c21147c17010a3505273d7 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:44:17 +0800 Subject: [PATCH 02/10] Set the pipe to non-blocking with a cross-platform API --- interpreter/cling/tools/Jupyter/kernel/clingkernel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py index 17b4d24f23d86..6aed6a9fee3ad 100644 --- a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py +++ b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py @@ -18,8 +18,6 @@ __version__ = '0.0.3' import ctypes -from contextlib import contextmanager -from fcntl import fcntl, F_GETFL, F_SETFL import os import shutil import select @@ -59,8 +57,9 @@ def __init__(self, name): os.dup2(pipe_in, self.real_fd) os.close(pipe_in) # make pipe_out non-blocking - flags = fcntl(self.pipe_out, F_GETFL) - fcntl(self.pipe_out, F_SETFL, flags|os.O_NONBLOCK) + # flags = fcntl(self.pipe_out, F_GETFL) + # fcntl(self.pipe_out, F_SETFL, flags|os.O_NONBLOCK) + os.set_blocking(self.pipe_out, False) def restore(self): os.close(self.real_fd) From 69de814a60e0981e421ab4b49eb504139d7a75bd Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:44:59 +0800 Subject: [PATCH 03/10] Load the default C library on Windows explicitly --- interpreter/cling/tools/Jupyter/kernel/clingkernel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py index 6aed6a9fee3ad..4b4e0b3309b66 100644 --- a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py +++ b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py @@ -37,7 +37,10 @@ class my_void_p(ctypes.c_void_p): pass -libc = ctypes.CDLL(None) +if sys.platform == 'win32': + libc = ctypes.cdll.msvcrt +else: + libc = ctypes.CDLL(None) try: c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout') c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr') From 111721328986c0550a1b2fb4bf7b809de9b8dcfe Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:46:33 +0800 Subject: [PATCH 04/10] Get pointers to standard devices in a Windows-specific way --- .../cling/tools/Jupyter/kernel/clingkernel.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py index 4b4e0b3309b66..772a2e2aa221c 100644 --- a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py +++ b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py @@ -39,15 +39,30 @@ class my_void_p(ctypes.c_void_p): if sys.platform == 'win32': libc = ctypes.cdll.msvcrt + + class FILE(ctypes.Structure): + pass + + FILE_p = ctypes.POINTER(FILE) + + libc._fdopen.argtypes = [ctypes.c_int, ctypes.c_char_p] + libc._fdopen.restype = FILE_p + + c_stdout_p = libc._fdopen(sys.stdout.fileno(), b"w") + c_stderr_p = libc._fdopen(sys.stderr.fileno(), b"w") + + libc.fflush.argtypes = [FILE_p] + libc.fflush.restype = ctypes.c_int else: libc = ctypes.CDLL(None) -try: - c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout') - c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr') -except ValueError: - # libc.stdout is has a funny name on OS X - c_stdout_p = ctypes.c_void_p.in_dll(libc, '__stdoutp') - c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp') + + try: + c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout') + c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr') + except ValueError: + # libc.stdout is has a funny name on OS X + c_stdout_p = ctypes.c_void_p.in_dll(libc, '__stdoutp') + c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp') class FdReplacer: From dfdfc8f53ed079b001e8c88f6740ea8d31a82bf4 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:47:51 +0800 Subject: [PATCH 05/10] Export all symbols --- interpreter/cling/tools/Jupyter/CMakeLists.txt | 4 ++++ interpreter/cling/tools/Jupyter/kernel/clingkernel.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/interpreter/cling/tools/Jupyter/CMakeLists.txt b/interpreter/cling/tools/Jupyter/CMakeLists.txt index f02612d2f9287..c2fc7b3b2b0dc 100644 --- a/interpreter/cling/tools/Jupyter/CMakeLists.txt +++ b/interpreter/cling/tools/Jupyter/CMakeLists.txt @@ -81,6 +81,10 @@ add_cling_library(libclingJupyter ${ENABLE_SHARED} ${ENABLE_STATIC} set_target_properties(libclingJupyter PROPERTIES ENABLE_EXPORTS 1) +if(MSVC) + set_target_properties(libclingJupyter PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) +endif() + if(ENABLE_SHARED) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LIBCLINGJUPYTER_LINK_FLAGS " -Wl,-compatibility_version -Wl,1") diff --git a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py index 772a2e2aa221c..8cbb52fe4ef02 100644 --- a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py +++ b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py @@ -137,7 +137,7 @@ def __init__(self, **kwargs): else: raise RuntimeError('cling at ' + clingInPath + ' is unusable. No cling, no fun.') - for libFolder in ["/lib/libclingJupyter.", "/libexec/lib/libclingJupyter."]: + for libFolder in ["/bin/libclingJupyter.", "/lib/libclingJupyter.", "/libexec/lib/libclingJupyter."]: for ext in ['so', 'dylib', 'dll']: libFilename = clingInstDir + libFolder + ext From 54ad39454d8ebc8dfb234e2747e8b65bc4c11d44 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 02:49:31 +0800 Subject: [PATCH 06/10] Peek at the pipes at every flush interval --- .../cling/tools/Jupyter/kernel/clingkernel.py | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py index 8cbb52fe4ef02..c1ae17e633cda 100644 --- a/interpreter/cling/tools/Jupyter/kernel/clingkernel.py +++ b/interpreter/cling/tools/Jupyter/kernel/clingkernel.py @@ -24,6 +24,7 @@ import struct import sys import threading +import time from traitlets import Unicode, Float, Dict, List, CaselessStrEnum from ipykernel.kernelbase import Kernel @@ -38,6 +39,7 @@ class my_void_p(ctypes.c_void_p): pass if sys.platform == 'win32': + import msvcrt libc = ctypes.cdll.msvcrt class FILE(ctypes.Structure): @@ -51,6 +53,17 @@ class FILE(ctypes.Structure): c_stdout_p = libc._fdopen(sys.stdout.fileno(), b"w") c_stderr_p = libc._fdopen(sys.stderr.fileno(), b"w") + peek_named_pipe = ctypes.windll.kernel32.PeekNamedPipe + peek_named_pipe.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.c_void_p, + ctypes.wintypes.DWORD, + ctypes.POINTER(ctypes.wintypes.DWORD), + ctypes.POINTER(ctypes.wintypes.DWORD), + ctypes.POINTER(ctypes.wintypes.DWORD), + ] + peek_named_pipe.restype = ctypes.c_bool + libc.fflush.argtypes = [FILE_p] libc.fflush.restype = ctypes.c_int else: @@ -242,6 +255,44 @@ def forward_streams(self): def handle_input(self): """Capture stdout, stderr and sideband. Forward them as stream messages.""" + if sys.platform == 'win32': + pipes = {"sideband": self.sideband_pipe} + for rs in self.replaced_streams: + if rs: + pipes[rs.name] = rs.pipe_out + + # wait for the flush interval before peeking at the pipe + time.sleep(self.flush_interval) + + pipe_bytes = {} + total_bytes = 0 + for name, pipe in pipes.items(): + bytes_available = ctypes.wintypes.DWORD(0) + peek_named_pipe( + ctypes.wintypes.HANDLE(msvcrt.get_osfhandle(pipe)), + None, + 0, + None, + ctypes.byref(bytes_available), + None, + ) + pipe_bytes[name] = bytes_available.value + total_bytes += bytes_available.value + + if total_bytes == 0: + libc.fflush(c_stdout_p) + libc.fflush(c_stderr_p) + return False + + for name, n_bytes in pipe_bytes.items(): + if n_bytes == 0: + continue + + if name == "sideband": + self._process_sideband_data() + else: + self._process_stdio_data(pipes[name], name) + return True # create pipe for stdout, stderr select_on = [self.sideband_pipe] for rs in self.replaced_streams: @@ -304,7 +355,8 @@ def do_execute(self, code, silent, store_history=True, run_cell_thread.join() # Any leftovers? - while self.handle_input(): True + while self.handle_input(): + pass self.close_forwards() status = 'ok' From 0801845074ed58b2482806aaaef32b3f858d61af Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 20:51:54 +0800 Subject: [PATCH 07/10] Export symbols and use the standard exception handling model for MSVC --- interpreter/cling/tools/Jupyter/CMakeLists.txt | 6 +++++- interpreter/cling/tools/libcling/CMakeLists.txt | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/interpreter/cling/tools/Jupyter/CMakeLists.txt b/interpreter/cling/tools/Jupyter/CMakeLists.txt index c2fc7b3b2b0dc..3acbea40f7bad 100644 --- a/interpreter/cling/tools/Jupyter/CMakeLists.txt +++ b/interpreter/cling/tools/Jupyter/CMakeLists.txt @@ -12,7 +12,11 @@ set(LLVM_NO_DEAD_STRIP 1) set(SOURCES Kernel.cpp ) -set_source_files_properties(Kernel.cpp COMPILE_FLAGS "-fexceptions -frtti") +if(MSVC) + set_source_files_properties(Kernel.cpp COMPILE_FLAGS "/EHsc /GR") +else() + set_source_files_properties(Kernel.cpp COMPILE_FLAGS "-fexceptions -frtti") +endif() #Solve unresolved symbols bug in unix #See https://github.com/vgvassilev/cling/issues/114 if(WIN32) diff --git a/interpreter/cling/tools/libcling/CMakeLists.txt b/interpreter/cling/tools/libcling/CMakeLists.txt index 7b52c0d0d8253..0e251ca7f0f01 100644 --- a/interpreter/cling/tools/libcling/CMakeLists.txt +++ b/interpreter/cling/tools/libcling/CMakeLists.txt @@ -94,6 +94,10 @@ add_cling_library(libcling ${ENABLE_SHARED} ${ENABLE_STATIC} set_target_properties(libcling PROPERTIES ENABLE_EXPORTS 1) +if(MSVC) + set_target_properties(libcling PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) +endif() + if(ENABLE_SHARED) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LIBCLING_LINK_FLAGS " -Wl,-compatibility_version -Wl,1") From edb3dee354fc1a50d6079ae7b7bb93661e153885 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 22:34:37 +0800 Subject: [PATCH 08/10] Align the exported symbols with the driver --- .../cling/tools/Jupyter/CMakeLists.txt | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/interpreter/cling/tools/Jupyter/CMakeLists.txt b/interpreter/cling/tools/Jupyter/CMakeLists.txt index 3acbea40f7bad..ccffbe09de7ef 100644 --- a/interpreter/cling/tools/Jupyter/CMakeLists.txt +++ b/interpreter/cling/tools/Jupyter/CMakeLists.txt @@ -87,7 +87,66 @@ set_target_properties(libclingJupyter if(MSVC) set_target_properties(libclingJupyter PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) -endif() + + # RTTI/C++ symbols + set(cling_exports ${cling_exports} ??_7type_info@@6B@ + ?__type_info_root_node@@3U__type_info_node@@A + ?nothrow@std@@3Unothrow_t@1@B + ) + + # Compiler added symbols for static variables. NOT for VStudio < 2015 + set(cling_exports ${cling_exports} _Init_thread_abort _Init_thread_epoch + _Init_thread_footer _Init_thread_header _tls_index + ) + + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + # new/delete variants needed when linking to static msvc runtime (esp. Debug) + set(cling_exports ${cling_exports} + ??2@YAPEAX_K@Z + ??3@YAXPEAX@Z + ??_U@YAPEAX_K@Z + ??_V@YAXPEAX@Z + ??3@YAXPEAX_K@Z + ??2@YAPEAX_KAEBUnothrow_t@std@@@Z + ??_U@YAPEAX_KAEBUnothrow_t@std@@@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@H@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@M@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@N@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@PEBX@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@P6AAEAV01@AEAV01@@Z@Z + ??$?6U?$char_traits@D@std@@@std@@YAAEAV?$basic_ostream@DU?$char_traits@D@std@@@0@AEAV10@D@Z + ??$?6U?$char_traits@D@std@@@std@@YAAEAV?$basic_ostream@DU?$char_traits@D@std@@@0@AEAV10@PEBD@Z + ?_Facet_Register@std@@YAXPEAV_Facet_base@1@@Z + ) + else() + set(cling_exports ${cling_exports} + ??2@YAPAXI@Z + ??3@YAXPAX@Z + ??3@YAXPAXI@Z + ??_U@YAPAXI@Z + ??_V@YAXPAX@Z + ??_V@YAXPAXI@Z + ??2@YAPAXIABUnothrow_t@std@@@Z + ??_U@YAPAXIABUnothrow_t@std@@@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QAEAAV01@H@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QAEAAV01@M@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QAEAAV01@N@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QAEAAV01@PBX@Z + ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QAEAAV01@P6AAAV01@AAV01@@Z@Z + ??$?6U?$char_traits@D@std@@@std@@YAAAV?$basic_ostream@DU?$char_traits@D@std@@@0@AAV10@D@Z + ??$?6U?$char_traits@D@std@@@std@@YAAAV?$basic_ostream@DU?$char_traits@D@std@@@0@AAV10@PBD@Z + ?_Facet_Register@std@@YAXPAV_Facet_base@1@@Z + ) + endif() + + # List to '/EXPORT:sym0 /EXPORT:sym1 /EXPORT:sym2 ...' + foreach(sym ${cling_exports}) + set(cling_link_str "${cling_link_str} /EXPORT:${sym}") + endforeach(sym ${cling_exports}) + + set_property(TARGET libclingJupyter APPEND_STRING PROPERTY LINK_FLAGS ${cling_link_str}) + +endif(MSVC) if(ENABLE_SHARED) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") From bf35254da79e8c6214b1b6eb240d95bdf9927769 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Tue, 15 Jul 2025 22:52:47 +0800 Subject: [PATCH 09/10] Rename from cling_exports to jupyter_exports --- interpreter/cling/tools/Jupyter/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/interpreter/cling/tools/Jupyter/CMakeLists.txt b/interpreter/cling/tools/Jupyter/CMakeLists.txt index ccffbe09de7ef..71a04dec0d7ec 100644 --- a/interpreter/cling/tools/Jupyter/CMakeLists.txt +++ b/interpreter/cling/tools/Jupyter/CMakeLists.txt @@ -89,19 +89,19 @@ if(MSVC) set_target_properties(libclingJupyter PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) # RTTI/C++ symbols - set(cling_exports ${cling_exports} ??_7type_info@@6B@ + set(jupyter_exports ${jupyter_exports} ??_7type_info@@6B@ ?__type_info_root_node@@3U__type_info_node@@A ?nothrow@std@@3Unothrow_t@1@B ) # Compiler added symbols for static variables. NOT for VStudio < 2015 - set(cling_exports ${cling_exports} _Init_thread_abort _Init_thread_epoch + set(jupyter_exports ${jupyter_exports} _Init_thread_abort _Init_thread_epoch _Init_thread_footer _Init_thread_header _tls_index ) if(CMAKE_SIZEOF_VOID_P EQUAL 8) # new/delete variants needed when linking to static msvc runtime (esp. Debug) - set(cling_exports ${cling_exports} + set(jupyter_exports ${jupyter_exports} ??2@YAPEAX_K@Z ??3@YAXPEAX@Z ??_U@YAPEAX_K@Z @@ -119,7 +119,7 @@ if(MSVC) ?_Facet_Register@std@@YAXPEAV_Facet_base@1@@Z ) else() - set(cling_exports ${cling_exports} + set(jupyter_exports ${jupyter_exports} ??2@YAPAXI@Z ??3@YAXPAX@Z ??3@YAXPAXI@Z @@ -140,9 +140,9 @@ if(MSVC) endif() # List to '/EXPORT:sym0 /EXPORT:sym1 /EXPORT:sym2 ...' - foreach(sym ${cling_exports}) + foreach(sym ${jupyter_exports}) set(cling_link_str "${cling_link_str} /EXPORT:${sym}") - endforeach(sym ${cling_exports}) + endforeach(sym ${jupyter_exports}) set_property(TARGET libclingJupyter APPEND_STRING PROPERTY LINK_FLAGS ${cling_link_str}) From 46474a1195af2510aac7eaee6fc70041281f18c3 Mon Sep 17 00:00:00 2001 From: Yeoh Joer Date: Wed, 16 Jul 2025 04:32:04 +0800 Subject: [PATCH 10/10] Support std::cin --- interpreter/cling/tools/Jupyter/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interpreter/cling/tools/Jupyter/CMakeLists.txt b/interpreter/cling/tools/Jupyter/CMakeLists.txt index 71a04dec0d7ec..d063c869ddecb 100644 --- a/interpreter/cling/tools/Jupyter/CMakeLists.txt +++ b/interpreter/cling/tools/Jupyter/CMakeLists.txt @@ -92,6 +92,8 @@ if(MSVC) set(jupyter_exports ${jupyter_exports} ??_7type_info@@6B@ ?__type_info_root_node@@3U__type_info_node@@A ?nothrow@std@@3Unothrow_t@1@B + __std_find_trivial_1 + __std_reverse_trivially_swappable_1 ) # Compiler added symbols for static variables. NOT for VStudio < 2015