[lldb][test] Improve toolchain detection in Makefile.rules

This fix is based on a problem with cxx_compiler and cxx_linker macros
on Windows.
There was an issue with compiler detection in paths containing "icc". In
such case, Makefile.rules thought it was provided with icc compiler.

To solve that, utilities detection has been rewritten in Python.
The last element of compiler's path is separated, taking into account
platform path delimiter, and compiler type is extracted, with regard
of possible cross-toolchain prefix.  Paths for additional tools,
like OBJCOPY, are initialized with tools built with LLVM if USE_LLVM_TOOLS
is on, to achieve better portability for Windows.
Quotes from paths are removed in Makefile.rules, that may be added when
arguments are passed from Python to make.

Author: dzhidzhoev

lldb/packages/Python/lldbsuite/test/builders/builder.py

@@ -1,10 +1,12 @@
 import os
+import pathlib
 import platform
 import subprocess
 import sys
 import itertools
 
 import lldbsuite.test.lldbtest as lldbtest
+import lldbsuite.test.lldbplatformutil as lldbplatformutil
 import lldbsuite.test.lldbutil as lldbutil
 from lldbsuite.test import configuration
 from lldbsuite.test_event import build_exception
@@ -96,16 +98,120 @@ def getArchSpec(self, architecture):
         """
         return ["ARCH=" + architecture] if architecture else []
 
-    def getCCSpec(self, compiler):
+    def getToolchainSpec(self, compiler):
         """
-        Helper function to return the key-value string to specify the compiler
+        Helper function to return the key-value strings to specify the toolchain
         used for the make system.
         """
         cc = compiler if compiler else None
         if not cc and configuration.compiler:
             cc = configuration.compiler
+
         if cc:
-            return ['CC="%s"' % cc]
+            exe_ext = ""
+            if lldbplatformutil.getHostPlatform() == "windows":
+                exe_ext = ".exe"
+
+            cc = cc.strip()
+            cc_path = pathlib.Path(cc)
+            cc = cc_path.as_posix()
+
+            # We can get CC compiler string in the following formats:
+            #  [<tool>] <compiler>    - such as 'xrun clang', 'xrun /usr/bin/clang' & etc
+            #
+            # Where <compiler> could contain the following parts:
+            #   <simple-name>[.<exe-ext>]                           - sucn as 'clang', 'clang.exe' ('clamg-cl.exe'?)
+            #   <target-triple>-<simple-name>[.<exe-ext>]           - such as 'armv7-linux-gnueabi-gcc'
+            #   <path>/<simple-name>[.<exe-ext>]                    - such as '/usr/bin/clang', 'c:\path\to\compiler\clang,exe'
+            #   <path>/<target-triple>-<simple-name>[.<exe-ext>]    - such as '/usr/bin/clang', 'c:\path\to\compiler\clang,exe'
+
+            cc_ext = cc_path.suffix
+            # Compiler name without extension
+            cc_name = cc_path.stem.split(" ")[-1]
+
+            # A kind of compiler (canonical name): clang, gcc, cc & etc.
+            cc_type = cc_name
+            # A triple prefix of compiler name: <armv7-none-linux-gnu->gcc
+            cc_prefix = ""
+            if not "clang-cl" in cc_name and not "llvm-gcc" in cc_name:
+                cc_name_parts = cc_name.split("-")
+                cc_type = cc_name_parts[-1]
+                if len(cc_name_parts) > 1:
+                    cc_prefix = "-".join(cc_name_parts[:-1]) + "-"
+
+            # A kind of C++ compiler.
+            cxx_types = {
+                "icc": "icpc",
+                "llvm-gcc": "llvm-g++",
+                "gcc": "g++",
+                "cc": "c++",
+                "clang": "clang++",
+            }
+            cxx_type = cxx_types.get(cc_type, cc_type)
+
+            cc_dir = cc_path.parent
+
+            def getLlvmUtil(util_name):
+                llvm_tools_dir = os.getenv("LLVM_TOOLS_DIR", cc_dir.as_posix())
+                return llvm_tools_dir + "/" + util_name + exe_ext
+
+            def getToolchainUtil(util_name):
+                return (cc_dir / (cc_prefix + util_name + cc_ext)).as_posix()
+
+            cxx = getToolchainUtil(cxx_type)
+
+            util_names = {
+                "OBJCOPY": "objcopy",
+                "STRIP": "objcopy",
+                "ARCHIVER": "ar",
+                "DWP": "dwp",
+            }
+            utils = []
+
+            # Note: LLVM_AR is currently required by API TestBSDArchives.py tests.
+            # Assembly a full path to llvm-ar for given LLVM_TOOLS_DIR;
+            # otherwise assume we have llvm-ar at the same place where is CC specified.
+            if not os.getenv("LLVM_AR"):
+                llvm_ar = getLlvmUtil("llvm-ar")
+                utils.extend(['LLVM_AR="%s"' % llvm_ar])
+
+            if not lldbplatformutil.platformIsDarwin():
+                if os.getenv("USE_LLVM_TOOLS"):
+                    # Use the llvm project tool instead of the system defaults.
+                    # Note: do not override explicity specified tool from the cmd line.
+                    for var, name in util_names.items():
+                        utils.append('%s="%s"' % (var, getLlvmUtil("llvm-" + name)))
+                    utils.extend(['AR="%s"' % llvm_ar])
+                elif cc_type in ["clang", "cc", "gcc"]:
+                    util_paths = {}
+                    # Assembly a toolchain side tool cmd based on passed CC.
+                    for var, name in util_names.items():
+                        if not os.getenv(var):
+                            util_paths[var] = getToolchainUtil(name)
+                        else:
+                            util_paths[var] = os.getenv(var)
+                    utils.extend(['AR="%s"' % util_paths["ARCHIVER"]])
+
+                    # Look for llvm-dwp or gnu dwp
+                    if not lldbutil.which(util_paths["DWP"]):
+                        util_paths["DWP"] = getToolchainUtil("llvm-dwp")
+                    if not lldbutil.which(util_paths["DWP"]):
+                        util_paths["DWP"] = lldbutil.which("llvm-dwp")
+                    if not util_paths["DWP"]:
+                        util_paths["DWP"] = lldbutil.which("dwp")
+                        if not util_paths["DWP"]:
+                            del util_paths["DWP"]
+
+                    for var, path in util_paths.items():
+                        utils.append('%s="%s"' % (var, path))
+            else:
+                utils.extend(['AR="%slibtool"' % os.getenv("CROSS_COMPILE", "")])
+
+            return [
+                'CC="%s"' % cc,
+                'CCC="%s"' % cc_type,
+                'CXX="%s"' % cxx,
+            ] + utils
         return []
 
     def getSDKRootSpec(self):
@@ -178,7 +284,7 @@ def getBuildCommand(
             make_targets,
             self.getArchCFlags(architecture),
             self.getArchSpec(architecture),
-            self.getCCSpec(compiler),
+            self.getToolchainSpec(compiler),
             self.getExtraMakeArgs(),
             self.getSDKRootSpec(),
             self.getModuleCacheSpec(),

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

@@ -102,15 +102,33 @@ endif
 # If you change the defaults of CC, be sure to also change it in the file
 # test/builders/builder_base.py, which provides a Python way to return the
 # value of the make variable CC -- getCompiler().
-#
-# See also these functions:
-#   o cxx_compiler
-#   o cxx_linker
 #----------------------------------------------------------------------
 ifeq "$(CC)" ""
 $(error "C compiler is not specified. Please run tests through lldb-dotest or lit")
 endif
 
+# Remove all " and ' characters from the path. Also remove surrounding space chars.
+cstrip = $(strip $(subst ",,$(subst ',,$(1))))
+override CC := $(call cstrip,$(CC))
+override CCC := $(call cstrip,$(CCC))
+override CXX := $(call cstrip,$(CXX))
+# Always override the linker. Assign already normalized CC.
+override LD := $(call cstrip,$(CC))
+# A kind of linker. It always gets retrieved from CC.
+override LDC := $(call cstrip,$(CCC))
+override OBJCOPY := $(call cstrip,$(OBJCOPY))
+override STRIP := $(call cstrip,$(STRIP))
+override ARCHIVER := $(call cstrip,$(ARCHIVER))
+override AR := $(call cstrip,$(AR))
+override DWP := $(call cstrip,$(DWP))
+override LLVM_AR := $(call cstrip,$(LLVM_AR))
+
+ifeq "$(HOST_OS)" "Windows_NT"
+       # This function enframes the full path with the platform specific quotes. This is necessary to run the c++ executable
+       # properly under 'sh' on Windows host (prevent the path breakage because of Windows style path separators).
+       override CXX := $(QUOTE)$(CXX)$(QUOTE)
+endif
+
 #----------------------------------------------------------------------
 # Handle SDKROOT for the cross platform builds.
 #----------------------------------------------------------------------
@@ -147,10 +165,8 @@ ifeq "$(OS)" "Darwin"
 	DS := $(DSYMUTIL)
 	DSFLAGS := $(DSFLAGS_EXTRAS)
 	DSYM = $(EXE).dSYM
-	AR := $(CROSS_COMPILE)libtool
 	ARFLAGS := -static -o
 else
-	AR := $(CROSS_COMPILE)ar
 	# On non-Apple platforms, -arch becomes -m
 	ARCHFLAG := -m
 
@@ -213,7 +229,7 @@ endif
 LIMIT_DEBUG_INFO_FLAGS =
 NO_LIMIT_DEBUG_INFO_FLAGS =
 MODULE_DEBUG_INFO_FLAGS =
-ifneq (,$(findstring clang,$(CC)))
+ifeq ($(CCC), clang)
    LIMIT_DEBUG_INFO_FLAGS += -flimit-debug-info
    NO_LIMIT_DEBUG_INFO_FLAGS += -fno-limit-debug-info
    MODULE_DEBUG_INFO_FLAGS += -gmodules
@@ -279,7 +295,6 @@ endif
 
 CFLAGS += $(CFLAGS_EXTRAS)
 CXXFLAGS += -std=c++11 $(CFLAGS) $(ARCH_CXXFLAGS)
-LD = $(CC)
 # Copy common options to the linker flags (dwarf, arch. & etc).
 # Note: we get some 'garbage' options for linker here (such as -I, --isystem & etc).
 LDFLAGS += $(CFLAGS)
@@ -312,61 +327,6 @@ ifneq "$(DYLIB_NAME)" ""
 	endif
 endif
 
-# Function that returns the counterpart C++ compiler, given $(CC) as arg.
-cxx_compiler_notdir = $(if $(findstring icc,$(1)), \
-			$(subst icc,icpc,$(1)), \
-			$(if $(findstring llvm-gcc,$(1)), \
-				$(subst llvm-gcc,llvm-g++,$(1)), \
-				$(if $(findstring gcc,$(1)), \
-					$(subst gcc,g++,$(1)), \
-					$(subst cc,c++,$(1)))))
-cxx_compiler = $(if $(findstring /,$(1)),$(join $(dir $(1)), $(call cxx_compiler_notdir,$(notdir $(1)))),$(call cxx_compiler_notdir,$(1)))
-
-# Function that returns the C++ linker, given $(CC) as arg.
-cxx_linker_notdir = $(if $(findstring icc,$(1)), \
-			$(subst icc,icpc,$(1)), \
-			$(if $(findstring llvm-gcc,$(1)), \
-				$(subst llvm-gcc,llvm-g++,$(1)), \
-				$(if $(findstring gcc,$(1)), \
-					$(subst gcc,g++,$(1)), \
-					$(subst cc,c++,$(1)))))
-cxx_linker = $(if $(findstring /,$(1)),$(join $(dir $(1)), $(call cxx_linker_notdir,$(notdir $(1)))),$(call cxx_linker_notdir,$(1)))
-
-ifneq "$(OS)" "Darwin"
-	CLANG_OR_GCC := $(strip $(if $(findstring clang,$(CC)), \
-				$(findstring clang,$(CC)), \
-				$(if $(findstring gcc,$(CC)), \
-					$(findstring gcc,$(CC)), \
-					cc)))
-
-	CC_LASTWORD := $(strip $(lastword $(subst -, ,$(CC))))
-
-	replace_with = $(strip $(if $(findstring $(3),$(CC_LASTWORD)), \
-			$(subst $(3),$(1),$(2)), \
-			$(subst $(3),$(1),$(subst -$(CC_LASTWORD),,$(2)))))
-
-	ifeq "$(notdir $(CC))" "$(CC)"
-		replace_cc_with = $(call replace_with,$(1),$(CC),$(CLANG_OR_GCC))
-	else
-		replace_cc_with = $(join $(dir $(CC)),$(call replace_with,$(1),$(notdir $(CC)),$(CLANG_OR_GCC)))
-	endif
-
-	OBJCOPY ?= $(call replace_cc_with,objcopy)
-	ARCHIVER ?= $(call replace_cc_with,ar)
-	# Look for llvm-dwp or gnu dwp
-	DWP ?= $(call replace_cc_with,llvm-dwp)
-	ifeq ($(wildcard $(DWP)),)
-		DWP = $(call replace_cc_with,dwp)
-		ifeq ($(wildcard $(DWP)),)
-			DWP = $(shell command -v llvm-dwp 2> /dev/null)
-			ifeq ($(wildcard $(DWP)),)
-				DWP = $(shell command -v dwp 2> /dev/null)
-			endif
-		endif
-	endif
-	override AR = $(ARCHIVER)
-endif
-
 ifdef PIE
 	LDFLAGS += -pie
 endif
@@ -375,7 +335,7 @@ endif
 # Windows specific options
 #----------------------------------------------------------------------
 ifeq "$(OS)" "Windows_NT"
-	ifneq (,$(findstring clang,$(CC)))
+	ifeq ($(CCC), clang)
 		# Clang for Windows doesn't support C++ Exceptions
 		CXXFLAGS += -fno-exceptions
 		CXXFLAGS += -D_HAS_EXCEPTIONS=0
@@ -420,7 +380,7 @@ endif
 
 ifeq (1,$(USE_LIBSTDCPP))
 	# Clang requires an extra flag: -stdlib=libstdc++
-	ifneq (,$(findstring clang,$(CC)))
+	ifeq ($(CCC), clang)
 		# Force clang looking for the gcc's headers at specific rootfs folder.
 		CXXFLAGS += -stdlib=libstdc++ $(GCC_TOOLCHAIN_FLAGS)
 		LDFLAGS += -stdlib=libstdc++ $(GCC_TOOLCHAIN_FLAGS)
@@ -458,7 +418,7 @@ ifeq (1, $(USE_SYSTEM_STDLIB))
         CXXFLAGS += -nostdlib++ -nostdinc++ -cxx-isystem $(SDKROOT)/usr/include/c++/v1
         LDFLAGS += -L$(SDKROOT)/usr/lib -Wl,-rpath,$(SDKROOT)/usr/lib -lc++
     else
-        ifneq (,$(findstring clang,$(CC)))
+        ifeq ($(CCC), clang)
             # Force clang looking for the gcc's headers at specific rootfs folder.
             CXXFLAGS += $(GCC_TOOLCHAIN_FLAGS)
             LDFLAGS += $(GCC_TOOLCHAIN_FLAGS)
@@ -485,8 +445,6 @@ DYLIB_OBJECTS +=$(strip $(DYLIB_C_SOURCES:.c=.o))
 DYLIB_OBJECTS +=$(strip $(DYLIB_OBJC_SOURCES:.m=.o))
 ifneq "$(strip $(DYLIB_CXX_SOURCES))" ""
 	DYLIB_OBJECTS +=$(strip $(patsubst %.mm, %.o, $(DYLIB_CXX_SOURCES:.cpp=.o)))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 endif
 
 #----------------------------------------------------------------------
@@ -509,8 +467,6 @@ endif
 #----------------------------------------------------------------------
 ifneq "$(strip $(CXX_SOURCES))" ""
 	OBJECTS +=$(strip $(CXX_SOURCES:.cpp=.o))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 endif
 
 #----------------------------------------------------------------------
@@ -526,19 +482,18 @@ endif
 #----------------------------------------------------------------------
 ifneq "$(strip $(OBJCXX_SOURCES))" ""
 	OBJECTS +=$(strip $(OBJCXX_SOURCES:.mm=.o))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 	ifeq "$(findstring lobjc,$(LDFLAGS))" ""
 		LDFLAGS +=-lobjc
 	endif
 endif
 
-ifeq ($(findstring clang, $(CXX)), clang)
+ifeq ($(CCC), clang)
 	CXXFLAGS += --driver-mode=g++
 endif
 
 ifneq "$(CXX)" ""
-	ifeq ($(findstring clang, $(LD)), clang)
+	# Specify the driver mode parameter if we use clang as the linker.
+	ifeq ($(LDC), clang)
 		LDFLAGS += --driver-mode=g++
 	endif
 endif

lldb/test/API/functionalities/breakpoint/breakpoint_ids/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/breakpoint_locations/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/cpp/Makefile

@@ -1,7 +1,7 @@
 CXX_SOURCES := main.cpp
 CXXFLAGS_EXTRAS := -std=c++14
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/step_over_breakpoint/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CCC), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif