From 4b56a3be5b6114efedcd391aab7adef7d76732ff Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 14 Aug 2019 14:14:16 +0100 Subject: [PATCH 1/9] [ML] Embed an Info.plist in macOS programs This functionality may be required for notarization on macOS Catalina. --- mk/linux_crosscompile_macosx.mk | 2 ++ mk/macosx.mk | 2 ++ mk/make_info_plist.sh | 24 ++++++++++++++++++++++++ mk/rules.mk | 3 +++ mk/stdapp.mk | 4 ++-- mk/stddevapp.mk | 4 ++-- 6 files changed, 35 insertions(+), 4 deletions(-) create mode 100755 mk/make_info_plist.sh diff --git a/mk/linux_crosscompile_macosx.mk b/mk/linux_crosscompile_macosx.mk index 14edb9b60f..42f4e4b1a7 100644 --- a/mk/linux_crosscompile_macosx.mk +++ b/mk/linux_crosscompile_macosx.mk @@ -62,6 +62,8 @@ LOG4CXXLIBS=-llog4cxx ZLIBLIBS=-lz EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/lib +PLIST_FILE=$(OBJS_DIR)/Info.plist +PLIST_FILE_LDFLAGS=-Wl,-sectcreate,__TEXT,__info_plist,$(PLIST_FILE) OBJECT_FILE_EXT=.o DYNAMIC_LIB_EXT=.dylib DYNAMIC_LIB_DIR=lib diff --git a/mk/macosx.mk b/mk/macosx.mk index 9012cdac89..3bdf5e391c 100644 --- a/mk/macosx.mk +++ b/mk/macosx.mk @@ -61,6 +61,8 @@ LOG4CXXLIBS=-llog4cxx ZLIBLIBS=-lz EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/lib +PLIST_FILE=$(OBJS_DIR)/Info.plist +PLIST_FILE_LDFLAGS=-Wl,-sectcreate,__TEXT,__info_plist,$(PLIST_FILE) OBJECT_FILE_EXT=.o DYNAMIC_LIB_EXT=.dylib DYNAMIC_LIB_DIR=lib diff --git a/mk/make_info_plist.sh b/mk/make_info_plist.sh new file mode 100755 index 0000000000..3fe20cb1f7 --- /dev/null +++ b/mk/make_info_plist.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +ML_VERSION_NUM=`cat $CPP_SRC_HOME/gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $2 }' | xargs echo | sed 's/-.*//'` +ML_TARGET=$1 +ML_SHORTENED_TARGET=`echo $ML_TARGET | cut -c1-15` + +cat < + + CFBundleDevelopmentRegion + English + CFBundleIdentifier + co.elastic.ml-cpp.$ML_TARGET + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $ML_SHORTENED_TARGET + CFBundleDisplayName + $ML_TARGET + CFBundleVersion + $ML_VERSION_NUM + + +EOF diff --git a/mk/rules.mk b/mk/rules.mk index fe7e390c7c..a83e28ae33 100644 --- a/mk/rules.mk +++ b/mk/rules.mk @@ -47,6 +47,9 @@ $(OBJS_DIR)/%$(OBJECT_FILE_EXT): %.cc $(OBJS_DIR)/%$(OBJECT_FILE_EXT): %.c $(CC) -c $(COMP_OUT_FLAG)$@ $(CFLAGS) $(PICFLAGS) $(PDB_FLAGS) $(CPPFLAGS) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< +$(OBJS_DIR)/Info.plist: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh + $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) > $@ + $(OBJS_DIR)/%.plist: %.cc $(CXX) $(ANALYZE_OUT_FLAG)$@ $(ANALYZEFLAGS) $(filter-out -DNDEBUG, $(CPPFLAGS)) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< diff --git a/mk/stdapp.mk b/mk/stdapp.mk index 93cfc802b0..fa2c91475f 100644 --- a/mk/stdapp.mk +++ b/mk/stdapp.mk @@ -28,8 +28,8 @@ ifdef TARGET_CONF #CONF_INSTALL_CMD=$(MKDIR) $(CONF_INSTALL_DIR); $(INSTALL) $(TARGET_CONF) $(CONF_INSTALL_DIR) endif -$(TARGET): $(OBJS) $(RESOURCE_FILE) - $(CXX) $(LINK_OUT_FLAG)$@ $(PDB_FLAGS) $(OBJS) $(RESOURCE_FILE) $(LDFLAGS) $(LIBS) +$(TARGET): $(OBJS) $(RESOURCE_FILE) $(PLIST_FILE) + $(CXX) $(LINK_OUT_FLAG)$@ $(PDB_FLAGS) $(OBJS) $(RESOURCE_FILE) $(LDFLAGS) $(PLIST_FILE_LDFLAGS) $(LIBS) test: +$(TEST_CMDS) diff --git a/mk/stddevapp.mk b/mk/stddevapp.mk index 76994b635c..1b7cdf8c2f 100644 --- a/mk/stddevapp.mk +++ b/mk/stddevapp.mk @@ -12,8 +12,8 @@ LDFLAGS:=$(UTLDFLAGS) $(LDFLAGS) $(LIB_PATH) $(ML_VER_LDFLAGS) PICFLAGS=$(PLATPIEFLAGS) LIBS:=$(LOCALLIBS) $(LIB_ML_VER) $(LIBS) -$(TARGET): $(OBJS) $(RESOURCE_FILE) - $(CXX) $(LINK_OUT_FLAG)$@ $(PDB_FLAGS) $(OBJS) $(RESOURCE_FILE) $(LDFLAGS) $(LIBS) +$(TARGET): $(OBJS) $(RESOURCE_FILE) $(PLIST_FILE) + $(CXX) $(LINK_OUT_FLAG)$@ $(PDB_FLAGS) $(OBJS) $(RESOURCE_FILE) $(LDFLAGS) $(PLIST_FILE_LDFLAGS) $(LIBS) test: +$(TEST_CMDS) From 61f22c35f7815d978fc3e362268e6938d521a393 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 28 Aug 2019 18:25:18 +0100 Subject: [PATCH 2/9] Avoid clashing use of .plist files for static analysis output Static analysis files are now named *.xml (after all .plist files really are .xml files) --- mk/dynamiclib.mk | 2 +- mk/rules.mk | 6 +++--- mk/staticlib.mk | 2 +- mk/stdapp.mk | 2 +- mk/stdcpptest.mk | 2 +- mk/stddevapp.mk | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mk/dynamiclib.mk b/mk/dynamiclib.mk index 75562ee34b..6dd40265ce 100644 --- a/mk/dynamiclib.mk +++ b/mk/dynamiclib.mk @@ -64,6 +64,6 @@ build: $(TARGET) $(MAKE) -f $(filter-out %.mk %.d,$(MAKEFILE_LIST)) install clean: - $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(IMPORT_LIB_NAME) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp + $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.xml $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(IMPORT_LIB_NAME) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp +$(CLEAN_CMDS) $(RMDIR) results diff --git a/mk/rules.mk b/mk/rules.mk index a83e28ae33..843ee34c27 100644 --- a/mk/rules.mk +++ b/mk/rules.mk @@ -36,7 +36,7 @@ OS_SRCS=$(PLATFORM_SRCS) endif OBJS:=$(OBJS) $(patsubst %.cc, $(OBJS_DIR)/%$(OBJECT_FILE_EXT), $(SRCS)) -ANALYZEOBJS:=$(ANALYZEOBJS) $(patsubst %.cc, $(OBJS_DIR)/%.plist, $(SRCS)) +ANALYZEOBJS:=$(ANALYZEOBJS) $(patsubst %.cc, $(OBJS_DIR)/%.xml, $(SRCS)) # define MAKE_PREFIX_SRC_PATH to prefix source file names with their path # leave undefined for normal behaviour @@ -47,10 +47,10 @@ $(OBJS_DIR)/%$(OBJECT_FILE_EXT): %.cc $(OBJS_DIR)/%$(OBJECT_FILE_EXT): %.c $(CC) -c $(COMP_OUT_FLAG)$@ $(CFLAGS) $(PICFLAGS) $(PDB_FLAGS) $(CPPFLAGS) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< -$(OBJS_DIR)/Info.plist: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh +$(OBJS_DIR)/%.plist: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) > $@ -$(OBJS_DIR)/%.plist: %.cc +$(OBJS_DIR)/%.xml: %.cc $(CXX) $(ANALYZE_OUT_FLAG)$@ $(ANALYZEFLAGS) $(filter-out -DNDEBUG, $(CPPFLAGS)) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< # JOB_NAME will be set for builds kicked off by Jenkins, where dependency diff --git a/mk/staticlib.mk b/mk/staticlib.mk index 8e2658a88a..489b95f59b 100644 --- a/mk/staticlib.mk +++ b/mk/staticlib.mk @@ -33,5 +33,5 @@ install: build: $(TARGET) clean: - $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb + $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.xml $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb +$(CLEAN_CMDS) diff --git a/mk/stdapp.mk b/mk/stdapp.mk index fa2c91475f..d510e0f984 100644 --- a/mk/stdapp.mk +++ b/mk/stdapp.mk @@ -58,7 +58,7 @@ build: $(TARGET) $(MAKE) -f $(filter-out %.mk %.d,$(MAKEFILE_LIST)) install clean: - $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp + $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.xml $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp +$(CLEAN_CMDS) $(RMDIR) results diff --git a/mk/stdcpptest.mk b/mk/stdcpptest.mk index 8119344256..e7e563651e 100644 --- a/mk/stdcpptest.mk +++ b/mk/stdcpptest.mk @@ -24,7 +24,7 @@ $(TARGET): $(OBJS) $(RESOURCE_FILE) build: $(TARGET) clean: - $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp cppunit_results.xml + $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.xml $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp cppunit_results.xml $(RMDIR) results $(RMDIR) data diff --git a/mk/stddevapp.mk b/mk/stddevapp.mk index 1b7cdf8c2f..d351b4f731 100644 --- a/mk/stddevapp.mk +++ b/mk/stddevapp.mk @@ -38,7 +38,7 @@ build: $(TARGET) $(MAKE) -f $(filter-out %.mk %.d,$(MAKEFILE_LIST)) install clean: - $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp + $(RM) $(OBJS_DIR)/*$(OBJECT_FILE_EXT) $(OBJS_DIR)/*.d* $(OBJS_DIR)/*.plist $(OBJS_DIR)/*.xml $(OBJS_DIR)/*.res $(APP_CLEAN) core core.* $(TARGET) $(basename $(TARGET)).pdb $(basename $(TARGET)).map $(basename $(TARGET)).exp +$(CLEAN_CMDS) $(RMDIR) results From e494b78e53bfc051486a8ab8c8fb3d6f8cc7c414 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 29 Aug 2019 10:36:40 +0100 Subject: [PATCH 3/9] Add CFBundlePackageType = APPL to Info.plist Also make sure the identifier only uses valid characters --- mk/make_info_plist.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mk/make_info_plist.sh b/mk/make_info_plist.sh index 3fe20cb1f7..39c28ff0d1 100755 --- a/mk/make_info_plist.sh +++ b/mk/make_info_plist.sh @@ -3,14 +3,17 @@ ML_VERSION_NUM=`cat $CPP_SRC_HOME/gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $2 }' | xargs echo | sed 's/-.*//'` ML_TARGET=$1 ML_SHORTENED_TARGET=`echo $ML_TARGET | cut -c1-15` +ML_URL_TARGET=`echo $ML_TARGET | tr _ - | tr -d -C A-Za-z0-9- | cut -c1-63` cat < CFBundleDevelopmentRegion English + CFBundlePackageType + APPL CFBundleIdentifier - co.elastic.ml-cpp.$ML_TARGET + co.elastic.ml-cpp.$ML_URL_TARGET CFBundleInfoDictionaryVersion 6.0 CFBundleName From de99ed9db18115cfc9b13901836a913f417422d8 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 19 Sep 2019 17:19:10 +0100 Subject: [PATCH 4/9] Change macOS directory structure --- bin/autoconfig/Makefile | 2 +- bin/autodetect/Makefile | 2 +- bin/categorize/Makefile | 2 +- bin/controller/Makefile | 2 +- bin/data_frame_analyzer/Makefile | 2 +- bin/normalize/Makefile | 2 +- dev-tools/strip_binaries.sh | 43 ++++++++++++++++------- devbin/analyze_test/Makefile | 2 +- include/core/CResourceLocator.h | 3 -- lib/core/CResourceLocator.cc | 24 ++++--------- lib/core/Makefile | 2 +- lib/core/unittest/CResourceLocatorTest.cc | 10 ------ lib/core/unittest/CResourceLocatorTest.h | 1 - mk/linux.mk | 20 ++++++----- mk/linux_crosscompile_macosx.mk | 21 ++++++----- mk/macosx.mk | 21 ++++++----- mk/make_info_plist.sh | 10 ++++++ mk/stdapp.mk | 4 +-- mk/windows.mk | 22 ++++++------ set_env.sh | 8 ++--- 20 files changed, 109 insertions(+), 94 deletions(-) diff --git a/bin/autoconfig/Makefile b/bin/autoconfig/Makefile index 9740197d67..c01e987dae 100644 --- a/bin/autoconfig/Makefile +++ b/bin/autoconfig/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/autoconfig$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) $(LIB_ML_CONFIG) diff --git a/bin/autodetect/Makefile b/bin/autodetect/Makefile index a4051da9cd..0150537ba6 100644 --- a/bin/autodetect/Makefile +++ b/bin/autodetect/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/autodetect$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) diff --git a/bin/categorize/Makefile b/bin/categorize/Makefile index 692ee5ce12..e1fdbeac86 100644 --- a/bin/categorize/Makefile +++ b/bin/categorize/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/categorize$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) diff --git a/bin/controller/Makefile b/bin/controller/Makefile index f12718ba02..e07dab750b 100644 --- a/bin/controller/Makefile +++ b/bin/controller/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/controller$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) diff --git a/bin/data_frame_analyzer/Makefile b/bin/data_frame_analyzer/Makefile index 21c130117c..29835a6f1e 100644 --- a/bin/data_frame_analyzer/Makefile +++ b/bin/data_frame_analyzer/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/data_frame_analyzer$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) diff --git a/bin/normalize/Makefile b/bin/normalize/Makefile index dae92c89b2..7480eddf7b 100644 --- a/bin/normalize/Makefile +++ b/bin/normalize/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=$(OBJS_DIR)/normalize$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh index 40345d1a05..634a4b8b3e 100755 --- a/dev-tools/strip_binaries.sh +++ b/dev-tools/strip_binaries.sh @@ -5,7 +5,26 @@ # you may not use this file except in compliance with the Elastic License. # -# Strips Ml native code binaries to make them smaller before distribution. +# Strips ML native code binaries to make them smaller before distribution. + +case `uname` in + + Darwin) + EXE_DIR=controller.app/Contents/MacOS + DYNAMIC_LIB_DIR=controller.app/Contents/lib + ;; + + Linux) + if [ -z "$CPP_CROSS_COMPILE" ] ; then + EXE_DIR=bin + DYNAMIC_LIB_DIR=lib + else + EXE_DIR=controller.app/Contents/MacOS + DYNAMIC_LIB_DIR=controller.app/Contents/lib + fi + ;; + +esac # Ensure $CPP_PLATFORM_HOME is set if [ -z "$CPP_PLATFORM_HOME" ] ; then @@ -13,15 +32,15 @@ if [ -z "$CPP_PLATFORM_HOME" ] ; then exit 1 fi -# Ensure the bin folder has been created. -if [ ! -d "$CPP_PLATFORM_HOME/bin" ] ; then - echo '$CPP_PLATFORM_HOME/bin does not exist' +# Ensure the executable programs folder has been created. +if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then + echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist" exit 2 fi # Ensure the lib folder has been created. -if [ ! -d "$CPP_PLATFORM_HOME/lib" ] ; then - echo '$CPP_PLATFORM_HOME/lib does not exist' +if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then + echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist" exit 3 fi @@ -32,13 +51,13 @@ cd "$CPP_PLATFORM_HOME" case `uname` in Darwin) - for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'` + for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $PROGRAM" dsymutil $PROGRAM strip -u -r $PROGRAM done - for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'` + for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $LIBRARY" case $LIBRARY in @@ -51,7 +70,7 @@ case `uname` in Linux) if [ -z "$CPP_CROSS_COMPILE" ] ; then - for PROGRAM in `ls -1 bin/* | egrep -v 'bin/core|-debug$'` + for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'` do echo "Stripping $PROGRAM" objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug" @@ -59,7 +78,7 @@ case `uname` in objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM" chmod -x "$PROGRAM-debug" done - for LIBRARY in `ls -1 lib/* | egrep -v 'lib/core|-debug$'` + for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'` do echo "Stripping $LIBRARY" objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug" @@ -68,13 +87,13 @@ case `uname` in done else if [ "$CPP_CROSS_COMPILE" = macosx ] ; then - for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'` + for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $PROGRAM" llvm-dsymutil-3.9 $PROGRAM /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM done - for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'` + for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $LIBRARY" case $LIBRARY in diff --git a/devbin/analyze_test/Makefile b/devbin/analyze_test/Makefile index 7fbf73917d..42a4f897c2 100644 --- a/devbin/analyze_test/Makefile +++ b/devbin/analyze_test/Makefile @@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk TARGET=analyze_test$(EXE_EXT) -INSTALL_DIR=$(CPP_PLATFORM_HOME)/devbin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) all: build diff --git a/include/core/CResourceLocator.h b/include/core/CResourceLocator.h index 0e00f95042..5eb2dd0073 100644 --- a/include/core/CResourceLocator.h +++ b/include/core/CResourceLocator.h @@ -36,9 +36,6 @@ class CORE_EXPORT CResourceLocator : private CNonInstantiatable { //! details and dictionary words. static std::string resourceDir(); - //! Get the directory that stores log files. - static std::string logDir(); - //! Get the root directory for the C++ section of the source tree. //! (Obviously this should only be used in test code!) static std::string cppRootDir(); diff --git a/lib/core/CResourceLocator.cc b/lib/core/CResourceLocator.cc index 7c0530e8ee..723b59d9f0 100644 --- a/lib/core/CResourceLocator.cc +++ b/lib/core/CResourceLocator.cc @@ -12,6 +12,12 @@ namespace { const char* const CPP_SRC_HOME("CPP_SRC_HOME"); + +#ifdef MacOSX +const std::string RESOURCE_RELATIVE_DIR{"../Resources"}; +#else +const std::string RESOURCE_RELATIVE_DIR{"../resources"}; +#endif } namespace ml { @@ -22,7 +28,7 @@ std::string CResourceLocator::resourceDir() { // $ES_HOME/plugin//resources // $ES_HOME/plugin//platform//bin - std::string productionDir(CProgName::progDir() + "/../../../resources"); + std::string productionDir(CProgName::progDir() + RESOURCE_RELATIVE_DIR); // If the production directory doesn't exist, return the dev directory if // that does, but if neither exist return the production directory so the @@ -42,22 +48,6 @@ std::string CResourceLocator::resourceDir() { return productionDir; } -std::string CResourceLocator::logDir() { - // Look relative to the program that's running, assuming this directory layout: - // $ES_HOME/logs - // $ES_HOME/plugin//platform//bin - - std::string productionDir(CProgName::progDir() + "/../../../../../logs"); - - COsFileFuncs::TStat buf; - if (COsFileFuncs::stat(productionDir.c_str(), &buf) != 0) { - // Assume we're running as a unit test - return "."; - } - - return productionDir; -} - std::string CResourceLocator::cppRootDir() { const char* cppSrcHome(::getenv(CPP_SRC_HOME)); if (cppSrcHome == nullptr) { diff --git a/lib/core/Makefile b/lib/core/Makefile index 23f657cf9e..17b645f098 100644 --- a/lib/core/Makefile +++ b/lib/core/Makefile @@ -21,7 +21,7 @@ USE_STRPTIME=1 CPPFLAGS+= -DDYNAMIC_LIB_EXT=$(DYNAMIC_LIB_EXT) -CONF_INSTALL_DIR=$(CPP_DISTRIBUTION_HOME)/resources +CONF_INSTALL_DIR=$(CPP_DISTRIBUTION_HOME)/$(RESOURCES_DIR) # On Windows we need to copy over the Boost date/time config file that lists # details of each timezone, whereas on Unix timezone config is part of the diff --git a/lib/core/unittest/CResourceLocatorTest.cc b/lib/core/unittest/CResourceLocatorTest.cc index a0abbf73ae..1cd00379ae 100644 --- a/lib/core/unittest/CResourceLocatorTest.cc +++ b/lib/core/unittest/CResourceLocatorTest.cc @@ -14,8 +14,6 @@ CppUnit::Test* CResourceLocatorTest::suite() { suiteOfTests->addTest(new CppUnit::TestCaller( "CResourceLocatorTest::testResourceDir", &CResourceLocatorTest::testResourceDir)); - suiteOfTests->addTest(new CppUnit::TestCaller( - "CResourceLocatorTest::testLogDir", &CResourceLocatorTest::testLogDir)); suiteOfTests->addTest(new CppUnit::TestCaller( "CResourceLocatorTest::testSrcRootDir", &CResourceLocatorTest::testSrcRootDir)); @@ -32,14 +30,6 @@ void CResourceLocatorTest::testResourceDir() { 0, ml::core::COsFileFuncs::stat((resourceDir + "/ml-en.dict").c_str(), &buf)); } -void CResourceLocatorTest::testLogDir() { - std::string logDir(ml::core::CResourceLocator::logDir()); - LOG_DEBUG(<< "Log directory is " << logDir); - - // Don't assert on this as it will be non-essential once - // we're an Elasticsearch plugin -} - void CResourceLocatorTest::testSrcRootDir() { std::string cppRootDir(ml::core::CResourceLocator::cppRootDir()); LOG_DEBUG(<< "C++ root directory is " << cppRootDir); diff --git a/lib/core/unittest/CResourceLocatorTest.h b/lib/core/unittest/CResourceLocatorTest.h index 38a48ff594..e32135387f 100644 --- a/lib/core/unittest/CResourceLocatorTest.h +++ b/lib/core/unittest/CResourceLocatorTest.h @@ -11,7 +11,6 @@ class CResourceLocatorTest : public CppUnit::TestFixture { public: void testResourceDir(); - void testLogDir(); void testSrcRootDir(); static CppUnit::Test* suite(); diff --git a/mk/linux.mk b/mk/linux.mk index 5ddd3d129f..879e4db6c1 100644 --- a/mk/linux.mk +++ b/mk/linux.mk @@ -32,6 +32,14 @@ CDEPFLAGS=-MM COMP_OUT_FLAG=-o LINK_OUT_FLAG=-o DEP_REFORMAT=sed 's,\($*\)\.o[ :]*,$(OBJS_DIR)\/\1.o $@ : ,g' +OBJECT_FILE_EXT=.o +EXE_DIR=bin +DYNAMIC_LIB_EXT=.so +DYNAMIC_LIB_DIR=lib +STATIC_LIB_EXT=.a +SHELL_SCRIPT_EXT=.sh +UT_TMP_DIR=/tmp/$(LOGNAME) +RESOURCES_DIR=resources LOCALLIBS=-lm -lpthread -ldl -lrt NETLIBS=-lnsl BOOSTVER=1_71 @@ -52,20 +60,14 @@ RAPIDJSONCPPFLAGS=-DRAPIDJSON_HAS_STDSTRING -DRAPIDJSON_SSE42 EIGENCPPFLAGS=-DEIGEN_MPL2_ONLY XMLINCLUDES=`/usr/local/gcc73/bin/xml2-config --cflags` XMLLIBS=`/usr/local/gcc73/bin/xml2-config --libs` -DYNAMICLIBLDFLAGS=$(PLATPICFLAGS) -shared -Wl,--as-needed -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/.' +DYNAMICLIBLDFLAGS=$(PLATPICFLAGS) -shared -Wl,--as-needed -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/.' JAVANATIVEINCLUDES=-I$(JAVA_HOME)/include JAVANATIVELDFLAGS=-L$(JAVA_HOME)/jre/lib/server JAVANATIVELIBS=-ljvm CPPUNITLIBS=-lcppunit ZLIBLIBS=-lz -EXELDFLAGS=-pie $(PLATPIEFLAGS) -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/../lib' -UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/lib -OBJECT_FILE_EXT=.o -DYNAMIC_LIB_EXT=.so -DYNAMIC_LIB_DIR=lib -STATIC_LIB_EXT=.a -SHELL_SCRIPT_EXT=.sh -UT_TMP_DIR=/tmp/$(LOGNAME) +EXELDFLAGS=-pie $(PLATPIEFLAGS) -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/../lib' +UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) LIB_ML_CORE=-lMlCore LIB_ML_VER=-lMlVer ML_VER_LDFLAGS=-L$(CPP_SRC_HOME)/lib/ver/.objs diff --git a/mk/linux_crosscompile_macosx.mk b/mk/linux_crosscompile_macosx.mk index ba56c10be6..9aab09aca2 100644 --- a/mk/linux_crosscompile_macosx.mk +++ b/mk/linux_crosscompile_macosx.mk @@ -7,6 +7,7 @@ OS=MacOSX CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/darwin-x86_64 +APP_CONTENTS=controller.app/Contents CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.11 SYSROOT=/usr/local/sysroot-$(CROSS_TARGET_PLATFORM) @@ -44,6 +45,14 @@ COMP_OUT_FLAG=-o ANALYZE_OUT_FLAG=-o LINK_OUT_FLAG=-o DEP_REFORMAT=sed 's,\($*\)\.o[ :]*,$(OBJS_DIR)\/\1.o $@ : ,g' +OBJECT_FILE_EXT=.o +EXE_DIR=$(APP_CONTENTS)/MacOS +DYNAMIC_LIB_EXT=.dylib +DYNAMIC_LIB_DIR=$(APP_CONTENTS)/lib +STATIC_LIB_EXT=.a +SHELL_SCRIPT_EXT=.sh +UT_TMP_DIR=/tmp/$(LOGNAME) +RESOURCES_DIR=$(APP_CONTENTS)/Resources LOCALLIBS= NETLIBS= BOOSTVER=1_71 @@ -65,19 +74,13 @@ XMLINCLUDES=-isystem $(SYSROOT)/usr/include/libxml2 XMLLIBLDFLAGS=-L$(SYSROOT)/usr/lib XMLLIBS=-lxml2 ML_VERSION_NUM=$(shell cat $(CPP_SRC_HOME)/gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $$2 }' | xargs echo | sed 's/-.*//') -DYNAMICLIBLDFLAGS=-current_version $(ML_VERSION_NUM) -compatibility_version $(ML_VERSION_NUM) -dynamiclib -Wl,-dead_strip_dylibs $(COVERAGE) -Wl,-install_name,@rpath/$(notdir $(TARGET)) -L$(CPP_PLATFORM_HOME)/lib -Wl,-rpath,@loader_path/. -Wl,-headerpad_max_install_names +DYNAMICLIBLDFLAGS=-current_version $(ML_VERSION_NUM) -compatibility_version $(ML_VERSION_NUM) -dynamiclib -Wl,-dead_strip_dylibs $(COVERAGE) -Wl,-install_name,@rpath/$(notdir $(TARGET)) -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) -Wl,-rpath,@loader_path/. -Wl,-headerpad_max_install_names CPPUNITLIBS=-lcppunit ZLIBLIBS=-lz -EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names -UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/lib +EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names +UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) PLIST_FILE=$(OBJS_DIR)/Info.plist PLIST_FILE_LDFLAGS=-Wl,-sectcreate,__TEXT,__info_plist,$(PLIST_FILE) -OBJECT_FILE_EXT=.o -DYNAMIC_LIB_EXT=.dylib -DYNAMIC_LIB_DIR=lib -STATIC_LIB_EXT=.a -SHELL_SCRIPT_EXT=.sh -UT_TMP_DIR=/tmp/$(LOGNAME) LIB_ML_API=-lMlApi LIB_ML_CORE=-lMlCore LIB_ML_VER=-lMlVer diff --git a/mk/macosx.mk b/mk/macosx.mk index 5ce52de5cb..1db98f7def 100644 --- a/mk/macosx.mk +++ b/mk/macosx.mk @@ -7,6 +7,7 @@ OS=MacOSX CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/darwin-x86_64 +APP_CONTENTS=controller.app/Contents CC=clang CXX=clang++ -std=c++14 -stdlib=libc++ @@ -33,6 +34,14 @@ COMP_OUT_FLAG=-o ANALYZE_OUT_FLAG=-o LINK_OUT_FLAG=-o DEP_REFORMAT=sed 's,\($*\)\.o[ :]*,$(OBJS_DIR)\/\1.o $@ : ,g' +OBJECT_FILE_EXT=.o +EXE_DIR=$(APP_CONTENTS)/MacOS +DYNAMIC_LIB_EXT=.dylib +DYNAMIC_LIB_DIR=$(APP_CONTENTS)/lib +STATIC_LIB_EXT=.a +SHELL_SCRIPT_EXT=.sh +UT_TMP_DIR=/tmp/$(LOGNAME) +RESOURCES_DIR=$(APP_CONTENTS)/Resources LOCALLIBS= NETLIBS= BOOSTVER=1_71 @@ -58,19 +67,13 @@ JAVANATIVEINCLUDES=-I`/usr/libexec/java_home`/include JAVANATIVELDFLAGS=-L`/usr/libexec/java_home`/jre/lib/server JAVANATIVELIBS=-ljvm ML_VERSION_NUM:=$(shell cat $(CPP_SRC_HOME)/gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $$2 }' | xargs echo | sed 's/-.*//') -DYNAMICLIBLDFLAGS=-current_version $(ML_VERSION_NUM) -compatibility_version $(ML_VERSION_NUM) -dynamiclib -Wl,-dead_strip_dylibs $(COVERAGE) -Wl,-install_name,@rpath/$(notdir $(TARGET)) -L$(CPP_PLATFORM_HOME)/lib -Wl,-rpath,@loader_path/. -Wl,-headerpad_max_install_names +DYNAMICLIBLDFLAGS=-current_version $(ML_VERSION_NUM) -compatibility_version $(ML_VERSION_NUM) -dynamiclib -Wl,-dead_strip_dylibs $(COVERAGE) -Wl,-install_name,@rpath/$(notdir $(TARGET)) -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) -Wl,-rpath,@loader_path/. -Wl,-headerpad_max_install_names CPPUNITLIBS=-lcppunit ZLIBLIBS=-lz -EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/lib $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names -UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/lib +EXELDFLAGS=-bind_at_load -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-rpath,@loader_path/../lib -Wl,-headerpad_max_install_names +UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) PLIST_FILE=$(OBJS_DIR)/Info.plist PLIST_FILE_LDFLAGS=-Wl,-sectcreate,__TEXT,__info_plist,$(PLIST_FILE) -OBJECT_FILE_EXT=.o -DYNAMIC_LIB_EXT=.dylib -DYNAMIC_LIB_DIR=lib -STATIC_LIB_EXT=.a -SHELL_SCRIPT_EXT=.sh -UT_TMP_DIR=/tmp/$(LOGNAME) LIB_ML_API=-lMlApi LIB_ML_CORE=-lMlCore LIB_ML_VER=-lMlVer diff --git a/mk/make_info_plist.sh b/mk/make_info_plist.sh index 39c28ff0d1..0bd8f79f3b 100755 --- a/mk/make_info_plist.sh +++ b/mk/make_info_plist.sh @@ -22,6 +22,16 @@ cat <$ML_TARGET CFBundleVersion $ML_VERSION_NUM +EOF + +if [ $# -gt 1 ] ; then + cat <CFBundleExecutable + $2 +EOF +fi + +cat < EOF diff --git a/mk/stdapp.mk b/mk/stdapp.mk index d510e0f984..f17449c619 100644 --- a/mk/stdapp.mk +++ b/mk/stdapp.mk @@ -11,12 +11,12 @@ PICFLAGS=$(PLATPIEFLAGS) LIBS:=$(LOCALLIBS) $(LIB_ML_VER) $(LIB_ML_SECCOMP) $(LIBS) ifndef INSTALL_DIR -INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin +INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) endif ifndef CONF_INSTALL_DIR ifdef TARGET_CONF -CONF_INSTALL_DIR=$(CPP_DISTRIBUTION_HOME)/resources +CONF_INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(RESOURCES_DIR) endif endif diff --git a/mk/windows.mk b/mk/windows.mk index 2368ea07dd..d6f67f8d84 100644 --- a/mk/windows.mk +++ b/mk/windows.mk @@ -62,6 +62,18 @@ AR_OUT_FLAG=-OUT: # do then we'll rebuild everything from scratch DEP_FILTER= 2>/dev/null | egrep "^.line .*(\\.h|$<)" | tr -s '\\\\' '/' | awk -F'"' '{ print $$2 }' | egrep -i -v "usr.local|$(LOCAL_DRIVE)..progra" | sed 's~/[a-z]*/\.\./~/~g' | sort -f -u | sort -t. -k2 | tr '\r\n\t' ' ' | sed 's/ / /g' | sed 's/^ //' | sed 's/ $$//' DEP_REFORMAT=sed 's,$<,$(basename $@)$(OBJECT_FILE_EXT) $@ : $<,' +OBJECT_FILE_EXT=.obj +EXE_EXT=.exe +EXE_DIR=bin +DYNAMIC_LIB_EXT=.dll +DYNAMIC_LIB_DIR=bin +IMPORT_LIB_DIR=lib +RESOURCE_FILE=$(OBJS_DIR)/ml.res +STATIC_LIB_EXT=.lib +SHELL_SCRIPT_EXT=.bat +# This temp directory assumes we're running in a Unix-like shell such as Git bash +UT_TMP_DIR=/tmp +RESOURCES_DIR=resources LOCALLIBS=AdvAPI32.lib shell32.lib Version.lib NETLIBS=WS2_32.lib BOOSTVER=1_71 @@ -90,16 +102,6 @@ ZLIBLIBS=zdll.lib STRPTIMELIBS=strptime.lib EXELDFLAGS=-nologo -Zi $(CRT_OPT) -link -MAP -OPT:REF -SUBSYSTEM:CONSOLE,6.1 -STACK:0x800000 -INCREMENTAL:NO -LIBPATH:$(CPP_PLATFORM_HOME)/$(IMPORT_LIB_DIR) UTLDFLAGS=$(EXELDFLAGS) -OBJECT_FILE_EXT=.obj -DYNAMIC_LIB_EXT=.dll -DYNAMIC_LIB_DIR=bin -IMPORT_LIB_DIR=lib -RESOURCE_FILE=$(OBJS_DIR)/ml.res -STATIC_LIB_EXT=.lib -SHELL_SCRIPT_EXT=.bat -# This temp directory assumes we're running in a Unix-like shell such as Git bash -UT_TMP_DIR=/tmp -EXE_EXT=.exe INSTALL=cp CP=cp RC=rc -nologo diff --git a/set_env.sh b/set_env.sh index 288b19c05a..166d07cb3c 100755 --- a/set_env.sh +++ b/set_env.sh @@ -23,7 +23,7 @@ export CPP_SRC_HOME=`cd "$MY_DIR" && pwd` case `uname` in Darwin) - SIMPLE_PLATFORM=macosx + SIMPLE_PLATFORM=macos BUNDLE_PLATFORM=darwin-x86_64 ;; @@ -36,7 +36,7 @@ case `uname` in if [ -z "$CPP_CROSS_COMPILE" ] ; then BUNDLE_PLATFORM=linux-x86_64 else - if [ "$CPP_CROSS_COMPILE" = macosx ] ; then + if [ "$CPP_CROSS_COMPILE" = macos ] ; then BUNDLE_PLATFORM=darwin-x86_64 else echo "Cannot cross compile to $CPP_CROSS_COMPILE" @@ -103,7 +103,7 @@ case $SIMPLE_PLATFORM in PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin ;; - macosx) + macos) PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin ;; @@ -160,7 +160,7 @@ case $SIMPLE_PLATFORM in MAKE=`which make` ;; - macosx|windows) + macos|windows) MAKE=`which gnumake` ;; From 3dc859849f6053eee7d68cc52a536bd68fa2b505 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 23 Sep 2019 12:43:22 +0100 Subject: [PATCH 5/9] Switch CPP_CROSS_COMPILE back to macosx --- set_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/set_env.sh b/set_env.sh index 166d07cb3c..601965d095 100755 --- a/set_env.sh +++ b/set_env.sh @@ -36,7 +36,7 @@ case `uname` in if [ -z "$CPP_CROSS_COMPILE" ] ; then BUNDLE_PLATFORM=linux-x86_64 else - if [ "$CPP_CROSS_COMPILE" = macos ] ; then + if [ "$CPP_CROSS_COMPILE" = macosx ] ; then BUNDLE_PLATFORM=darwin-x86_64 else echo "Cannot cross compile to $CPP_CROSS_COMPILE" From 3dddb81bfcce0f0b1719455d360f6d8eacd5762c Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 23 Sep 2019 15:19:51 +0100 Subject: [PATCH 6/9] Fix resource dir and include top level Info.plist with XML header --- bin/controller/Makefile | 7 ++++++- lib/core/Makefile | 2 +- mk/make_info_plist.sh | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/controller/Makefile b/bin/controller/Makefile index e07dab750b..3b8573123b 100644 --- a/bin/controller/Makefile +++ b/bin/controller/Makefile @@ -16,7 +16,7 @@ USE_BOOST_PROGRAMOPTIONS_LIBS=1 LIBS=$(ML_LIBS) -all: build +all:: build SRCS= \ Main.cc \ @@ -24,6 +24,11 @@ SRCS= \ CCmdLineParser.cc \ CCommandProcessor.cc \ +#ifneq ($(PLIST_FILE),) +all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh + $(MKDIR) $(INSTALL_DIR) + $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) $(notdir $(TARGET)) > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE)) +#endif include $(CPP_SRC_HOME)/mk/stdapp.mk diff --git a/lib/core/Makefile b/lib/core/Makefile index 17b645f098..35f3c00d79 100644 --- a/lib/core/Makefile +++ b/lib/core/Makefile @@ -21,7 +21,7 @@ USE_STRPTIME=1 CPPFLAGS+= -DDYNAMIC_LIB_EXT=$(DYNAMIC_LIB_EXT) -CONF_INSTALL_DIR=$(CPP_DISTRIBUTION_HOME)/$(RESOURCES_DIR) +CONF_INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(RESOURCES_DIR) # On Windows we need to copy over the Boost date/time config file that lists # details of each timezone, whereas on Unix timezone config is part of the diff --git a/mk/make_info_plist.sh b/mk/make_info_plist.sh index 0bd8f79f3b..21c9183409 100755 --- a/mk/make_info_plist.sh +++ b/mk/make_info_plist.sh @@ -6,6 +6,8 @@ ML_SHORTENED_TARGET=`echo $ML_TARGET | cut -c1-15` ML_URL_TARGET=`echo $ML_TARGET | tr _ - | tr -d -C A-Za-z0-9- | cut -c1-63` cat < + CFBundleDevelopmentRegion From 99c602725cea69832b6588c3e124d9a3acc129f2 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 23 Sep 2019 17:30:14 +0100 Subject: [PATCH 7/9] Oops --- bin/controller/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/controller/Makefile b/bin/controller/Makefile index 3b8573123b..9fb72d23b6 100644 --- a/bin/controller/Makefile +++ b/bin/controller/Makefile @@ -24,11 +24,11 @@ SRCS= \ CCmdLineParser.cc \ CCommandProcessor.cc \ -#ifneq ($(PLIST_FILE),) +ifneq ($(PLIST_FILE),) all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh $(MKDIR) $(INSTALL_DIR) $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) $(notdir $(TARGET)) > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE)) -#endif +endif include $(CPP_SRC_HOME)/mk/stdapp.mk From 08975981810ea9e7a5825a857a12b0c39ab2c5c3 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 24 Sep 2019 12:14:08 +0100 Subject: [PATCH 8/9] Fix problem with ordering of static initialization --- lib/core/CResourceLocator.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/core/CResourceLocator.cc b/lib/core/CResourceLocator.cc index 723b59d9f0..b33789e8ac 100644 --- a/lib/core/CResourceLocator.cc +++ b/lib/core/CResourceLocator.cc @@ -8,15 +8,18 @@ #include #include -#include +#include namespace { -const char* const CPP_SRC_HOME("CPP_SRC_HOME"); +// Important: all file scope in this file must be simple types that don't +// require construction, as they could be accessed before the static +// constructors are run. +const char* const CPP_SRC_HOME{"CPP_SRC_HOME"}; #ifdef MacOSX -const std::string RESOURCE_RELATIVE_DIR{"../Resources"}; +const char* const RESOURCE_RELATIVE_DIR{"../Resources"}; #else -const std::string RESOURCE_RELATIVE_DIR{"../resources"}; +const char* const RESOURCE_RELATIVE_DIR{"../resources"}; #endif } @@ -24,10 +27,9 @@ namespace ml { namespace core { std::string CResourceLocator::resourceDir() { - // Look relative to the program that's running, assuming this directory layout: - // $ES_HOME/plugin//resources - // $ES_HOME/plugin//platform//bin + // Look relative to the program that's running, assuming that the resource + // directory is located relative to the directory the current program is in. std::string productionDir(CProgName::progDir() + RESOURCE_RELATIVE_DIR); // If the production directory doesn't exist, return the dev directory if @@ -35,7 +37,7 @@ std::string CResourceLocator::resourceDir() { // error message is nicer for the end user. COsFileFuncs::TStat buf; if (COsFileFuncs::stat(productionDir.c_str(), &buf) != 0) { - const char* cppSrcHome(::getenv(CPP_SRC_HOME)); + const char* cppSrcHome(std::getenv(CPP_SRC_HOME)); if (cppSrcHome != nullptr) { std::string devDir(cppSrcHome); devDir += "/lib/core"; @@ -49,7 +51,7 @@ std::string CResourceLocator::resourceDir() { } std::string CResourceLocator::cppRootDir() { - const char* cppSrcHome(::getenv(CPP_SRC_HOME)); + const char* cppSrcHome(std::getenv(CPP_SRC_HOME)); if (cppSrcHome == nullptr) { // Assume we're in a unittest directory return "../../.."; From d0761b5c2d1c5951a33d223c724163c97a7f6a05 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 24 Sep 2019 13:51:25 +0100 Subject: [PATCH 9/9] Now we're shipping an app, use the app name in all Info.plist entries When this PR started out we thought we'd be shipping a number of standalone "command line tools". But that didn't work with notarization, so we had to make a more radical change, to become an app. (I also removed some dead code related to detecting Linux MUSL.) --- 3rd_party/3rd_party.sh | 26 +++++++-------- bin/controller/Makefile | 2 +- dev-tools/strip_binaries.sh | 56 +++++++++++++++++---------------- docs/CHANGELOG.asciidoc | 7 +++++ mk/defines.mk | 5 --- mk/linux_crosscompile_macosx.mk | 3 +- mk/macosx.mk | 3 +- mk/make_info_plist.sh | 14 ++++----- mk/rules.mk | 2 +- set_env.sh | 21 +++++-------- 10 files changed, 67 insertions(+), 72 deletions(-) diff --git a/3rd_party/3rd_party.sh b/3rd_party/3rd_party.sh index feca2f781b..e038f6f422 100755 --- a/3rd_party/3rd_party.sh +++ b/3rd_party/3rd_party.sh @@ -59,21 +59,19 @@ case `uname` in STL_PREFIX=libstdc++ STL_EXTENSION=.so.6 ZLIB_LOCATION= + elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11 + BOOST_LOCATION=$SYSROOT/usr/local/lib + BOOST_COMPILER=clang + BOOST_EXTENSION=mt-x64-1_71.dylib + BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread' + XML_LOCATION= + GCC_RT_LOCATION= + STL_LOCATION= + ZLIB_LOCATION= else - if [ "$CPP_CROSS_COMPILE" = macosx ] ; then - SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11 - BOOST_LOCATION=$SYSROOT/usr/local/lib - BOOST_COMPILER=clang - BOOST_EXTENSION=mt-x64-1_71.dylib - BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread' - XML_LOCATION= - GCC_RT_LOCATION= - STL_LOCATION= - ZLIB_LOCATION= - else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 3 - fi + echo "Cannot cross compile to $CPP_CROSS_COMPILE" + exit 3 fi ;; diff --git a/bin/controller/Makefile b/bin/controller/Makefile index 9fb72d23b6..758fa8ce07 100644 --- a/bin/controller/Makefile +++ b/bin/controller/Makefile @@ -27,7 +27,7 @@ SRCS= \ ifneq ($(PLIST_FILE),) all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh $(MKDIR) $(INSTALL_DIR) - $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) $(notdir $(TARGET)) > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE)) + $(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" true > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE)) endif include $(CPP_SRC_HOME)/mk/stdapp.mk diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh index 634a4b8b3e..0831dd9c8b 100755 --- a/dev-tools/strip_binaries.sh +++ b/dev-tools/strip_binaries.sh @@ -7,20 +7,24 @@ # Strips ML native code binaries to make them smaller before distribution. +ML_APP_NAME=controller case `uname` in Darwin) - EXE_DIR=controller.app/Contents/MacOS - DYNAMIC_LIB_DIR=controller.app/Contents/lib + EXE_DIR="$ML_APP_NAME.app/Contents/MacOS" + DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib" ;; Linux) if [ -z "$CPP_CROSS_COMPILE" ] ; then EXE_DIR=bin DYNAMIC_LIB_DIR=lib + elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + EXE_DIR="$ML_APP_NAME.app/Contents/MacOS" + DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib" else - EXE_DIR=controller.app/Contents/MacOS - DYNAMIC_LIB_DIR=controller.app/Contents/lib + echo "Cannot cross compile to $CPP_CROSS_COMPILE" + exit 1 fi ;; @@ -29,19 +33,19 @@ esac # Ensure $CPP_PLATFORM_HOME is set if [ -z "$CPP_PLATFORM_HOME" ] ; then echo '$CPP_PLATFORM_HOME is not set' - exit 1 + exit 2 fi # Ensure the executable programs folder has been created. if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist" - exit 2 + exit 3 fi # Ensure the lib folder has been created. if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist" - exit 3 + exit 4 fi cd "$CPP_PLATFORM_HOME" @@ -85,27 +89,25 @@ case `uname` in strip --strip-unneeded $LIBRARY objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY" done + elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` + do + echo "Stripping $PROGRAM" + llvm-dsymutil-3.9 $PROGRAM + /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM + done + for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` + do + echo "Stripping $LIBRARY" + case $LIBRARY in + *Ml*) + llvm-dsymutil-3.9 $LIBRARY + esac + /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY + done else - if [ "$CPP_CROSS_COMPILE" = macosx ] ; then - for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` - do - echo "Stripping $PROGRAM" - llvm-dsymutil-3.9 $PROGRAM - /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM - done - for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` - do - echo "Stripping $LIBRARY" - case $LIBRARY in - *Ml*) - llvm-dsymutil-3.9 $LIBRARY - esac - /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY - done - else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 4 - fi + echo "Cannot cross compile to $CPP_CROSS_COMPILE" + exit 5 fi ;; diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 3f1ca56001..7f20129633 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -37,6 +37,13 @@ For large data sets this change was observed to give a 10% to 20% decrease in train time. (See {ml-pull}622[#622].) * Upgrade Boost libraries to version 1.71. (See {ml-pull}638[#638].) +== {es} version 7.4.1 + +=== Enhancements + +* The {ml} native processes are now arranged in a .app directory structure on + macOS, to allow for notarization on macOS Catalina. (See {ml-pull}593[#593].) + == {es} version 7.4.0 === Bug Fixes diff --git a/mk/defines.mk b/mk/defines.mk index ff3f233247..f173463ee3 100644 --- a/mk/defines.mk +++ b/mk/defines.mk @@ -42,12 +42,7 @@ ifeq ($(OS),$(linuxOS)) ifdef CPP_CROSS_COMPILE include $(CPP_SRC_HOME)/mk/linux_crosscompile_$(CPP_CROSS_COMPILE).mk else -MUSL:=$(shell ldd --version 2>&1 | grep musl) -ifeq ($(MUSL),) include $(CPP_SRC_HOME)/mk/linux.mk -else -include $(CPP_SRC_HOME)/mk/linux-musl.mk -endif endif endif diff --git a/mk/linux_crosscompile_macosx.mk b/mk/linux_crosscompile_macosx.mk index 9aab09aca2..99da66041e 100644 --- a/mk/linux_crosscompile_macosx.mk +++ b/mk/linux_crosscompile_macosx.mk @@ -7,7 +7,8 @@ OS=MacOSX CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/darwin-x86_64 -APP_CONTENTS=controller.app/Contents +ML_APP_NAME=controller +APP_CONTENTS=$(ML_APP_NAME).app/Contents CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.11 SYSROOT=/usr/local/sysroot-$(CROSS_TARGET_PLATFORM) diff --git a/mk/macosx.mk b/mk/macosx.mk index 1db98f7def..0c8f22a688 100644 --- a/mk/macosx.mk +++ b/mk/macosx.mk @@ -7,7 +7,8 @@ OS=MacOSX CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/darwin-x86_64 -APP_CONTENTS=controller.app/Contents +ML_APP_NAME=controller +APP_CONTENTS=$(ML_APP_NAME).app/Contents CC=clang CXX=clang++ -std=c++14 -stdlib=libc++ diff --git a/mk/make_info_plist.sh b/mk/make_info_plist.sh index 21c9183409..79b0676ed0 100755 --- a/mk/make_info_plist.sh +++ b/mk/make_info_plist.sh @@ -1,9 +1,7 @@ #!/bin/bash ML_VERSION_NUM=`cat $CPP_SRC_HOME/gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $2 }' | xargs echo | sed 's/-.*//'` -ML_TARGET=$1 -ML_SHORTENED_TARGET=`echo $ML_TARGET | cut -c1-15` -ML_URL_TARGET=`echo $ML_TARGET | tr _ - | tr -d -C A-Za-z0-9- | cut -c1-63` +ML_APP_NAME=$1 cat < @@ -15,21 +13,21 @@ cat <CFBundlePackageType APPL CFBundleIdentifier - co.elastic.ml-cpp.$ML_URL_TARGET + co.elastic.ml-cpp.$ML_APP_NAME CFBundleInfoDictionaryVersion 6.0 CFBundleName - $ML_SHORTENED_TARGET + $ML_APP_NAME CFBundleDisplayName - $ML_TARGET + $ML_APP_NAME CFBundleVersion $ML_VERSION_NUM EOF -if [ $# -gt 1 ] ; then +if [ "x$2" = xtrue ] ; then cat <CFBundleExecutable - $2 + $ML_APP_NAME EOF fi diff --git a/mk/rules.mk b/mk/rules.mk index d591aba10c..7093dd3e22 100644 --- a/mk/rules.mk +++ b/mk/rules.mk @@ -48,7 +48,7 @@ $(OBJS_DIR)/%$(OBJECT_FILE_EXT): %.c $(CC) -c $(COMP_OUT_FLAG)$@ $(CFLAGS) $(PICFLAGS) $(PDB_FLAGS) $(CPPFLAGS) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< $(OBJS_DIR)/%.plist: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh - $(CPP_SRC_HOME)/mk/make_info_plist.sh $(notdir $(TARGET)) > $@ + $(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" false > $@ $(OBJS_DIR)/%.xml: %.cc $(CXX) $(ANALYZE_OUT_FLAG)$@ $(ANALYZEFLAGS) $(filter-out -DNDEBUG, $(CPPFLAGS)) $(if $(MAKE_PREFIX_SRC_PATH), $(shell pwd)/)$< diff --git a/set_env.sh b/set_env.sh index 601965d095..edc9a92427 100755 --- a/set_env.sh +++ b/set_env.sh @@ -28,21 +28,14 @@ case `uname` in ;; Linux) - if ldd --version 2>&1 | grep musl > /dev/null ; then - SIMPLE_PLATFORM=linux-musl - BUNDLE_PLATFORM=linux-musl-x86_64 + SIMPLE_PLATFORM=linux + if [ -z "$CPP_CROSS_COMPILE" ] ; then + BUNDLE_PLATFORM=linux-x86_64 + elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + BUNDLE_PLATFORM=darwin-x86_64 else - SIMPLE_PLATFORM=linux - if [ -z "$CPP_CROSS_COMPILE" ] ; then - BUNDLE_PLATFORM=linux-x86_64 - else - if [ "$CPP_CROSS_COMPILE" = macosx ] ; then - BUNDLE_PLATFORM=darwin-x86_64 - else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 1 - fi - fi + echo "Cannot cross compile to $CPP_CROSS_COMPILE" + exit 1 fi ;;