[ML] Make the ML native code more ".app"-like on macOS

3rd_party/3rd_party.sh

@@ -59,21 +59,19 @@ case `uname` in
             STL_PREFIX=libstdc++
             STL_EXTENSION=.so.6
             ZLIB_LOCATION=
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
+            BOOST_LOCATION=$SYSROOT/usr/local/lib
+            BOOST_COMPILER=clang
+            BOOST_EXTENSION=mt-x64-1_71.dylib
+            BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread'
+            XML_LOCATION=
+            GCC_RT_LOCATION=
+            STL_LOCATION=
+            ZLIB_LOCATION=
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
-                BOOST_LOCATION=$SYSROOT/usr/local/lib
-                BOOST_COMPILER=clang
-                BOOST_EXTENSION=mt-x64-1_71.dylib
-                BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread'
-                XML_LOCATION=
-                GCC_RT_LOCATION=
-                STL_LOCATION=
-                ZLIB_LOCATION=
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 3
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 3
         fi
         ;;
 

bin/autoconfig/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autoconfig$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) $(LIB_ML_CONFIG)
 

bin/autodetect/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autodetect$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/categorize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/categorize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/controller/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/controller$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE)
 
@@ -16,14 +16,19 @@ USE_BOOST_PROGRAMOPTIONS_LIBS=1
 
 LIBS=$(ML_LIBS)
 
-all: build
+all:: build
 
 SRCS= \
     Main.cc \
     CBlockingCallCancellerThread.cc \
     CCmdLineParser.cc \
     CCommandProcessor.cc \
 
+ifneq ($(PLIST_FILE),)
+all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh
+	$(MKDIR) $(INSTALL_DIR)
+	$(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" true > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE))
+endif
 
 include $(CPP_SRC_HOME)/mk/stdapp.mk
 

bin/data_frame_analyzer/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/data_frame_analyzer$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/normalize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/normalize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

dev-tools/strip_binaries.sh

@@ -5,24 +5,47 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-# Strips Ml native code binaries to make them smaller before distribution.
+# Strips ML native code binaries to make them smaller before distribution.
+
+ML_APP_NAME=controller
+case `uname` in
+
+    Darwin)
+        EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+        DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        ;;
+
+    Linux)
+        if [ -z "$CPP_CROSS_COMPILE" ] ; then
+            EXE_DIR=bin
+            DYNAMIC_LIB_DIR=lib
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+            DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        else
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 1
+        fi
+        ;;
+
+esac
 
 # Ensure $CPP_PLATFORM_HOME is set
 if [ -z "$CPP_PLATFORM_HOME" ] ; then
     echo '$CPP_PLATFORM_HOME is not set'
-    exit 1
+    exit 2
 fi
 
-# Ensure the bin folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/bin" ] ; then
-    echo '$CPP_PLATFORM_HOME/bin does not exist'
-    exit 2
+# Ensure the executable programs folder has been created.
+if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist"
+    exit 3
 fi
 
 # Ensure the lib folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/lib" ] ; then
-    echo '$CPP_PLATFORM_HOME/lib does not exist'
-    exit 3
+if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist"
+    exit 4
 fi
 
 cd "$CPP_PLATFORM_HOME"
@@ -32,13 +55,13 @@ cd "$CPP_PLATFORM_HOME"
 case `uname` in
 
     Darwin)
-        for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
+        for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $PROGRAM"
             dsymutil $PROGRAM
             strip -u -r $PROGRAM
         done
-        for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
+        for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $LIBRARY"
             case $LIBRARY in
@@ -51,42 +74,40 @@ case `uname` in
 
     Linux)
         if [ -z "$CPP_CROSS_COMPILE" ] ; then
-            for PROGRAM in `ls -1 bin/* | egrep -v 'bin/core|-debug$'`
+            for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`
             do
                 echo "Stripping $PROGRAM"
                 objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug"
                 strip --strip-all $PROGRAM
                 objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM"
                 chmod -x "$PROGRAM-debug"
             done
-            for LIBRARY in `ls -1 lib/* | egrep -v 'lib/core|-debug$'`
+            for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'`
             do
                 echo "Stripping $LIBRARY"
                 objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug"
                 strip --strip-unneeded $LIBRARY
                 objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY"
             done
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $PROGRAM"
+                llvm-dsymutil-3.9 $PROGRAM
+                /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
+            done
+            for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $LIBRARY"
+                case $LIBRARY in
+                    *Ml*)
+                        llvm-dsymutil-3.9 $LIBRARY
+                esac
+                /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
+            done
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $PROGRAM"
-                    llvm-dsymutil-3.9 $PROGRAM
-                    /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
-                done
-                for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $LIBRARY"
-                    case $LIBRARY in
-                        *Ml*)
-                            llvm-dsymutil-3.9 $LIBRARY
-                    esac
-                    /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
-                done
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 4
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 5
         fi
         ;;
 

devbin/analyze_test/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=analyze_test$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/devbin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 all: build
 

docs/CHANGELOG.asciidoc

@@ -39,6 +39,11 @@ train time. (See {ml-pull}622[#622].)
 
 == {es} version 7.4.1
 
+=== Enhancements
+
+* The {ml} native processes are now arranged in a .app directory structure on
+  macOS, to allow for notarization on macOS Catalina. (See {ml-pull}593[#593].)
+
 === Bug Fixes
 
 * A reference to a temporary variable was causing forecast model restoration to fail.

include/core/CResourceLocator.h

@@ -36,9 +36,6 @@ class CORE_EXPORT CResourceLocator : private CNonInstantiatable {
     //! details and dictionary words.
     static std::string resourceDir();
 
-    //! Get the directory that stores log files.
-    static std::string logDir();
-
     //! Get the root directory for the C++ section of the source tree.
     //! (Obviously this should only be used in test code!)
     static std::string cppRootDir();

lib/core/CResourceLocator.cc

@@ -8,28 +8,36 @@
 #include <core/COsFileFuncs.h>
 #include <core/CProgName.h>
 
-#include <stdlib.h>
+#include <cstdlib>
 
 namespace {
-const char* const CPP_SRC_HOME("CPP_SRC_HOME");
+// Important: all file scope in this file must be simple types that don't
+// require construction, as they could be accessed before the static
+// constructors are run.
+const char* const CPP_SRC_HOME{"CPP_SRC_HOME"};
+
+#ifdef MacOSX
+const char* const RESOURCE_RELATIVE_DIR{"../Resources"};
+#else
+const char* const RESOURCE_RELATIVE_DIR{"../resources"};
+#endif
 }
 
 namespace ml {
 namespace core {
 
 std::string CResourceLocator::resourceDir() {
-    // Look relative to the program that's running, assuming this directory layout:
-    // $ES_HOME/plugin/<plugin name>/resources
-    // $ES_HOME/plugin/<plugin name>/platform/<platform name>/bin
 
-    std::string productionDir(CProgName::progDir() + "/../../../resources");
+    // Look relative to the program that's running, assuming that the resource
+    // directory is located relative to the directory the current program is in.
+    std::string productionDir(CProgName::progDir() + RESOURCE_RELATIVE_DIR);
 
     // If the production directory doesn't exist, return the dev directory if
     // that does, but if neither exist return the production directory so the
     // error message is nicer for the end user.
     COsFileFuncs::TStat buf;
     if (COsFileFuncs::stat(productionDir.c_str(), &buf) != 0) {
-        const char* cppSrcHome(::getenv(CPP_SRC_HOME));
+        const char* cppSrcHome(std::getenv(CPP_SRC_HOME));
         if (cppSrcHome != nullptr) {
             std::string devDir(cppSrcHome);
             devDir += "/lib/core";
@@ -42,24 +50,8 @@ std::string CResourceLocator::resourceDir() {
     return productionDir;
 }
 
-std::string CResourceLocator::logDir() {
-    // Look relative to the program that's running, assuming this directory layout:
-    // $ES_HOME/logs
-    // $ES_HOME/plugin/<plugin name>/platform/<platform name>/bin
-
-    std::string productionDir(CProgName::progDir() + "/../../../../../logs");
-
-    COsFileFuncs::TStat buf;
-    if (COsFileFuncs::stat(productionDir.c_str(), &buf) != 0) {
-        // Assume we're running as a unit test
-        return ".";
-    }
-
-    return productionDir;
-}
-
 std::string CResourceLocator::cppRootDir() {
-    const char* cppSrcHome(::getenv(CPP_SRC_HOME));
+    const char* cppSrcHome(std::getenv(CPP_SRC_HOME));
     if (cppSrcHome == nullptr) {
         // Assume we're in a unittest directory
         return "../../..";

lib/core/Makefile

@@ -21,7 +21,7 @@ USE_STRPTIME=1
 
 CPPFLAGS+= -DDYNAMIC_LIB_EXT=$(DYNAMIC_LIB_EXT)
 
-CONF_INSTALL_DIR=$(CPP_DISTRIBUTION_HOME)/resources
+CONF_INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(RESOURCES_DIR)
 
 # On Windows we need to copy over the Boost date/time config file that lists
 # details of each timezone, whereas on Unix timezone config is part of the

lib/core/unittest/CResourceLocatorTest.cc

@@ -14,8 +14,6 @@ CppUnit::Test* CResourceLocatorTest::suite() {
 
     suiteOfTests->addTest(new CppUnit::TestCaller<CResourceLocatorTest>(
         "CResourceLocatorTest::testResourceDir", &CResourceLocatorTest::testResourceDir));
-    suiteOfTests->addTest(new CppUnit::TestCaller<CResourceLocatorTest>(
-        "CResourceLocatorTest::testLogDir", &CResourceLocatorTest::testLogDir));
     suiteOfTests->addTest(new CppUnit::TestCaller<CResourceLocatorTest>(
         "CResourceLocatorTest::testSrcRootDir", &CResourceLocatorTest::testSrcRootDir));
 
@@ -32,14 +30,6 @@ void CResourceLocatorTest::testResourceDir() {
         0, ml::core::COsFileFuncs::stat((resourceDir + "/ml-en.dict").c_str(), &buf));
 }
 
-void CResourceLocatorTest::testLogDir() {
-    std::string logDir(ml::core::CResourceLocator::logDir());
-    LOG_DEBUG(<< "Log directory is " << logDir);
-
-    // Don't assert on this as it will be non-essential once
-    // we're an Elasticsearch plugin
-}
-
 void CResourceLocatorTest::testSrcRootDir() {
     std::string cppRootDir(ml::core::CResourceLocator::cppRootDir());
     LOG_DEBUG(<< "C++ root directory is " << cppRootDir);

lib/core/unittest/CResourceLocatorTest.h

@@ -11,7 +11,6 @@
 class CResourceLocatorTest : public CppUnit::TestFixture {
 public:
     void testResourceDir();
-    void testLogDir();
     void testSrcRootDir();
 
     static CppUnit::Test* suite();

mk/defines.mk

@@ -42,12 +42,7 @@ ifeq ($(OS),$(linuxOS))
 ifdef CPP_CROSS_COMPILE
 include $(CPP_SRC_HOME)/mk/linux_crosscompile_$(CPP_CROSS_COMPILE).mk
 else
-MUSL:=$(shell ldd --version 2>&1 | grep musl)
-ifeq ($(MUSL),)
 include $(CPP_SRC_HOME)/mk/linux.mk
-else
-include $(CPP_SRC_HOME)/mk/linux-musl.mk
-endif
 endif
 endif