[ML] Make the ML native code more ".app"-like on macOS (#593)

Successful notarization and acceptance by Gatekeeper on macOS
Catalina requires that the ML programs and dynamic libraries be
arranged into the directory structure used by macOS apps.

This functionality may be required for notarization on
macOS Catalina.

* Avoid clashing use of .plist files for static analysis output

Static analysis files are now named *.xml (after all .plist
files really are .xml files)

* Add CFBundlePackageType = APPL to Info.plist

Also make sure the identifier only uses valid characters

* Change macOS directory structure

* Switch CPP_CROSS_COMPILE back to macosx

* Fix resource dir and include top level Info.plist with XML header

* Oops

* Fix problem with ordering of static initialization

* Now we're shipping an app, use the app name in all Info.plist entries

When this PR started out we thought we'd be shipping a number
of standalone "command line tools".  But that didn't work with
notarization, so we had to make a more radical change, to become
an app.

(I also removed some dead code related to detecting Linux MUSL.)

Author: David Roberts

3rd_party/3rd_party.sh

@@ -59,21 +59,19 @@ case `uname` in
             STL_PREFIX=libstdc++
             STL_EXTENSION=.so.6
             ZLIB_LOCATION=
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
+            BOOST_LOCATION=$SYSROOT/usr/local/lib
+            BOOST_COMPILER=clang
+            BOOST_EXTENSION=mt-x64-1_71.dylib
+            BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread'
+            XML_LOCATION=
+            GCC_RT_LOCATION=
+            STL_LOCATION=
+            ZLIB_LOCATION=
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
-                BOOST_LOCATION=$SYSROOT/usr/local/lib
-                BOOST_COMPILER=clang
-                BOOST_EXTENSION=mt-x64-1_71.dylib
-                BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread'
-                XML_LOCATION=
-                GCC_RT_LOCATION=
-                STL_LOCATION=
-                ZLIB_LOCATION=
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 3
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 3
         fi
         ;;
 

bin/autoconfig/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autoconfig$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) $(LIB_ML_CONFIG)
 

bin/autodetect/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autodetect$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/categorize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/categorize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/controller/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/controller$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE)
 
@@ -16,14 +16,19 @@ USE_BOOST_PROGRAMOPTIONS_LIBS=1
 
 LIBS=$(ML_LIBS)
 
-all: build
+all:: build
 
 SRCS= \
     Main.cc \
     CBlockingCallCancellerThread.cc \
     CCmdLineParser.cc \
     CCommandProcessor.cc \
 
+ifneq ($(PLIST_FILE),)
+all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh
+	$(MKDIR) $(INSTALL_DIR)
+	$(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" true > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE))
+endif
 
 include $(CPP_SRC_HOME)/mk/stdapp.mk
 

bin/data_frame_analyzer/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/data_frame_analyzer$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/normalize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/normalize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

dev-tools/strip_binaries.sh

@@ -5,24 +5,47 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-# Strips Ml native code binaries to make them smaller before distribution.
+# Strips ML native code binaries to make them smaller before distribution.
+
+ML_APP_NAME=controller
+case `uname` in
+
+    Darwin)
+        EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+        DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        ;;
+
+    Linux)
+        if [ -z "$CPP_CROSS_COMPILE" ] ; then
+            EXE_DIR=bin
+            DYNAMIC_LIB_DIR=lib
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+            DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        else
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 1
+        fi
+        ;;
+
+esac
 
 # Ensure $CPP_PLATFORM_HOME is set
 if [ -z "$CPP_PLATFORM_HOME" ] ; then
     echo '$CPP_PLATFORM_HOME is not set'
-    exit 1
+    exit 2
 fi
 
-# Ensure the bin folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/bin" ] ; then
-    echo '$CPP_PLATFORM_HOME/bin does not exist'
-    exit 2
+# Ensure the executable programs folder has been created.
+if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist"
+    exit 3
 fi
 
 # Ensure the lib folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/lib" ] ; then
-    echo '$CPP_PLATFORM_HOME/lib does not exist'
-    exit 3
+if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist"
+    exit 4
 fi
 
 cd "$CPP_PLATFORM_HOME"
@@ -32,13 +55,13 @@ cd "$CPP_PLATFORM_HOME"
 case `uname` in
 
     Darwin)
-        for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
+        for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $PROGRAM"
             dsymutil $PROGRAM
             strip -u -r $PROGRAM
         done
-        for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
+        for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $LIBRARY"
             case $LIBRARY in
@@ -51,42 +74,40 @@ case `uname` in
 
     Linux)
         if [ -z "$CPP_CROSS_COMPILE" ] ; then
-            for PROGRAM in `ls -1 bin/* | egrep -v 'bin/core|-debug$'`
+            for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`
             do
                 echo "Stripping $PROGRAM"
                 objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug"
                 strip --strip-all $PROGRAM
                 objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM"
                 chmod -x "$PROGRAM-debug"
             done
-            for LIBRARY in `ls -1 lib/* | egrep -v 'lib/core|-debug$'`
+            for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'`
             do
                 echo "Stripping $LIBRARY"
                 objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug"
                 strip --strip-unneeded $LIBRARY
                 objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY"
             done
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $PROGRAM"
+                llvm-dsymutil-3.9 $PROGRAM
+                /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
+            done
+            for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $LIBRARY"
+                case $LIBRARY in
+                    *Ml*)
+                        llvm-dsymutil-3.9 $LIBRARY
+                esac
+                /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
+            done
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $PROGRAM"
-                    llvm-dsymutil-3.9 $PROGRAM
-                    /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
-                done
-                for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $LIBRARY"
-                    case $LIBRARY in
-                        *Ml*)
-                            llvm-dsymutil-3.9 $LIBRARY
-                    esac
-                    /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
-                done
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 4
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 5
         fi
         ;;
 

devbin/analyze_test/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=analyze_test$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/devbin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 all: build
 

docs/CHANGELOG.asciidoc

@@ -41,6 +41,11 @@ find lower loss models faster. (See {ml-pull}686[#686].)
 
 == {es} version 7.4.1
 
+=== Enhancements
+
+* The {ml} native processes are now arranged in a .app directory structure on
+  macOS, to allow for notarization on macOS Catalina. (See {ml-pull}593[#593].)
+
 === Bug Fixes
 
 * A reference to a temporary variable was causing forecast model restoration to fail.