Skip to content

Commit 973380b

Browse files
author
David Roberts
authored
[7.4][ML] Make the ML native code more ".app"-like on macOS (#704)
Successful notarization and acceptance by Gatekeeper on macOS Catalina requires that the ML programs and dynamic libraries be arranged into the directory structure used by macOS apps. Backport of #593
1 parent 3e5f87b commit 973380b

31 files changed

+255
-252
lines changed

‎3rd_party/3rd_party.sh‎

Lines changed: 36 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -52,65 +52,43 @@ case `uname` in
5252

5353
Linux)
5454
if [ -z "$CPP_CROSS_COMPILE" ] ; then
55-
ldd --version 2>&1 | grep musl > /dev/null
56-
if [ $? -ne 0 ] ; then
57-
APR_LOCATION=/usr/local/gcc73/lib
58-
APR_EXTENSION=1.so.0
59-
BOOST_LOCATION=/usr/local/gcc73/lib
60-
BOOST_COMPILER=gcc
61-
BOOST_EXTENSION=mt-1_65_1.so.1.65.1
62-
BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
63-
LOG4CXX_LOCATION=/usr/local/gcc73/lib
64-
LOG4CXX_EXTENSION=.so.10
65-
XML_LOCATION=/usr/local/gcc73/lib
66-
XML_EXTENSION=.so.2
67-
EXPAT_LOCATION=/usr/local/gcc73/lib
68-
EXPAT_EXTENSION=.so.1
69-
GCC_RT_LOCATION=/usr/local/gcc73/lib64
70-
GCC_RT_EXTENSION=.so.1
71-
STL_LOCATION=/usr/local/gcc73/lib64
72-
STL_PREFIX=libstdc++
73-
STL_EXTENSION=.so.6
74-
ATOMIC_LOCATION=
75-
ZLIB_LOCATION=
76-
else
77-
APR_LOCATION=/usr/local/lib
78-
APR_EXTENSION=1.so.0
79-
BOOST_LOCATION=/usr/local/lib
80-
BOOST_COMPILER=gcc
81-
BOOST_EXTENSION=mt-1_65_1.so.1.65.1
82-
BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
83-
LOG4CXX_LOCATION=/usr/local/lib
84-
LOG4CXX_EXTENSION=.so.10
85-
XML_LOCATION=/usr/local/lib
86-
XML_EXTENSION=.so.2
87-
EXPAT_LOCATION=/usr/local/lib
88-
EXPAT_EXTENSION=.so.1
89-
GCC_RT_LOCATION=
90-
STL_LOCATION=
91-
ATOMIC_LOCATION=
92-
ZLIB_LOCATION=
93-
fi
55+
APR_LOCATION=/usr/local/gcc73/lib
56+
APR_EXTENSION=1.so.0
57+
BOOST_LOCATION=/usr/local/gcc73/lib
58+
BOOST_COMPILER=gcc
59+
BOOST_EXTENSION=mt-1_65_1.so.1.65.1
60+
BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
61+
LOG4CXX_LOCATION=/usr/local/gcc73/lib
62+
LOG4CXX_EXTENSION=.so.10
63+
XML_LOCATION=/usr/local/gcc73/lib
64+
XML_EXTENSION=.so.2
65+
EXPAT_LOCATION=/usr/local/gcc73/lib
66+
EXPAT_EXTENSION=.so.1
67+
GCC_RT_LOCATION=/usr/local/gcc73/lib64
68+
GCC_RT_EXTENSION=.so.1
69+
STL_LOCATION=/usr/local/gcc73/lib64
70+
STL_PREFIX=libstdc++
71+
STL_EXTENSION=.so.6
72+
ATOMIC_LOCATION=
73+
ZLIB_LOCATION=
74+
elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
75+
SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
76+
APR_LOCATION=
77+
BOOST_LOCATION=$SYSROOT/usr/local/lib
78+
BOOST_COMPILER=clang
79+
BOOST_EXTENSION=mt-1_65_1.dylib
80+
BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
81+
LOG4CXX_LOCATION=$SYSROOT/usr/local/lib
82+
LOG4CXX_EXTENSION=.10.dylib
83+
XML_LOCATION=
84+
EXPAT_LOCATION=
85+
GCC_RT_LOCATION=
86+
STL_LOCATION=
87+
ATOMIC_LOCATION=
88+
ZLIB_LOCATION=
9489
else
95-
if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
96-
SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11
97-
APR_LOCATION=
98-
BOOST_LOCATION=$SYSROOT/usr/local/lib
99-
BOOST_COMPILER=clang
100-
BOOST_EXTENSION=mt-1_65_1.dylib
101-
BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
102-
LOG4CXX_LOCATION=$SYSROOT/usr/local/lib
103-
LOG4CXX_EXTENSION=.10.dylib
104-
XML_LOCATION=
105-
EXPAT_LOCATION=
106-
GCC_RT_LOCATION=
107-
STL_LOCATION=
108-
ATOMIC_LOCATION=
109-
ZLIB_LOCATION=
110-
else
111-
echo "Cannot cross compile to $CPP_CROSS_COMPILE"
112-
exit 3
113-
fi
90+
echo "Cannot cross compile to $CPP_CROSS_COMPILE"
91+
exit 3
11492
fi
11593
;;
11694

@@ -292,27 +270,5 @@ case `uname` in
292270
fi
293271
;;
294272

295-
SunOS)
296-
if [ -n "$INSTALL_DIR" ] ; then
297-
cd "$INSTALL_DIR"
298-
for FILE in `find . -type f | egrep -v '^core|-debug$|libMl'`
299-
do
300-
# Replace RPATH for 3rd party libraries that already have one
301-
elfedit -r -e 'dyn:runpath' $FILE >/dev/null 2>&1 && chmod u+wx $FILE && elfedit -e 'dyn:runpath $ORIGIN/.' $FILE
302-
if [ $? -eq 0 ] ; then
303-
echo "Set RPATH in $FILE"
304-
else
305-
# Set RPATH for 3rd party libraries that reference other libraries we ship
306-
ldd $FILE | grep /usr/local/lib >/dev/null 2>&1 && chmod u+wx $FILE && elfedit -e 'dyn:runpath $ORIGIN/.' $FILE
307-
if [ $? -eq 0 ] ; then
308-
echo "Set RPATH in $FILE"
309-
else
310-
echo "Did not set RPATH in $FILE"
311-
fi
312-
fi
313-
done
314-
fi
315-
;;
316-
317273
esac
318274

‎bin/autoconfig/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/autoconfig$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) $(LIB_ML_CONFIG)
1313

‎bin/autodetect/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/autodetect$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
1313

‎bin/categorize/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/categorize$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
1313

‎bin/controller/Makefile‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/controller$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE)
1313

@@ -16,14 +16,19 @@ USE_BOOST_PROGRAMOPTIONS_LIBS=1
1616

1717
LIBS=$(ML_LIBS)
1818

19-
all: build
19+
all:: build
2020

2121
SRCS= \
2222
Main.cc \
2323
CBlockingCallCancellerThread.cc \
2424
CCmdLineParser.cc \
2525
CCommandProcessor.cc \
2626

27+
ifneq ($(PLIST_FILE),)
28+
all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh
29+
$(MKDIR) $(INSTALL_DIR)
30+
$(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" true > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE))
31+
endif
2732

2833
include $(CPP_SRC_HOME)/mk/stdapp.mk
2934

‎bin/data_frame_analyzer/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/data_frame_analyzer$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
1313

‎bin/normalize/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=$(OBJS_DIR)/normalize$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
1313

‎dev-tools/strip_binaries.sh‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,47 @@
55
# you may not use this file except in compliance with the Elastic License.
66
#
77

8-
# Strips Ml native code binaries to make them smaller before distribution.
8+
# Strips ML native code binaries to make them smaller before distribution.
9+
10+
ML_APP_NAME=controller
11+
case `uname` in
12+
13+
Darwin)
14+
EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
15+
DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
16+
;;
17+
18+
Linux)
19+
if [ -z "$CPP_CROSS_COMPILE" ] ; then
20+
EXE_DIR=bin
21+
DYNAMIC_LIB_DIR=lib
22+
elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
23+
EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
24+
DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
25+
else
26+
echo "Cannot cross compile to $CPP_CROSS_COMPILE"
27+
exit 1
28+
fi
29+
;;
30+
31+
esac
932

1033
# Ensure $CPP_PLATFORM_HOME is set
1134
if [ -z "$CPP_PLATFORM_HOME" ] ; then
1235
echo '$CPP_PLATFORM_HOME is not set'
13-
exit 1
36+
exit 2
1437
fi
1538

16-
# Ensure the bin folder has been created.
17-
if [ ! -d "$CPP_PLATFORM_HOME/bin" ] ; then
18-
echo '$CPP_PLATFORM_HOME/bin does not exist'
19-
exit 2
39+
# Ensure the executable programs folder has been created.
40+
if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then
41+
echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist"
42+
exit 3
2043
fi
2144

2245
# Ensure the lib folder has been created.
23-
if [ ! -d "$CPP_PLATFORM_HOME/lib" ] ; then
24-
echo '$CPP_PLATFORM_HOME/lib does not exist'
25-
exit 3
46+
if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then
47+
echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist"
48+
exit 4
2649
fi
2750

2851
cd "$CPP_PLATFORM_HOME"
@@ -32,13 +55,13 @@ cd "$CPP_PLATFORM_HOME"
3255
case `uname` in
3356

3457
Darwin)
35-
for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
58+
for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
3659
do
3760
echo "Stripping $PROGRAM"
3861
dsymutil $PROGRAM
3962
strip -u -r $PROGRAM
4063
done
41-
for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
64+
for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
4265
do
4366
echo "Stripping $LIBRARY"
4467
case $LIBRARY in
@@ -51,42 +74,40 @@ case `uname` in
5174

5275
Linux)
5376
if [ -z "$CPP_CROSS_COMPILE" ] ; then
54-
for PROGRAM in `ls -1 bin/* | egrep -v 'bin/core|-debug$'`
77+
for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`
5578
do
5679
echo "Stripping $PROGRAM"
5780
objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug"
5881
strip --strip-all $PROGRAM
5982
objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM"
6083
chmod -x "$PROGRAM-debug"
6184
done
62-
for LIBRARY in `ls -1 lib/* | egrep -v 'lib/core|-debug$'`
85+
for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'`
6386
do
6487
echo "Stripping $LIBRARY"
6588
objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug"
6689
strip --strip-unneeded $LIBRARY
6790
objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY"
6891
done
92+
elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
93+
for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
94+
do
95+
echo "Stripping $PROGRAM"
96+
llvm-dsymutil-3.9 $PROGRAM
97+
/usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
98+
done
99+
for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
100+
do
101+
echo "Stripping $LIBRARY"
102+
case $LIBRARY in
103+
*Ml*)
104+
llvm-dsymutil-3.9 $LIBRARY
105+
esac
106+
/usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
107+
done
69108
else
70-
if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
71-
for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
72-
do
73-
echo "Stripping $PROGRAM"
74-
llvm-dsymutil-3.9 $PROGRAM
75-
/usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM
76-
done
77-
for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
78-
do
79-
echo "Stripping $LIBRARY"
80-
case $LIBRARY in
81-
*Ml*)
82-
llvm-dsymutil-3.9 $LIBRARY
83-
esac
84-
/usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY
85-
done
86-
else
87-
echo "Cannot cross compile to $CPP_CROSS_COMPILE"
88-
exit 4
89-
fi
109+
echo "Cannot cross compile to $CPP_CROSS_COMPILE"
110+
exit 5
90111
fi
91112
;;
92113

‎devbin/analyze_test/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
77

88
TARGET=analyze_test$(EXE_EXT)
99

10-
INSTALL_DIR=$(CPP_PLATFORM_HOME)/devbin
10+
INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
1111

1212
all: build
1313

‎docs/CHANGELOG.asciidoc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ train time. (See {ml-pull}622[#622].)
3939

4040
== {es} version 7.4.1
4141

42+
=== Enhancements
43+
44+
* The {ml} native processes are now arranged in a .app directory structure on
45+
macOS, to allow for notarization on macOS Catalina. (See {ml-pull}593[#593].)
46+
4247
=== Bug Fixes
4348

4449
* A reference to a temporary variable was causing forecast model restoration to fail.

0 commit comments

Comments
 (0)