bpo-36721: Add --embed option to python-config (GH-13500)

To embed Python into an application, a new --embed option must be
passed to "python3-config --libs --embed" to get "-lpython3.8" (link
the application to libpython). To support both 3.8 and older, try
"python3-config --libs --embed" first and fallback to "python3-config
--libs" (without --embed) if the previous command fails.

Add a pkg-config "python-3.8-embed" module to embed Python into an
application: "pkg-config python-3.8-embed --libs" includes
"-lpython3.8".  To support both 3.8 and older, try "pkg-config
python-X.Y-embed --libs" first and fallback to "pkg-config python-X.Y
--libs" (without --embed) if the previous command fails (replace
"X.Y" with the Python version).

On the other hand, "pkg-config python3.8 --libs" no longer contains
"-lpython3.8". C extensions must not be linked to libpython (except
on Android, case handled by the script); this change is backward
incompatible on purpose.

"make install" now also installs "python-3.8-embed.pc".

Author: vstinner

.gitignore

@@ -36,6 +36,7 @@ Lib/test/data/*
 Makefile
 Makefile.pre
 Misc/python.pc
+Misc/python-embed.pc
 Misc/python-config.sh
 Modules/Setup
 Modules/Setup.config

Doc/whatsnew/3.8.rst

@@ -149,6 +149,24 @@ extensions compiled in release mode and for C extensions compiled with the
 stable ABI.
 (Contributed by Victor Stinner in :issue:`36722`.)
 
+To embed Python into an application, a new ``--embed`` option must be passed to
+``python3-config --libs --embed`` to get ``-lpython3.8`` (link the application
+to libpython). To support both 3.8 and older, try ``python3-config --libs
+--embed`` first and fallback to ``python3-config --libs`` (without ``--embed``)
+if the previous command fails.
+
+Add a pkg-config ``python-3.8-embed`` module to embed Python into an
+application: ``pkg-config python-3.8-embed --libs`` includes ``-lpython3.8``.
+To support both 3.8 and older, try ``pkg-config python-X.Y-embed --libs`` first
+and fallback to ``pkg-config python-X.Y --libs`` (without ``--embed``) if the
+previous command fails (replace ``X.Y`` with the Python version).
+
+On the other hand, ``pkg-config python3.8 --libs`` no longer contains
+``-lpython3.8``. C extensions must not be linked to libpython (except on
+Android, case handled by the script); this change is backward incompatible on
+purpose.
+(Contributed by Victor Stinner in :issue:`36721`.)
+
 f-strings now support =  for quick and easy debugging
 -----------------------------------------------------
 

Makefile.pre.in

@@ -1258,11 +1258,15 @@ bininstall: altbininstall
 		(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \
 		rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc; \
 		(cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python-$(LDVERSION).pc); \
+		rm -f $(DESTDIR)$(LIBPC)/python-embed-$(LDVERSION).pc; \
+		(cd $(DESTDIR)$(LIBPC); $(LN) -s python-embed-$(VERSION).pc python-embed-$(LDVERSION).pc); \
 	fi
 	-rm -f $(DESTDIR)$(BINDIR)/python3-config
 	(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config)
 	-rm -f $(DESTDIR)$(LIBPC)/python3.pc
 	(cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python3.pc)
+	-rm -f $(DESTDIR)$(LIBPC)/python3-embed.pc
+	(cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION)-embed.pc python3-embed.pc)
 	-rm -f $(DESTDIR)$(BINDIR)/idle3
 	(cd $(DESTDIR)$(BINDIR); $(LN) -s idle$(VERSION) idle3)
 	-rm -f $(DESTDIR)$(BINDIR)/pydoc3
@@ -1552,6 +1556,7 @@ libainstall:	@DEF_MAKE_RULE@ python-config
 	$(INSTALL_DATA) $(srcdir)/Modules/Setup $(DESTDIR)$(LIBPL)/Setup
 	$(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local
 	$(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc
+	$(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc
 	$(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup
 	$(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh
 	$(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py
@@ -1766,7 +1771,7 @@ distclean: clobber
 	done
 	-rm -f core Makefile Makefile.pre config.status Modules/Setup.local \
 		Modules/ld_so_aix Modules/python.exp Misc/python.pc \
-		Misc/python-config.sh
+		Misc/python-embed.pc Misc/python-config.sh
 	-rm -f python*-gdb.py
 	# Issue #28258: set LC_ALL to avoid issues with Estonian locale.
 	# Expansion is performed here by shell (spawned by make) itself before

Misc/NEWS.d/next/Build/2019-05-22-16-19-18.bpo-36721.9aRwfZ.rst

@@ -0,0 +1,16 @@
+To embed Python into an application, a new ``--embed`` option must be passed to
+``python3-config --libs --embed`` to get ``-lpython3.8`` (link the application
+to libpython). To support both 3.8 and older, try ``python3-config --libs
+--embed`` first and fallback to ``python3-config --libs`` (without ``--embed``)
+if the previous command fails.
+
+Add a pkg-config ``python-3.8-embed`` module to embed Python into an
+application: ``pkg-config python-3.8-embed --libs`` includes ``-lpython3.8``.
+To support both 3.8 and older, try ``pkg-config python-X.Y-embed --libs`` first
+and fallback to ``pkg-config python-X.Y --libs`` (without ``--embed``) if the
+previous command fails (replace ``X.Y`` with the Python version).
+
+On the other hand, ``pkg-config python3.8 --libs`` no longer contains
+``-lpython3.8``. C extensions must not be linked to libpython (except on
+Android, case handled by the script); this change is backward incompatible on
+purpose.

Misc/python-config.in

@@ -9,7 +9,8 @@ import sys
 import sysconfig
 
 valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags',
-              'ldflags', 'extension-suffix', 'help', 'abiflags', 'configdir']
+              'ldflags', 'extension-suffix', 'help', 'abiflags', 'configdir',
+              'embed']
 
 def exit_with_usage(code=1):
     print("Usage: {0} [{1}]".format(
@@ -47,8 +48,13 @@ for opt in opt_flags:
         print(' '.join(flags))
 
     elif opt in ('--libs', '--ldflags'):
-        libpython = getvar('LIBPYTHON')
-        libs = [libpython] if libpython else []
+        libs = []
+        if '--embed' in opt_flags:
+            libs.append('-lpython' + pyver + sys.abiflags)
+        else:
+            libpython = getvar('LIBPYTHON')
+            if libpython:
+                libs.append(libpython)
         libs.extend(getvar('LIBS').split() + getvar('SYSLIBS').split())
 
         # add the prefix/lib/pythonX.Y/config dir, but only if there is no

Misc/python-config.sh.in

@@ -42,6 +42,7 @@ LIBC="@LIBC@"
 SYSLIBS="$LIBM $LIBC"
 ABIFLAGS="@ABIFLAGS@"
 LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS"
+LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS"
 BASECFLAGS="@BASECFLAGS@"
 LDLIBRARY="@LDLIBRARY@"
 OPT="@OPT@"
@@ -53,6 +54,7 @@ SO="@EXT_SUFFIX@"
 PYTHONFRAMEWORK="@PYTHONFRAMEWORK@"
 INCDIR="-I$includedir/python${VERSION}${ABIFLAGS}"
 PLATINCDIR="-I$includedir/python${VERSION}${ABIFLAGS}"
+PY_EMBED=0
 
 # Scan for --help or unknown argument.
 for ARG in $*
@@ -61,6 +63,9 @@ do
         --help)
             exit_with_usage 0
         ;;
+        --embed)
+            PY_EMBED=1
+        ;;
         --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir)
         ;;
         *)
@@ -69,6 +74,10 @@ do
     esac
 done
 
+if [ $PY_EMBED = 1 ] ; then
+    LIBS="$LIBS_EMBED"
+fi
+
 for ARG in "$@"
 do
     case "$ARG" in

Misc/python-embed.pc.in

@@ -0,0 +1,13 @@
+# See: man pkg-config
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Python
+Description: Embed Python into an application
+Requires:
+Version: @VERSION@
+Libs.private: @LIBS@
+Libs: -L${libdir} -lpython@VERSION@@ABIFLAGS@
+Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@

Misc/python.pc.in

@@ -5,9 +5,9 @@ libdir=@libdir@
 includedir=@includedir@
 
 Name: Python
-Description: Python library
-Requires: 
+Description: Build a C extension for Python
+Requires:
 Version: @VERSION@
 Libs.private: @LIBS@
-Libs: -L${libdir} -lpython@VERSION@@ABIFLAGS@
+Libs:
 Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@

configure

@@ -17315,7 +17315,7 @@ fi
 
 
 # generate output files
-ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-config.sh"
+ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh"
 
 ac_config_files="$ac_config_files Modules/ld_so_aix"
 
@@ -18018,6 +18018,7 @@ do
     "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;;
     "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;;
     "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;;
+    "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;;
     "Misc/python-config.sh") CONFIG_FILES="$CONFIG_FILES Misc/python-config.sh" ;;
     "Modules/ld_so_aix") CONFIG_FILES="$CONFIG_FILES Modules/ld_so_aix" ;;
 

configure.ac

@@ -5589,7 +5589,7 @@ AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1)
 
 
 # generate output files
-AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-config.sh)
+AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh)
 AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix])
 AC_OUTPUT