Skip to content

Commit 24706b4

Browse files
T-DynamosT-Dynamos
committed
recipes: Introduce RustCompiledComponentsRecipe, add pydantic-core and update cryptography 
1 parent fb97f1a commit 24706b4

File tree

7 files changed

+229
-29
lines changed

7 files changed

+229
-29
lines changed

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,13 @@ virtualenv: $(VIRTUAL_ENV)
3131
test:
3232
$(TOX) -- tests/ --ignore tests/test_pythonpackage.py
3333

34+
# Also install and configure rust
3435
rebuild_updated_recipes: virtualenv
3536
. $(ACTIVATE) && \
37+
curl https://sh.rustup.rs -sSf | sh -s -- -y && \
38+
. "$(HOME)/.cargo/env" && \
39+
rustup target list && \
40+
pip3 install maturin && \
3641
ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \
3742
$(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS)
3843

‎pythonforandroid/recipe.py‎

Lines changed: 185 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
import sh
88
import shutil
99
import fnmatch
10+
import zipfile
1011
import urllib.request
1112
from urllib.request import urlretrieve
12-
from os import listdir, unlink, environ, curdir, walk
13+
from os import listdir, unlink, environ, curdir, walk, remove
1314
from sys import stdout
1415
import time
1516
try:
@@ -20,7 +21,7 @@
2021
import packaging.version
2122

2223
from pythonforandroid.logger import (
23-
logger, info, warning, debug, shprint, info_main)
24+
logger, info, warning, debug, shprint, info_main, error)
2425
from pythonforandroid.util import (
2526
current_directory, ensure_dir, BuildInterruptingException, rmdir, move,
2627
touch)
@@ -169,13 +170,14 @@ def versioned_url(self):
169170
return None
170171
return self.url.format(version=self.version)
171172

172-
def download_file(self, url, target, cwd=None):
173+
def download_file(self, url, target, cwd=None, msg=True):
173174
"""
174175
(internal) Download an ``url`` to a ``target``.
175176
"""
176177
if not url:
177178
return
178-
info('Downloading {} from {}'.format(self.name, url))
179+
if msg:
180+
info('Downloading {} from {}'.format(self.name, url))
179181

180182
if cwd:
181183
target = join(cwd, target)
@@ -458,7 +460,6 @@ def unpack(self, arch):
458460
# apparently happens sometimes with
459461
# github zips
460462
pass
461-
import zipfile
462463
fileh = zipfile.ZipFile(extraction_filename, 'r')
463464
root_directory = fileh.filelist[0].filename.split('/')[0]
464465
if root_directory != basename(directory_name):
@@ -1127,6 +1128,185 @@ def get_recipe_env(self, arch, with_flags_in_cc=True):
11271128
return env
11281129

11291130

1131+
class RustCompiledComponentsRecipe(PythonRecipe):
1132+
1133+
# Rust toolchain codes
1134+
# https://doc.rust-lang.org/nightly/rustc/platform-support.html
1135+
RUST_ARCH_CODES = {
1136+
"arm64-v8a": "aarch64-linux-android",
1137+
"armeabi-v7a": "armv7-linux-androideabi",
1138+
"x86_64": "x86_64-linux-android",
1139+
"x86": "i686-linux-android",
1140+
}
1141+
1142+
# Build python wheel using `maturin` instead
1143+
# of default `python -m build [...]`
1144+
# Note: This option requires `maturin` to be installed in host system
1145+
use_maturin = False
1146+
1147+
# Directory where to find built wheel
1148+
# For normal build: "dist/*-linux_*.whl"
1149+
# For maturin: "target/wheels/*-linux_*.whl"
1150+
built_wheel_pattern = None
1151+
1152+
# Required are required for building
1153+
# These unpacked in hostpython's site-packages
1154+
# before building
1155+
# Warning: Don't specify arch specific wheels here
1156+
hostpython_wheels = {}
1157+
1158+
call_hostpython_via_targetpython = False
1159+
1160+
def __init__(self, *arg, **kwargs):
1161+
super().__init__(*arg, **kwargs)
1162+
self.append_deps_if_absent(["python3"])
1163+
1164+
# Check for host deps
1165+
if not hasattr(sh, "rustup"):
1166+
error(
1167+
"`rustup` was not found on host system. Please install it using:"
1168+
"\n`curl https://sh.rustup.rs -sSf | sh`\n"
1169+
)
1170+
exit(1)
1171+
1172+
if self.use_maturin and not hasattr(sh, "maturin"):
1173+
error(
1174+
"maturin was not found on host system, please install with pip and rerun"
1175+
)
1176+
exit(1)
1177+
1178+
if not self.use_maturin:
1179+
self.hostpython_wheels["build"] = (
1180+
"https://files.pythonhosted.org/packages/"
1181+
"93/dd/b464b728b866aaa62785a609e0dd8c72201d62c5f7c53e7c20f4dceb085f/"
1182+
"build-1.0.3-py3-none-any.whl"
1183+
)
1184+
# Rust build requires setuptools-rust
1185+
self.hostpython_wheels["setuptools_rust"] = (
1186+
"https://files.pythonhosted.org/packages/"
1187+
"e0/34/d88a7ceb193fbcee6c8992d1b1e33ed20361027e07fea1676efc45ec7a43/"
1188+
"setuptools_rust-1.8.1-py3-none-any.whl"
1189+
)
1190+
self.hostpython_wheels["wheel"] = (
1191+
"https://files.pythonhosted.org/packages/"
1192+
"c7/c3/55076fc728723ef927521abaa1955213d094933dc36d4a2008d5101e1af5/"
1193+
"wheel-0.42.0-py3-none-any.whl"
1194+
)
1195+
1196+
if self.built_wheel_pattern is None:
1197+
self.built_wheel_pattern = (
1198+
"target/wheels/*-linux_*.whl"
1199+
if self.use_maturin else
1200+
"dist/*-linux_*.whl"
1201+
)
1202+
1203+
def install_hostpython_wheels(self):
1204+
workdir = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
1205+
temp_zip = join(workdir, "temp.zip")
1206+
info("Processing hostpython wheels: {}".format(str(list(self.hostpython_wheels.keys()))))
1207+
for folder in self.hostpython_wheels.keys():
1208+
self.download_file(self.hostpython_wheels[folder], temp_zip, msg=False)
1209+
with zipfile.ZipFile(join(workdir, temp_zip), "r") as zip_ref:
1210+
zip_ref.extractall(workdir)
1211+
remove(temp_zip)
1212+
1213+
def append_deps_if_absent(self, deps):
1214+
# Just to ensure if 'build' and 'python' is present in deps
1215+
for dep in deps:
1216+
if dep not in self.depends:
1217+
self.depends.append(dep)
1218+
1219+
def get_recipe_env(self, arch):
1220+
env = super().get_recipe_env(arch)
1221+
1222+
# Set rust build target
1223+
build_target = self.RUST_ARCH_CODES[arch.arch]
1224+
cargo_linker_name = "CARGO_TARGET_{}_LINKER".format(
1225+
build_target.upper().replace("-", "_")
1226+
)
1227+
env["CARGO_BUILD_TARGET"] = build_target
1228+
env[cargo_linker_name] = join(
1229+
self.ctx.ndk.llvm_prebuilt_dir,
1230+
"bin",
1231+
"{}{}-clang".format(
1232+
# NDK's Clang format
1233+
build_target.replace("7", "7a")
1234+
if build_target.startswith("armv7")
1235+
else build_target,
1236+
self.ctx.ndk_api,
1237+
),
1238+
)
1239+
env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format(
1240+
# App libs dir
1241+
self.ctx.get_libs_dir(arch.arch),
1242+
# Python lib dir
1243+
join(
1244+
Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch),
1245+
"android-build",
1246+
)
1247+
)
1248+
env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join(
1249+
Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch),
1250+
'android-build', "build",
1251+
"lib.linux-*-3.11/"
1252+
))[0])
1253+
1254+
info_main("Ensuring rust build toolchain")
1255+
shprint(sh.rustup, "target", "add", build_target)
1256+
1257+
# Add host python to PATH
1258+
env["PATH"] = ("{hostpython_dir}:{old_path}").format(
1259+
hostpython_dir=Recipe.get_recipe(
1260+
"hostpython3", self.ctx
1261+
).get_path_to_python(),
1262+
old_path=env["PATH"],
1263+
)
1264+
return env
1265+
1266+
def build_arch(self, arch):
1267+
build_dir = self.get_build_dir(arch.arch)
1268+
env = self.get_recipe_env(arch)
1269+
self.install_hostpython_wheels()
1270+
python_recipe = Recipe.get_recipe("python3", self.ctx)
1271+
built_wheel = None
1272+
1273+
# TODO: explain why
1274+
env['PYTHONPATH'] = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
1275+
1276+
with current_directory(build_dir):
1277+
if self.use_maturin:
1278+
shprint(
1279+
sh.maturin,
1280+
"build",
1281+
"-i",
1282+
"python{}".format(
1283+
".".join(python_recipe.version.split(".")[:2])
1284+
),
1285+
"--skip-auditwheel",
1286+
_env=env,
1287+
)
1288+
else:
1289+
shprint(
1290+
sh.Command(self.hostpython_location),
1291+
"-m",
1292+
"build",
1293+
"--no-isolation",
1294+
"--skip-dependency-check",
1295+
"--wheel",
1296+
_env=env,
1297+
)
1298+
# Find the built wheel
1299+
built_wheel = realpath(glob.glob(self.built_wheel_pattern)[0])
1300+
1301+
if built_wheel:
1302+
info("Unzipping built wheel '{}'".format(basename(built_wheel)))
1303+
1304+
# Unzip .whl file into site-packages
1305+
with zipfile.ZipFile(built_wheel, "r") as zip_ref:
1306+
zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch))
1307+
info("Successfully installed '{}'".format(basename(built_wheel)))
1308+
1309+
11301310
class TargetPythonRecipe(Recipe):
11311311
'''Class for target python recipes. Sets ctx.python_recipe to point to
11321312
itself, so as to know later what kind of Python was built or used.'''

‎pythonforandroid/recipes/build/__init__.py‎

Whitespace-only changes.

‎pythonforandroid/recipes/cryptography/__init__.py‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,26 @@
1-
from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
1+
from pythonforandroid.recipe import RustCompiledComponentsRecipe
2+
from os.path import join
23

34

4-
class CryptographyRecipe(CompiledComponentsPythonRecipe):
5+
class CryptographyRecipe(RustCompiledComponentsRecipe):
6+
57
name = 'cryptography'
6-
version = '2.8'
7-
url = 'https://github.com/pyca/cryptography/archive/{version}.tar.gz'
8+
version = '42.0.1'
9+
url = 'https://github.com/pyca/cryptography/archive/refs/tags/{version}.tar.gz'
810
depends = ['openssl', 'six', 'setuptools', 'cffi']
9-
call_hostpython_via_targetpython = False
11+
hostpython_wheels = {
12+
"semantic_version": "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl",
13+
"pyproject_hooks": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl"
14+
}
1015

1116
def get_recipe_env(self, arch):
1217
env = super().get_recipe_env(arch)
13-
14-
openssl_recipe = Recipe.get_recipe('openssl', self.ctx)
15-
env['CFLAGS'] += openssl_recipe.include_flags(arch)
16-
env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
17-
env['LIBS'] = openssl_recipe.link_libs_flags()
18-
18+
openssl_build_dir = self.get_recipe('openssl', self.ctx).get_build_dir(arch.arch)
19+
build_target = self.RUST_ARCH_CODES[arch.arch].upper().replace("-", "_")
20+
openssl_include = "{}_OPENSSL_INCLUDE_DIR".format(build_target)
21+
openssl_libs = "{}_OPENSSL_LIB_DIR".format(build_target)
22+
env[openssl_include] = join(openssl_build_dir, 'include')
23+
env[openssl_libs] = join(openssl_build_dir)
1924
return env
2025

2126

‎pythonforandroid/recipes/pydantic-core/__init__.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
from pythonforandroid.recipe import RustCompiledComponentsRecipe
2+
from pythonforandroid.logger import info
3+
4+
5+
class PydanticcoreRecipe(RustCompiledComponentsRecipe):
6+
version = "2.16.1"
7+
url = "https://github.com/pydantic/pydantic-core/archive/refs/tags/v{version}.tar.gz"
8+
use_maturin = True
9+
hostpython_wheels = {
10+
"typing_extensions": "https://files.pythonhosted.org/packages/b7/f4/6a90020cd2d93349b442bfcb657d0dc91eee65491600b2cb1d388bc98e6b/typing_extensions-4.9.0-py3-none-any.whl"
11+
}
12+
13+
def should_build(self, arch):
14+
name = self.folder_name
15+
if self.ctx.has_package(name.replace("-", "_"), arch):
16+
info('Python package already exists in site-packages')
17+
return False
18+
info('{} apparently isn\'t already in site-packages'.format(name))
19+
return True
20+
21+
22+
recipe = PydanticcoreRecipe()

‎pythonforandroid/recipes/pydantic/__init__.py‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎pythonforandroid/recipes/setuptools/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33

44
class SetuptoolsRecipe(PythonRecipe):
5-
version = '51.3.3'
5+
version = '69.0.3'
66
url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz'
77
call_hostpython_via_targetpython = False
88
install_in_hostpython = True

0 commit comments

Comments
 (0)