Skip to content

Commit c9abff8

Browse files
T-DynamosT-Dynamos
committed
recipes: Introduce RustCompiledComponentsRecipe, add pydantic-core and update cryptography 
1 parent fb97f1a commit c9abff8

File tree

7 files changed

+223
-30
lines changed

7 files changed

+223
-30
lines changed

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,13 @@ virtualenv: $(VIRTUAL_ENV)
3131
test:
3232
$(TOX) -- tests/ --ignore tests/test_pythonpackage.py
3333

34+
# Also install and configure rust
3435
rebuild_updated_recipes: virtualenv
3536
. $(ACTIVATE) && \
37+
curl https://sh.rustup.rs -sSf | sh -s -- -y && \
38+
. "$(HOME)/.cargo/env" && \
39+
rustup target list && \
40+
pip3 install maturin && \
3641
ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \
3742
$(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS)
3843

‎pythonforandroid/recipe.py‎

Lines changed: 183 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
import sh
88
import shutil
99
import fnmatch
10+
import zipfile
1011
import urllib.request
1112
from urllib.request import urlretrieve
12-
from os import listdir, unlink, environ, curdir, walk
13+
from os import listdir, unlink, environ, curdir, walk, remove
1314
from sys import stdout
1415
import time
1516
try:
@@ -20,7 +21,7 @@
2021
import packaging.version
2122

2223
from pythonforandroid.logger import (
23-
logger, info, warning, debug, shprint, info_main)
24+
logger, info, warning, debug, shprint, info_main, error)
2425
from pythonforandroid.util import (
2526
current_directory, ensure_dir, BuildInterruptingException, rmdir, move,
2627
touch)
@@ -169,13 +170,14 @@ def versioned_url(self):
169170
return None
170171
return self.url.format(version=self.version)
171172

172-
def download_file(self, url, target, cwd=None):
173+
def download_file(self, url, target, cwd=None, msg=True):
173174
"""
174175
(internal) Download an ``url`` to a ``target``.
175176
"""
176177
if not url:
177178
return
178-
info('Downloading {} from {}'.format(self.name, url))
179+
if msg:
180+
info('Downloading {} from {}'.format(self.name, url))
179181

180182
if cwd:
181183
target = join(cwd, target)
@@ -458,7 +460,6 @@ def unpack(self, arch):
458460
# apparently happens sometimes with
459461
# github zips
460462
pass
461-
import zipfile
462463
fileh = zipfile.ZipFile(extraction_filename, 'r')
463464
root_directory = fileh.filelist[0].filename.split('/')[0]
464465
if root_directory != basename(directory_name):
@@ -1126,7 +1127,184 @@ def get_recipe_env(self, arch, with_flags_in_cc=True):
11261127

11271128
return env
11281129

1130+
class RustCompiledComponentsRecipe(PythonRecipe):
1131+
1132+
# Rust toolchain codes
1133+
# https://doc.rust-lang.org/nightly/rustc/platform-support.html
1134+
RUST_ARCH_CODES = {
1135+
"arm64-v8a": "aarch64-linux-android",
1136+
"armeabi-v7a": "armv7-linux-androideabi",
1137+
"x86_64": "x86_64-linux-android",
1138+
"x86": "i686-linux-android",
1139+
}
1140+
1141+
# Build python wheel using `maturin` instead
1142+
# of default `python -m build [...]`
1143+
# Note: This option requires `maturin` to be installed in host system
1144+
use_maturin = False
1145+
1146+
# Directory where to find built wheel
1147+
# For normal build: "dist/*-linux_*.whl"
1148+
# For maturin: "target/wheels/*-linux_*.whl"
1149+
built_wheel_pattern = None
1150+
1151+
# Required are required for building
1152+
# These unpacked in hostpython's site-packages
1153+
# before building
1154+
# Warning: Don't specify arch specific wheels here
1155+
hostpython_wheels = {}
11291156

1157+
call_hostpython_via_targetpython = False
1158+
1159+
def __init__(self, *arg, **kwargs):
1160+
super().__init__(*arg, **kwargs)
1161+
self.append_deps_if_absent(["python3"])
1162+
1163+
# Check for host deps
1164+
if not hasattr(sh, "rustup"):
1165+
error(
1166+
"`rustup` was not found on host system. Please install it using:"
1167+
"\n`curl https://sh.rustup.rs -sSf | sh`\n"
1168+
)
1169+
exit(1)
1170+
1171+
if self.use_maturin and not hasattr(sh, "maturin"):
1172+
error(
1173+
"maturin was not found on host system, please install with pip and rerun"
1174+
)
1175+
exit(1)
1176+
1177+
if not self.use_maturin:
1178+
self.hostpython_wheels["build"] = (
1179+
"https://files.pythonhosted.org/packages/"
1180+
"93/dd/b464b728b866aaa62785a609e0dd8c72201d62c5f7c53e7c20f4dceb085f/"
1181+
"build-1.0.3-py3-none-any.whl"
1182+
)
1183+
# Rust build requires setuptools-rust
1184+
self.hostpython_wheels["setuptools_rust"] = (
1185+
"https://files.pythonhosted.org/packages/"
1186+
"e0/34/d88a7ceb193fbcee6c8992d1b1e33ed20361027e07fea1676efc45ec7a43/"
1187+
"setuptools_rust-1.8.1-py3-none-any.whl"
1188+
)
1189+
self.hostpython_wheels["wheel"] = (
1190+
"https://files.pythonhosted.org/packages/"
1191+
"c7/c3/55076fc728723ef927521abaa1955213d094933dc36d4a2008d5101e1af5/"
1192+
"wheel-0.42.0-py3-none-any.whl"
1193+
)
1194+
1195+
if self.built_wheel_pattern is None:
1196+
self.built_wheel_pattern = (
1197+
"target/wheels/*-linux_*.whl"
1198+
if self.use_maturin else
1199+
"dist/*-linux_*.whl"
1200+
)
1201+
1202+
def install_hostpython_wheels(self):
1203+
workdir = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
1204+
temp_zip = join(workdir, "temp.zip")
1205+
info("Processing hostpython wheels: {}".format(str(list(self.hostpython_wheels.keys()))))
1206+
for folder in self.hostpython_wheels.keys():
1207+
self.download_file(self.hostpython_wheels[folder], temp_zip, msg=False)
1208+
with zipfile.ZipFile(join(workdir, temp_zip), "r") as zip_ref:
1209+
zip_ref.extractall(workdir)
1210+
remove(temp_zip)
1211+
1212+
def append_deps_if_absent(self, deps):
1213+
# Just to ensure if 'build' and 'python' is present in deps
1214+
for dep in deps:
1215+
if dep not in self.depends:
1216+
self.depends.append(dep)
1217+
1218+
def get_recipe_env(self, arch):
1219+
env = super().get_recipe_env(arch)
1220+
1221+
# Set rust build target
1222+
build_target = self.RUST_ARCH_CODES[arch.arch]
1223+
cargo_linker_name = "CARGO_TARGET_{}_LINKER".format(
1224+
build_target.upper().replace("-", "_")
1225+
)
1226+
env["CARGO_BUILD_TARGET"] = build_target
1227+
env[cargo_linker_name] = join(
1228+
self.ctx.ndk.llvm_prebuilt_dir,
1229+
"bin",
1230+
"{}{}-clang".format(
1231+
# NDK's Clang format
1232+
build_target.replace("7", "7a")
1233+
if build_target.startswith("armv7")
1234+
else build_target,
1235+
self.ctx.ndk_api,
1236+
),
1237+
)
1238+
env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format(
1239+
# App libs dir
1240+
self.ctx.get_libs_dir(arch.arch),
1241+
# Python lib dir
1242+
join(
1243+
Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch),
1244+
"android-build",
1245+
)
1246+
)
1247+
env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join(
1248+
Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch),
1249+
'android-build', "build",
1250+
"lib.linux-*-3.11/"
1251+
))[0])
1252+
1253+
info_main("Ensuring rust build toolchain")
1254+
shprint(sh.rustup, "target", "add", build_target)
1255+
1256+
# Add host python to PATH
1257+
env["PATH"] = ("{hostpython_dir}:{old_path}").format(
1258+
hostpython_dir=Recipe.get_recipe(
1259+
"hostpython3", self.ctx
1260+
).get_path_to_python(),
1261+
old_path=env["PATH"],
1262+
)
1263+
return env
1264+
1265+
def build_arch(self, arch):
1266+
build_dir = self.get_build_dir(arch.arch)
1267+
env = self.get_recipe_env(arch)
1268+
self.install_hostpython_wheels()
1269+
python_recipe = Recipe.get_recipe("python3", self.ctx)
1270+
built_wheel = None
1271+
1272+
#TODO: explain why
1273+
env['PYTHONPATH'] = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
1274+
1275+
with current_directory(build_dir):
1276+
if self.use_maturin:
1277+
shprint(
1278+
sh.maturin,
1279+
"build",
1280+
"-i",
1281+
"python{}".format(
1282+
".".join(python_recipe.version.split(".")[:2])
1283+
),
1284+
"--skip-auditwheel",
1285+
_env=env,
1286+
)
1287+
else:
1288+
shprint(
1289+
sh.Command(self.hostpython_location),
1290+
"-m",
1291+
"build",
1292+
"--no-isolation",
1293+
"--skip-dependency-check",
1294+
"--wheel",
1295+
_env=env,
1296+
)
1297+
# Find the built wheel
1298+
built_wheel = realpath(glob.glob(self.built_wheel_pattern)[0])
1299+
1300+
if built_wheel:
1301+
info("Unzipping built wheel '{}'".format(basename(built_wheel)))
1302+
1303+
# Unzip .whl file into site-packages
1304+
with zipfile.ZipFile(built_wheel, "r") as zip_ref:
1305+
zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch))
1306+
info("Successfully installed '{}'".format(basename(built_wheel)))
1307+
11301308
class TargetPythonRecipe(Recipe):
11311309
'''Class for target python recipes. Sets ctx.python_recipe to point to
11321310
itself, so as to know later what kind of Python was built or used.'''

‎pythonforandroid/recipes/build/__init__.py‎

Whitespace-only changes.

‎pythonforandroid/recipes/cryptography/__init__.py‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,25 @@
1-
from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
1+
from pythonforandroid.recipe import RustCompiledComponentsRecipe
2+
from os.path import join
23

4+
class CryptographyRecipe(RustCompiledComponentsRecipe):
35

4-
class CryptographyRecipe(CompiledComponentsPythonRecipe):
56
name = 'cryptography'
6-
version = '2.8'
7-
url = 'https://github.com/pyca/cryptography/archive/{version}.tar.gz'
7+
version = '42.0.1'
8+
url = 'https://github.com/pyca/cryptography/archive/refs/tags/{version}.tar.gz'
89
depends = ['openssl', 'six', 'setuptools', 'cffi']
9-
call_hostpython_via_targetpython = False
10+
hostpython_wheels = {
11+
"semantic_version":"https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl",
12+
"pyproject_hooks":"https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl"
13+
}
1014

1115
def get_recipe_env(self, arch):
1216
env = super().get_recipe_env(arch)
13-
14-
openssl_recipe = Recipe.get_recipe('openssl', self.ctx)
15-
env['CFLAGS'] += openssl_recipe.include_flags(arch)
16-
env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
17-
env['LIBS'] = openssl_recipe.link_libs_flags()
18-
17+
openssl_build_dir = self.get_recipe('openssl', self.ctx).get_build_dir(arch.arch)
18+
build_target = self.RUST_ARCH_CODES[arch.arch].upper().replace("-", "_")
19+
openssl_include = "{}_OPENSSL_INCLUDE_DIR".format(build_target)
20+
openssl_libs = "{}_OPENSSL_LIB_DIR".format(build_target)
21+
env[openssl_include] = join(openssl_build_dir, 'include')
22+
env[openssl_libs] = join(openssl_build_dir)
1923
return env
2024

21-
2225
recipe = CryptographyRecipe()

‎pythonforandroid/recipes/pydantic-core/__init__.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from os.path import join
2+
from pythonforandroid.recipe import RustCompiledComponentsRecipe
3+
from pythonforandroid.logger import info
4+
5+
class PydanticcoreRecipe(RustCompiledComponentsRecipe):
6+
version = "2.16.1"
7+
url = "https://github.com/pydantic/pydantic-core/archive/refs/tags/v{version}.tar.gz"
8+
use_maturin = True
9+
hostpython_wheels = {"typing_extensions":"https://files.pythonhosted.org/packages/b7/f4/6a90020cd2d93349b442bfcb657d0dc91eee65491600b2cb1d388bc98e6b/typing_extensions-4.9.0-py3-none-any.whl"}
10+
11+
def should_build(self, arch):
12+
name = self.folder_name
13+
if self.ctx.has_package(name.replace("-", "_"), arch):
14+
info('Python package already exists in site-packages')
15+
return False
16+
info('{} apparently isn\'t already in site-packages'.format(name))
17+
return True
18+
19+
recipe = PydanticcoreRecipe()

‎pythonforandroid/recipes/pydantic/__init__.py‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎pythonforandroid/recipes/setuptools/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33

44
class SetuptoolsRecipe(PythonRecipe):
5-
version = '51.3.3'
5+
version = '69.0.3'
66
url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz'
77
call_hostpython_via_targetpython = False
88
install_in_hostpython = True

0 commit comments

Comments
 (0)