Skip to content

Commit 2d6d26a

Browse files
hallvictoriahallvictoria
authored
fixes for linux-arm64 build (#1686)
1 parent ae4d1bc commit 2d6d26a

File tree

3 files changed

+180
-4
lines changed

3 files changed

+180
-4
lines changed

‎eng/pack/scripts/nix_arm64_deps.sh

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/bash
2+
3+
python -m venv .env
4+
source .env/bin/activate
5+
python -m pip install --upgrade pip
6+
7+
version_minor=$(echo $1 | cut -d '.' -f 2)
8+
mkdir -p $BUILD_SOURCESDIRECTORY/deps
9+
10+
# Targeting: grpcio manylinux_2_17_aarch64.whl build
11+
12+
# Starts a docker container using the linux/arm64 platform
13+
# Inside the container, we perform the same steps as our typical builds
14+
# However, since we're running them on the linux/arm64 platform, we ensure
15+
# that we pull in the correct grpc, etc. builds
16+
docker run --privileged --rm tonistiigi/binfmt --install all
17+
docker run --name my-arm64-container --platform linux/arm64 \
18+
-v ./:/src \
19+
-w /src \
20+
python:3.$version_minor bash -c "
21+
ls -la /src # debug: see what files exist
22+
apt-get update && apt-get install -y git curl && \
23+
pip install --upgrade pip && \
24+
pip install . && \
25+
pip install . --target /src && \
26+
pip install invoke && \
27+
cd tests && \
28+
python -m invoke -c test_setup build-protos && \
29+
ls -la /src
30+
"
31+
32+
# This copies over the build files from the docker container to the local pipeline
33+
docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/
34+
docker rm my-arm64-container
35+
36+
# From the container, we have many unnecessary files. Here, we only
37+
# copy over the relevant files to the 'deps/' directory.
38+
copy_list=(
39+
"azure"
40+
"azure_functions_worker"
41+
"azurefunctions"
42+
"dateutil"
43+
"google"
44+
"grpc"
45+
"markupsafe"
46+
"proxy_worker"
47+
"six.py"
48+
"werkzeug"
49+
)
50+
51+
for dir in "${copy_list[@]}"; do
52+
src="$BUILD_SOURCESDIRECTORY/all/$dir"
53+
dest="$BUILD_SOURCESDIRECTORY/deps"
54+
55+
if [ -e $src ]; then
56+
echo "Copying $dir..."
57+
cp -r $src $dest
58+
else
59+
echo "Directory $dir not found in deps — skipping"
60+
fi
61+
done
62+
63+
cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps"
64+
65+
version_minor=$(echo $1 | cut -d '.' -f 2)
66+
if [[ $version_minor -lt 13 ]]; then
67+
cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker"
68+
else
69+
cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker"
70+
fi
71+
72+
echo "Listing contents of deps directory:"
73+
ls -la $BUILD_SOURCESDIRECTORY/deps

‎eng/pack/templates/nix_arm64_env_gen.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
parameters:
2+
pythonVersion: ''
3+
artifactName: ''
4+
5+
steps:
6+
- task: UsePythonVersion@0
7+
inputs:
8+
versionSpec: ${{ parameters.pythonVersion }}
9+
addToPath: true
10+
- powershell: |
11+
# Parse the Python minor version
12+
$PY_VER = "$(pythonVersion)"
13+
Write-Host "Python version: $PY_VER"
14+
# Extract minor version as integers
15+
$versionParts = $PY_VER.Split('.')
16+
$PY_MINOR = [int]$versionParts[1]
17+
Write-Host "Minor version: $PY_MINOR"
18+
19+
# Set build-related variables based on Python minor version
20+
if( $PY_MINOR -ge 13 )
21+
{
22+
Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)"
23+
Write-Host "##vso[task.setvariable variable=proxyWorker;]true"
24+
}
25+
else
26+
{
27+
Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)"
28+
Write-Host "##vso[task.setvariable variable=proxyWorker;]false"
29+
}
30+
displayName: 'Set necessary variables'
31+
env:
32+
pythonVersion: ${{ parameters.pythonVersion }}
33+
- powershell: |
34+
Write-Host "workerPath: $(workerPath)"
35+
Write-Host "proxyWorker: $(proxyWorker)"
36+
displayName: 'Display variables'
37+
- script: |
38+
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
39+
displayName: 'Docker login to avoid rate limits'
40+
env:
41+
DOCKER_USERNAME: $(DOCKER_USERNAME)
42+
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
43+
- task: ShellScript@2
44+
inputs:
45+
disableAutoCwd: true
46+
scriptPath: 'eng/pack/scripts/nix_arm64_deps.sh'
47+
args: '${{ parameters.pythonVersion }}'
48+
displayName: 'Install Dependencies'
49+
- bash: |
50+
pip install pip-audit
51+
pip-audit -r requirements.txt
52+
displayName: 'Run vulnerability scan'
53+
condition: ne(variables['pythonVersion'], '3.7')
54+
- task: CopyFiles@2
55+
inputs:
56+
contents: '$(workerPath)'
57+
targetFolder: '$(Build.ArtifactStagingDirectory)'
58+
flattenFolders: true
59+
- task: CopyFiles@2
60+
inputs:
61+
sourceFolder: '$(Build.SourcesDirectory)/deps'
62+
contents: |
63+
**
64+
!grpc_tools/**/*
65+
!grpcio_tools*/*
66+
!build/**
67+
!docs/**
68+
!eng/**
69+
!python/**
70+
!tests/**
71+
!setuptools*/**
72+
!_distutils_hack/**
73+
!distutils-precedence.pth
74+
!pkg_resources/**
75+
!*.dist-info/**
76+
!werkzeug/debug/shared/debugger.js
77+
!proxy_worker/**
78+
targetFolder: '$(Build.ArtifactStagingDirectory)'
79+
condition: eq(variables['proxyWorker'], false)
80+
displayName: 'Copy azure_functions_worker files'
81+
- task: CopyFiles@2
82+
inputs:
83+
sourceFolder: '$(Build.SourcesDirectory)/deps'
84+
contents: |
85+
**
86+
!grpc_tools/**/*
87+
!grpcio_tools*/*
88+
!build/**
89+
!docs/**
90+
!eng/**
91+
!python/**
92+
!tests/**
93+
!setuptools*/**
94+
!_distutils_hack/**
95+
!distutils-precedence.pth
96+
!pkg_resources/**
97+
!*.dist-info/**
98+
!werkzeug/debug/shared/debugger.js
99+
!dateutil/**
100+
!azure_functions_worker/**
101+
targetFolder: '$(Build.ArtifactStagingDirectory)'
102+
condition: eq(variables['proxyWorker'], true)
103+
displayName: 'Copy proxy_worker files'

‎eng/templates/official/jobs/build-artifacts.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,9 @@ jobs:
159159
artifactName: '$(pythonVersion)_OSX_ARM64'
160160
- job: Build_LINUX_ARM64
161161
pool:
162-
name: Azure Pipelines
163-
image: macOS-latest
164-
os: macOS
162+
name: 1es-pool-azfunc
163+
image: 1es-ubuntu-22.04
164+
os: linux
165165
strategy:
166166
matrix:
167167
Python39V4:
@@ -181,7 +181,7 @@ jobs:
181181
targetPath: $(Build.ArtifactStagingDirectory)
182182
artifactName: "$(pythonVersion)_LINUX_ARM64"
183183
steps:
184-
- template: ../../../pack/templates/nix_env_gen.yml
184+
- template: ../../../pack/templates/nix_arm64_env_gen.yml
185185
parameters:
186186
pythonVersion: '$(pythonVersion)'
187187
artifactName: '$(pythonVersion)_LINUX_ARM64'

0 commit comments

Comments
 (0)