Skip to content

Commit 01e912a

Browse files
rafierafie
committed
Build system refactoring #3 
1 parent 1961583 commit 01e912a

File tree

7 files changed

+113
-52
lines changed

7 files changed

+113
-52
lines changed

‎build/libtorch-arm/Dockerfile‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,24 @@
1-
ARG OS=debian:stretch-slim
1+
# BUILD redisfab/libtorch-${ARCH}:$(PT_VER)
22

3-
#----------------------------------------------------------------------------------------------
4-
FROM ${OS}
3+
# stretch|bionic
4+
ARG OSNICK=stretch
5+
6+
# arm32v7|arm64v8
7+
ARG ARCH=arm64v8
58

6-
# ARG ARCH=x64
79
ARG PT_VER=v1.1.0
810

9-
RUN set -ex; apt-get update; apt-get install -y git
11+
#----------------------------------------------------------------------------------------------
12+
FROM redisfab/${ARCH}-xbuild:${OSNICK} as builder
13+
14+
RUN [ "cross-build-start" ]
15+
16+
RUN set -ex; apt-get update -qq; apt-get install -y git
1017

1118
WORKDIR /build
1219

13-
RUN apt-get install -y build-essential
14-
RUN apt-get install -y python3-pip python3-cffi
15-
RUN apt-get install -y ninja-build cmake
16-
RUN pip3 install setuptools pyyaml typing
17-
RUN pip3 install numpy
20+
RUN apt-get install -y build-essential ninja-build cmake python3-pip python3-cffi
21+
RUN pip3 install setuptools pyyaml typing numpy
1822
# mkl mkl-include
1923

2024
RUN git clone https://github.com/pytorch/pytorch.git ;\
@@ -35,3 +39,10 @@ RUN cd pytorch ;\
3539
NO_NNPACK=1 \
3640
NO_QNNPACK=1 \
3741
python3 setup.py install
42+
43+
ADD ./deps/readies/ /build/deps/
44+
ADD ./collect.py /build/
45+
46+
RUN ./collect.py
47+
48+
RUN [ "cross-build-end" ]

‎build/libtorch-arm/collect.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
#----------------------------------------------------------------------------------------------
1414

15-
PYTORCH_VERSION = '1.0.1'
15+
PYTORCH_VERSION = '1.1.0'
1616

1717
parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.')
1818
parser.add_argument('--pytorch', default='../pytorch', help='root of pytorch repository')

‎build/libtorch-x64/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ FROM ${OS}
55

66
ARG PT_VER=v1.1.0
77

8-
RUN set -ex; apt-get update; apt-get install -y git
8+
RUN set -ex; apt-get -qq update; apt-get -q install -y git
99

1010
WORKDIR /build
1111

‎build/libtorch-x64/Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11

2-
.PHONY: repack upload
2+
.PHONY: pack publish
33

44
VERSION=1.1.0
55

66
S3_URL=redismodules/pytorch
77

8-
repack:
8+
pack:
99
@PT_VERSION=$(VERSION) ./repack.sh
1010

11-
upload:
11+
publish:
1212
@aws s3 cp libtorch-cpu-linux-x86_64-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read

‎deps/collect-bin.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@
44
import sys
55
from pathlib import Path
66
import itertools
7+
import argparse
78

89
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "readies"))
910
import paella
1011

12+
parser = argparse.ArgumentParser(description='Report platform characteristics.')
13+
parser.add_argument('--into', action="store", help='Install into ')
14+
args = parser.parse_args()
15+
16+
1117
paella.mkdir_p('install')
1218
for f in itertools.chain(Path('tensorflow').glob('**/*.so*'), Path('libtorch').glob('**/*.so*')):
1319
link = Path('install')/os.path.basename(f)

‎deps/readies/bin/getredis5‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,35 +17,35 @@ class Redis5Setup(paella.Setup):
1717
paella.Setup.__init__(self, nop)
1818

1919
def common_first(self):
20-
pass
20+
pass
2121

2222
def debian_compat(self):
23-
# https://chilts.org/installing-redis-from-chris-leas-ppa/
23+
# https://chilts.org/installing-redis-from-chris-leas-ppa/
2424
self.run("add-apt-repository -y ppa:chris-lea/redis-server")
25-
self.install("redis-server")
25+
self.install("redis-server")
2626

2727
def redhat_compat(self):
28-
# https://linuxize.com/post/how-to-install-and-configure-redis-on-centos-7/
28+
# https://linuxize.com/post/how-to-install-and-configure-redis-on-centos-7/
2929
self.install("epel-release yum-utils")
3030

3131
self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm")
32-
self.run("yum-config-manager -y --enable remi")
33-
self.install("redis")
32+
self.run("yum-config-manager -y --enable remi")
33+
self.install("redis")
3434

3535
def fedora(self):
36-
self.install("dnf-plugins-core")
37-
38-
self.install("https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
39-
self.install("--allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
40-
self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm")
41-
self.run("dnf config-manager -y --set-enabled remi")
42-
self.install("redis")
36+
self.install("dnf-plugins-core")
37+
38+
self.install("https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
39+
self.install("--allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
40+
self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm")
41+
self.run("dnf config-manager -y --set-enabled remi")
42+
self.install("redis")
4343

4444
def macosx(self):
45-
self.install("redis")
45+
self.install("redis")
4646

4747
def common_last(self):
48-
pass
48+
pass
4949

5050
#----------------------------------------------------------------------------------------------
5151

‎get_deps.sh‎

Lines changed: 66 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/env bash
22

3+
# set -x
34
set -e
45

56
if [[ "$1" == "cpu" ]]; then
@@ -20,6 +21,9 @@ cd deps
2021
if [[ ! -d dlpack ]]; then
2122
echo "Cloning dlpack..."
2223
git clone --depth 1 https://github.com/dmlc/dlpack.git
24+
echo "Done."
25+
else
26+
echo "dlpack is in place."
2327
fi
2428

2529
### TENSORFLOW
@@ -57,8 +61,14 @@ if [[ ! -d tensorflow ]]; then
5761

5862
[[ ! -f $LIBTF_ARCHIVE ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE
5963

60-
mkdir -p tensorflow
61-
tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C tensorflow
64+
rm -rf tensorflow.x
65+
mkdir tensorflow.x
66+
tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C tensorflow.x
67+
mv tensorflow.x tensorflow
68+
69+
echo "Done."
70+
else
71+
echo "TensorFlow is in place."
6272
fi
6373

6474
### PYTORCH
@@ -91,9 +101,18 @@ if [[ ! -d libtorch ]]; then
91101
LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz
92102
[[ -z $LIBTORCH_URL ]] && LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE
93103

94-
[[ ! -f $LIBTORCH_ARCHIVE ]] && wget --quiet $LIBTORCH_URL
104+
[[ ! -f $LIBTORCH_ARCHIVE ]] && wget -q $LIBTORCH_URL
105+
106+
rm -rf libtorch.x
107+
mkdir libtorch.x
95108

96-
tar xf $LIBTORCH_ARCHIVE --no-same-owner
109+
tar xf $LIBTORCH_ARCHIVE --no-same-owner -C libtorch.x
110+
mv libtorch.x/libtorch libtorch
111+
rmdir libtorch.x
112+
113+
echo "Done."
114+
else
115+
echo "librotch is in place."
97116
fi
98117

99118
### MKL
@@ -106,44 +125,69 @@ if [[ ! -d mkl ]]; then
106125

107126
MKL_OS=mac
108127
MKL_ARCHIVE=mklml_${MKL_OS}_${MKL_BUNDLE_VER}.tgz
109-
[[ ! -e ${MKL_ARCHIVE} ]] && wget --quiet https://github.com/intel/mkl-dnn/releases/download/v${MKL_VERSION}/${MKL_ARCHIVE}
110-
mkdir -p mkl
111-
tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C deps/mkl
128+
[[ ! -e ${MKL_ARCHIVE} ]] && wget -q https://github.com/intel/mkl-dnn/releases/download/v${MKL_VERSION}/${MKL_ARCHIVE}
129+
130+
rm -rf mkl.x
131+
mkdir mkl.x
132+
tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C mkl.x
133+
mv mkl.x mkl
134+
135+
136+
echo "Done."
112137
fi
138+
else
139+
echo "mkl is in place."
113140
fi
114141

115142
### ONNXRUNTIME
116143

117144
ORT_VERSION="0.4.0"
118145

119146
if [[ $OS == linux ]]; then
120-
if [[ $GPU == no ]]; then
121-
ORT_OS="linux-x64"
122-
ORT_BUILD="cpu"
123-
else
124-
ORT_OS="linux-x64-gpu"
125-
ORT_BUILD="gpu"
126-
fi
147+
if [[ $GPU == no ]]; then
148+
ORT_OS="linux-x64"
149+
ORT_BUILD="cpu"
150+
else
151+
ORT_OS="linux-x64-gpu"
152+
ORT_BUILD="gpu"
153+
fi
127154
elif [[ $OS == macosx ]]; then
128-
ORT_OS="osx-x64"
129-
ORT_BUILD=""
155+
ORT_OS="osx-x64"
156+
ORT_BUILD=""
130157
fi
131158

132159
ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_VERSION}.tgz
133160

134-
if [ ! -e ${ORT_ARCHIVE} ]; then
135-
echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD}"
136-
wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
137-
fi
161+
if [[ ! -d onnx ]]; then
162+
echo "Installing onnx..."
138163

139-
tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${PREFIX}
164+
if [[ ! -e ${ORT_ARCHIVE} ]]; then
165+
echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD} ..."
166+
wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
167+
echo "Done."
168+
fi
169+
170+
rm -rf onnx.x
171+
mkdir onnx.x
172+
tar xzf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C onnx.x
173+
mv onnx.x onnx
174+
175+
echo "Done."
176+
else
177+
echo "onnx is in place."
178+
fi
140179

141180
### Collect libraries
142181

143182
if [[ ! -d install ]]; then
144183
echo "Collecting binaries..."
145184

146-
python3 collect-bin.py
185+
rm -rf install.x
186+
mkdir install.x
187+
python3 collect-bin.py --into install.x
188+
mv install.x install
189+
190+
echo "Done."
147191
fi
148192

149193
# echo "Done"

0 commit comments

Comments
 (0)