Skip to content

Commit 9766e2a

Browse files
author
Sherin Thomas
authored
 Jobs on GPU (#230) - GPU build on CI - ORT CUDA compatibility issue fix
1 parent 737d156 commit 9766e2a

File tree

13 files changed

+207
-121
lines changed

13 files changed

+207
-121
lines changed

‎.circleci/config.yml

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,12 @@ jobs:
6868
steps:
6969
- ci_steps:
7070
platform: debian
71-
7271
# build-macos:
7372
# macos:
7473
# xcode: 10.2.1
7574
# steps:
7675
# - ci_steps:
7776
# platform: macosx
78-
7977
build-multiarch-docker:
8078
machine:
8179
enabled: true
@@ -100,7 +98,32 @@ jobs:
10098
sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
10199
make -C opt/build/docker build
102100
sudo make -C opt/build/docker publish
103-
101+
build-gpu:
102+
machine:
103+
enabled: true
104+
docker_layer_caching: true
105+
resource_class: gpu.nvidia.small
106+
image: ubuntu-1604-cuda-10.1:201909-23
107+
steps:
108+
- checkout
109+
# - run:
110+
# name: Checkout LFS
111+
# command: |
112+
# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
113+
# sudo apt-get install -y git-lfs
114+
# git lfs install
115+
# git lfs pull
116+
- run:
117+
name: Build
118+
command: |
119+
docker build -f Dockerfile.gpu-test -t redisai-gpu:latest-x64-bionic-test .
120+
- run:
121+
name: Test
122+
command: |
123+
mkdir -p ~/workspace/tests
124+
docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
125+
- store_test_results:
126+
path: ~/workspace/tests
104127
deploy_package:
105128
parameters:
106129
package:
@@ -143,9 +166,13 @@ workflows:
143166
#- build-multiarch-docker:
144167
# filters:
145168
# tags:
146-
# only: /.*/
169+
# only: /^v[0-9].*/
147170
# branches:
148-
# only: master
171+
# ignore: /.*/
172+
- build-gpu:
173+
filters:
174+
tags:
175+
only: /.*/
149176
- deploy_package:
150177
name: deploy_branch
151178
package: branch

‎CMakeLists.txt

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,8 @@ ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API)
9696

9797
IF(BUILD_TF)
9898
FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow
99-
PATHS ${depsAbs}/libtensorflow/lib)
99+
PATHS ${depsAbs}/libtensorflow/lib)
100+
MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
100101
IF (NOT TF_LIBRARIES)
101102
MESSAGE(WARNING "Could not find TensorFlow in ${depsAbs}/libtensorflow/lib. Trying find_package method")
102103
FIND_PACKAGE(TensorFlow REQUIRED)
@@ -112,8 +113,11 @@ ENDIF()
112113
#----------------------------------------------------------------------------------------------
113114

114115
IF(BUILD_TFLITE)
115-
FIND_LIBRARY(TFLITE_LIBRARIES NAMES tensorflow-lite
116+
FIND_LIBRARY(TFLITE_LIBRARIES_1 NAMES tensorflow-lite
116117
PATHS ${depsAbs}/libtensorflow-lite/lib)
118+
FIND_LIBRARY(TFLITE_LIBRARIES_2 NAMES benchmark-lib.a
119+
PATHS ${depsAbs}/libtensorflow-lite/lib)
120+
SET(TFLITE_LIBRARIES ${TFLITE_LIBRARIES_1} ${TFLITE_LIBRARIES_2})
117121
MESSAGE(STATUS "Found TensorFlow Lite Libraries: \"${TFLITE_LIBRARIES}\")")
118122
IF (NOT TFLITE_LIBRARIES)
119123
MESSAGE(FATAL_ERROR "Could not find TensorFlow Lite")

‎Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH}
1+
# BUILD redisfab/redisai-cpu-${OSNICK}:${VERSION}-${ARCH}
22

33
ARG REDIS_VER=5.0.7
44

‎Dockerfile.gpu

100755100644
Lines changed: 50 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,57 @@
1-
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 AS builder
1+
# BUILD redisfab/redisai-gpu:${VERSION}-${ARCH}-${OSNICK}
22

3-
ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
3+
ARG REDIS_VER=5.0.7
4+
5+
# OSNICK=bionic|centos7|centos6
6+
ARG OSNICK=bionic
7+
8+
# ARCH=x64|arm64v8|arm32v7
9+
ARG ARCH=x64
10+
11+
#----------------------------------------------------------------------------------------------
12+
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
13+
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder
14+
15+
# ENV DEPS "git ca-certificates wget unzip cmake libgomp1 patchelf coreutils tcl libjemalloc-dev"
416
ENV NVIDIA_VISIBLE_DEVICES all
517
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
618

7-
RUN apt update && apt -y upgrade
8-
RUN apt-get install -y libgomp1
9-
RUN apt-get install -y wget
10-
11-
RUN apt install -y build-essential tcl libjemalloc-dev
12-
RUN wget http://download.redis.io/redis-stable.tar.gz
13-
RUN tar xzvf redis-stable.tar.gz && cd redis-stable && make && make install && cd .. && rm redis-stable.tar.gz
14-
15-
# install latest cmake
16-
ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh
17-
RUN mkdir /opt/cmake
18-
RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
19-
RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
20-
RUN cmake --version
21-
22-
# Set up a build environment
23-
RUN set -ex;\
24-
deps="$DEPS";\
25-
apt-get update;\
26-
apt-get install -y --no-install-recommends $deps
27-
28-
# Get the dependencies
29-
WORKDIR /redisai
30-
ADD ./ /redisai
31-
RUN set -ex;\
32-
mkdir -p deps;\
33-
DEPS_DIRECTORY=deps bash ./get_deps.sh gpu
34-
35-
# Configurations
36-
ADD http://download.redis.io/redis-stable/redis.conf /usr/local/etc/redis/redis.conf
37-
RUN sed -i 's/bind 127.0.0.1/bind 0.0.0.0/g' /usr/local/etc/redis/redis.conf
38-
39-
# Build the source
40-
RUN set -ex;\
41-
rm -rf build;\
42-
mkdir -p build;\
43-
cd build;\
44-
cmake -DDEVICE=gpu ..;\
45-
make && make install;\
46-
cd ..
47-
48-
# Package the runner
49-
FROM builder
50-
51-
RUN mkdir -p /usr/lib/redis/modules/
52-
53-
COPY --from=builder /redisai/install-gpu/ /usr/lib/redis/modules/
19+
WORKDIR /build
20+
COPY --from=redis /usr/local/ /usr/local/
21+
22+
COPY ./opt/ opt/
23+
COPY ./test/test_requirements.txt test/
24+
25+
RUN ./opt/readies/bin/getpy
26+
RUN ./opt/system-setup.py
27+
28+
ARG DEPS_ARGS=""
29+
COPY ./get_deps.sh .
30+
RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
31+
32+
ARG BUILD_ARGS=""
33+
ADD ./ /build
34+
RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
35+
36+
ARG PACK=0
37+
ARG TEST=0
38+
39+
RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
40+
RUN if [ "$TEST" = "1" ]; then TEST="" make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
41+
42+
#----------------------------------------------------------------------------------------------
43+
FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04
44+
45+
RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
46+
47+
ENV REDIS_MODULES /usr/lib/redis/modules
48+
RUN mkdir -p $REDIS_MODULES/
49+
50+
COPY --from=redis /usr/local/ /usr/local/
51+
COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
52+
# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
5453

5554
WORKDIR /data
5655
EXPOSE 6379
57-
CMD ["redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
56+
# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
57+
CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]

‎Dockerfile.gpu-test

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# BUILD redisfab/redisai-gpu:${VERSION}-${ARCH}-${OSNICK}-test
2+
3+
ARG REDIS_VER=5.0.7
4+
5+
# OSNICK=bionic|centos7|centos6
6+
ARG OSNICK=bionic
7+
8+
# ARCH=x64|arm64v8|arm32v7
9+
ARG ARCH=x64
10+
11+
#----------------------------------------------------------------------------------------------
12+
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
13+
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder
14+
15+
ENV NVIDIA_VISIBLE_DEVICES all
16+
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
17+
18+
WORKDIR /build
19+
COPY --from=redis /usr/local/ /usr/local/
20+
21+
COPY ./opt/ opt/
22+
COPY ./test/test_requirements.txt test/
23+
24+
RUN ./opt/readies/bin/getpy
25+
RUN ./opt/system-setup.py
26+
27+
ARG DEPS_ARGS=""
28+
COPY ./get_deps.sh .
29+
RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
30+
31+
ARG BUILD_ARGS=""
32+
ADD ./ /build
33+
RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
34+
35+
ARG PACK=1
36+
37+
RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
38+
39+
RUN git remote set-url origin https://github.com/RedisAI/RedisAI
40+
41+
CMD ["bash", "-c", "make -C opt test GPU=1 SHOW=1"]

‎get_deps.sh

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ set -e
3030
[[ $VERBOSE == 1 ]] && set -x
3131

3232
if [[ "$1" == "cpu" ]]; then
33-
GPU=no
33+
GPU=0
3434
DEVICE=cpu
3535
elif [[ "$1" == "gpu" ]]; then
36-
GPU=yes
36+
GPU=1
3737
DEVICE=gpu
3838
else
39-
GPU=${GPU:-no}
39+
GPU=${GPU:-0}
4040
if [[ $GPU == 1 ]]; then
4141
DEVICE=gpu
4242
else
@@ -92,7 +92,7 @@ if [[ $WITH_TF != 0 ]]; then
9292

9393
if [[ $OS == linux ]]; then
9494
TF_OS="linux"
95-
if [[ $GPU == no ]]; then
95+
if [[ $GPU != 1 ]]; then
9696
TF_BUILD="cpu"
9797
else
9898
TF_BUILD="gpu"
@@ -148,7 +148,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
148148
LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow
149149
if [[ $OS == linux ]]; then
150150
TFLITE_OS="linux"
151-
# if [[ $GPU == no ]]; then
151+
# if [[ $GPU != 1 ]]; then
152152
# TFLITE_BUILD="cpu"
153153
# else
154154
# TFLITE_BUILD="gpu"
@@ -169,7 +169,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
169169

170170
LIBTFLITE_ARCHIVE=libtensorflowlite-${TFLITE_OS}-${TFLITE_ARCH}-${TFLITE_VERSION}.tar.gz
171171

172-
[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
172+
[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
173173

174174
rm -rf $LIBTFLITE.x
175175
mkdir $LIBTFLITE.x
@@ -195,13 +195,13 @@ if [[ $WITH_PT != 0 ]]; then
195195
echo "Installing libtorch ..."
196196

197197
PT_REPACK=0
198-
198+
199199
if [[ $OS == linux ]]; then
200200
PT_OS=linux
201-
if [[ $GPU == no ]]; then
201+
if [[ $GPU != 1 ]]; then
202202
PT_BUILD=cpu
203203
else
204-
PT_BUILD=cu100
204+
PT_BUILD=cu101
205205
fi
206206
if [[ $ARCH == x64 ]]; then
207207
PT_ARCH=x86_64
@@ -223,7 +223,7 @@ if [[ $WITH_PT != 0 ]]; then
223223
LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz
224224

225225
if [[ $PT_REPACK == 1 ]]; then
226-
PT_VERSION=$PT_VERSION $HERE/opt/build/libtorch/repack.sh
226+
PT_VERSION=$PT_VERSION GPU=$GPU $HERE/opt/build/libtorch/repack.sh
227227
else
228228
LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE
229229

@@ -280,7 +280,7 @@ if [[ $WITH_ORT != 0 ]]; then
280280

281281
if [[ $OS == linux ]]; then
282282
ORT_OS=linux
283-
if [[ $GPU == no ]]; then
283+
if [[ $GPU != 1 ]]; then
284284
ORT_BUILD=""
285285
else
286286
ORT_BUILD="-gpu"

‎opt/Makefile

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ fetch and build options:
2222
WITH_TFLITE=0 # SKip TensofFlowLite
2323
WITH_PT=0 # Skip PyTorch
2424
WITH_ORT=0 # SKip ONNXRuntime
25+
26+
device selection options (fetch, build, and test):
27+
CPU=1 # build for CPU
28+
GPU=1 # build for GPU
29+
CUDA=1 # build for GPU
2530
endef
2631

2732
#----------------------------------------------------------------------------------------------
@@ -127,17 +132,22 @@ endif
127132

128133
TEST_REPORT_DIR ?= $(PWD)
129134

135+
ifeq ($(TEST),)
136+
TEST=basic_tests.py
137+
endif
138+
130139
test:
131140
ifneq ($(NO_LFS),1)
132141
$(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi
133142
$(SHOW)cd $(ROOT); git lfs pull
134143
endif
135144
$(SHOW)set -e ;\
136145
cd $(ROOT)/test ;\
137-
python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so ;\
138-
python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so --use-slaves ;\
139-
python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so --use-aof
140-
# Commented until the MacOS CI is available
146+
export DEVICE=$(DEVICE) ;\
147+
python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so ;\
148+
python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-slaves ;\
149+
python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-aof
150+
# Commented until the MacOS CI is available
141151
# python3 -m RLTest $(TEST_ARGS) --test double-panda.py --module $(INSTALL_DIR)/redisai.so
142152

143153
#----------------------------------------------------------------------------------------------

‎opt/build/libtorch/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
ROOT=../../..
33

4-
VERSION ?= 1.2.0
4+
VERSION ?= 1.3.1
55
OSNICK ?= buster
66

77
#----------------------------------------------------------------------------------------------
@@ -100,7 +100,7 @@ $(eval $(call publish_arm,arm64v8,arm64))
100100
$(eval $(call publish_arm,arm32v7,arm))
101101

102102
repack:
103-
@PT_VERSION=$(VERSION) ./repack.sh
103+
@PT_VERSION=$(VERSION) GPU=$(GPU) ./repack.sh
104104

105105
help:
106106
@echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1]"

0 commit comments

Comments
 (0)