Skip to content

Commit dd2848a

Browse files
rafierafie
authored
[1.0] Added /opt/readies submodule (#426)
* Added submodule opt/readies * Build system sysnc
1 parent e3e3401 commit dd2848a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+403
-2163
lines changed

‎.circleci/config.yml

Lines changed: 178 additions & 103 deletions
Large diffs are not rendered by default.

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ venv*/
1616
*.tar.gz
1717
/VARIANT
1818

19+
# Docs API reference
20+
docs/api_reference.md
21+
1922
### Cmake auto tools
2023
cmake-build-debug
2124

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "opt/readies"]
2+
path = opt/readies
3+
url = https://github.com/RedisLabsModules/readies.git

‎Dockerfile

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,20 @@ ARG OS=debian:buster-slim
1111
# ARCH=x64|arm64v8|arm32v7
1212
ARG ARCH=x64
1313

14+
ARG PACK=0
15+
ARG TEST=0
16+
1417
#----------------------------------------------------------------------------------------------
1518
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
1619
FROM ${OS} AS builder
1720

21+
ARG OSNICK
22+
ARG OS
23+
ARG ARCH
24+
ARG REDIS_VER
25+
26+
RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
27+
1828
WORKDIR /build
1929
COPY --from=redis /usr/local/ /usr/local/
2030

@@ -30,18 +40,29 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
3040

3141
ARG BUILD_ARGS=""
3242
ADD ./ /build
33-
RUN make -C opt build $BUILD_ARGS SHOW=1
43+
RUN set -e ;\
44+
. ./opt/readies/bin/sourced ./profile.d ;\
45+
make -C opt build $BUILD_ARGS SHOW=1
3446

35-
ARG PACK=0
36-
ARG TEST=0
47+
ARG PACK
48+
ARG TEST
3749

50+
RUN mkdir -p bin/artifacts
3851
RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
39-
RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi
52+
53+
RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
4054

4155
#----------------------------------------------------------------------------------------------
4256
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK}
4357

44-
RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
58+
ARG OSNICK
59+
ARG OS
60+
ARG ARCH
61+
ARG REDIS_VER
62+
ARG PACK
63+
64+
RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
65+
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
4566

4667
ENV REDIS_MODULES /usr/lib/redis/modules
4768
ENV LD_LIBRARY_PATH $REDIS_MODULES
@@ -50,6 +71,8 @@ RUN mkdir -p $REDIS_MODULES/
5071

5172
COPY --from=builder /build/install-cpu/ $REDIS_MODULES/
5273

74+
COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
75+
5376
WORKDIR /data
5477
EXPOSE 6379
5578
CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]

‎Dockerfile.arm

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,24 @@ ARG OS=debian:buster-slim
1111
# ARCH=arm64v8|arm32v7
1212
ARG ARCH=arm64v8
1313

14+
ARG PACK=0
15+
ARG TEST=0
16+
1417
#----------------------------------------------------------------------------------------------
15-
FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} AS builder
18+
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
19+
FROM redisfab/xbuild:${ARCH}-${OS} AS builder
20+
21+
ARG OSNICK
22+
ARG OS
23+
ARG ARCH
24+
ARG REDIS_VER
1625

1726
RUN [ "cross-build-start" ]
1827

28+
RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}"
29+
1930
WORKDIR /build
31+
COPY --from=redis /usr/local/ /usr/local/
2032

2133
COPY ./opt/ opt/
2234
COPY ./test/test_requirements.txt test/
@@ -30,13 +42,15 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
3042

3143
ARG BUILD_ARGS=""
3244
ADD ./ /build
33-
RUN make -C opt build $BUILD_ARGS SHOW=1
45+
RUN set -e ;\
46+
. ./opt/readies/bin/sourced ./profile.d ;\
47+
make -C opt build $BUILD_ARGS SHOW=1
3448

35-
ARG PACK=0
36-
ARG TEST=0
49+
ARG PACK
50+
ARG TEST
3751

3852
RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
39-
RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi
53+
RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
4054

4155
RUN [ "cross-build-end" ]
4256

@@ -45,7 +59,14 @@ FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK}
4559

4660
RUN [ "cross-build-start" ]
4761

48-
RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
62+
ARG OSNICK
63+
ARG OS
64+
ARG ARCH
65+
ARG REDIS_VER
66+
ARG PACK
67+
68+
RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
69+
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
4970

5071
ENV REDIS_MODULES /usr/lib/redis/modules
5172
ENV LD_LIBRARY_PATH $REDIS_MODULES

‎Dockerfile.gpu

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,25 @@ ARG OSNICK=bionic
88
# ARCH=x64|arm64v8|arm32v7
99
ARG ARCH=x64
1010

11+
# OS=ubuntu18.04|ubuntu16.04|centos7
12+
ARG OS=ubuntu18.04
13+
14+
ARG CUDA_VER=10.1-cudnn7
15+
16+
ARG PACK=0
17+
ARG TEST=0
18+
1119
#----------------------------------------------------------------------------------------------
1220
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
13-
FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder
21+
FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
22+
23+
ARG OSNICK
24+
ARG OS
25+
ARG REDIS_VER
26+
ARG CUDA_VER
27+
28+
RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]"
1429

15-
# ENV DEPS "git ca-certificates wget unzip cmake libgomp1 patchelf coreutils tcl libjemalloc-dev"
1630
ENV NVIDIA_VISIBLE_DEVICES all
1731
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
1832

@@ -31,24 +45,34 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_
3145

3246
ARG BUILD_ARGS=""
3347
ADD ./ /build
34-
RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
48+
RUN set -e ;\
49+
. ./opt/readies/bin/sourced ./profile.d ;\
50+
make -C opt build GPU=1 $BUILD_ARGS SHOW=1
3551

36-
ARG PACK=0
37-
ARG TEST=0
52+
ARG PACK
53+
ARG TEST
3854

55+
RUN mkdir -p bin/artifacts
3956
RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
40-
RUN if [ "$TEST" = "1" ]; then TEST="" make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
57+
58+
RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
4159

4260
#----------------------------------------------------------------------------------------------
43-
FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
61+
FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}
4462

45-
RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
63+
ARG OS
64+
65+
RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
66+
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
4667

4768
ENV REDIS_MODULES /usr/lib/redis/modules
4869
RUN mkdir -p $REDIS_MODULES/
4970

5071
COPY --from=redis /usr/local/ /usr/local/
5172
COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
73+
74+
COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
75+
5276
# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
5377

5478
WORKDIR /data

‎Dockerfile.gpu-test

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,15 @@ ARG OSNICK=bionic
88
# ARCH=x64|arm64v8|arm32v7
99
ARG ARCH=x64
1010

11+
# OS=ubuntu18.04|ubuntu16.04|centos7
12+
ARG OS=ubuntu18.04
13+
14+
ARG CUDA_VER=10.1-cudnn7
15+
16+
ARG PACK=1
1117
#----------------------------------------------------------------------------------------------
1218
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
13-
FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder
19+
FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
1420

1521
SHELL ["/bin/bash", "-c"]
1622

@@ -43,10 +49,11 @@ RUN set -e ;\
4349
ARG BUILD_ARGS=""
4450
ADD ./ /build
4551
RUN set -e ;\
46-
. venv/bin/activate ;\
47-
make -C opt build GPU=1 $BUILD_ARGS SHOW=1
52+
. venv/bin/activate ;\
53+
. ./opt/readies/bin/sourced ./profile.d ;\
54+
make -C opt build GPU=1 $BUILD_ARGS SHOW=1
4855

49-
ARG PACK=1
56+
ARG PACK
5057

5158
RUN set -e ;\
5259
if [[ $PACK == 1 ]]; then \

‎get_deps.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ LIBTORCH=libtorch
6666
MKL=mkl
6767
ONNXRUNTIME=onnxruntime
6868

69+
git submodule update --init --recursive || true
70+
6971
######################################################################################## DLPACK
7072

7173
if [[ $WITH_DLPACK != 0 ]]; then

‎hooks/post_checkout

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
# Docker hub does a recursive clone, then checks the branch out,
3+
# so when a PR adds a submodule (or updates it), it fails.
4+
git submodule update --init --recursive
5+

‎mkdocs.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,4 @@ plugins:
5252
- search
5353
- include:
5454
src_path: docs/snippets
55-
- versions-menu
56-
55+
- versions-menu:

0 commit comments

Comments
 (0)