Merge branch 'master' into align_script_get_command

Author: alonre24

.circleci/config.yml

@@ -138,7 +138,7 @@ commands:
             - artifacts/*.tgz
             - artifacts/*.tar
       - store_artifacts:
-          path: tests/logs
+          path: tests/flow/logs
 
   valgrind-general-steps:
     parameters:
@@ -191,11 +191,10 @@ commands:
           name: Test
           command: |
             mkdir -p $HOME/tests
-            docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
+            docker run --gpus all -v $HOME/tests:/build/tests/flow/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 40m
       - store_artifacts:
-          path: tests/logs
-
+          path: /home/circleci/tests
 
 
 jobs:
@@ -254,8 +253,6 @@ jobs:
           root: bin/
           paths:
             - artifacts/*
-      - store_artifacts:
-          path: test/logs
 
   coverage:
     docker:
@@ -284,6 +281,8 @@ jobs:
             make -C opt test SHOW=1 COV=1 CLUSTER=1
             make -C opt cov-upload
           no_output_timeout: 30m
+      - store_artifacts:
+          path: tests/flow/logs
 
   valgrind:
     parameters:

Dockerfile.gpu-test

@@ -23,7 +23,12 @@ SHELL ["/bin/bash", "-c"]
 
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-
+ENV LANG=en_US.UTF-8
+RUN apt-get update
+RUN apt-get install -y locales && \
+    sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=$LANG
 WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 

docs/commands.md

@@ -606,7 +606,7 @@ redis> AI.TENSORGET result{tag} VALUES
 ```
 
 ### Redis Commands support.
-In RedisAI TorchScript now supports simple (non-blocking) Redis commnands via the `redis.execute` API. The following (usless) script gets a key name (`x{1}`), and an `int` value (3). First, the script `SET`s the value in the key. Next, the script `GET`s the value back from the key, and sets it in a tensor which is eventually stored under the key 'y{1}'. Note that the inputs are `str` and `int`. The script sets and gets the value and set it into a tensor.
+RedisAI TorchScript now supports simple (non-blocking) Redis commands via the `redis.execute` API. The following (useless) script gets a key name (`x{1}`), and an `int` value (3). First, the script `SET`s the value in the key. Next, the script `GET`s the value back from the key, and sets it in a tensor which is eventually stored under the key 'y{1}'. Note that the inputs are `str` and `int`. The script sets and gets the value and set it into a tensor.
 
 ```
 def redis_int_to_tensor(redis_value: int):
@@ -624,6 +624,30 @@ redis> AI.TENSORGET y{1} VALUES
 1) (integer) 3
 ```
 
+### RedisAI model execution support.
+RedisAI TorchScript also supports executing models which are stored in RedisAI by calling `redisAI.model_execute` command. 
+The command receives 3 inputs:
+1. model name (string)
+2. model inputs (List of torch.Tensor)
+3. number of model outputs (int)
+Return value - the model execution output tensors (List of torch.Tensor)
+The following script creates two tensors, and executes the (tensorflow) model which is stored under the name 'tf_mul{1}' with these two tensors as inputs.
+```
+def test_model_execute(keys:List[str]):
+    a = torch.tensor([[2.0, 3.0], [2.0, 3.0]])
+    b = torch.tensor([[2.0, 3.0], [2.0, 3.0]])
+    return redisAI.model_execute(keys[0], [a, b], 1) # assume keys[0] is the model name stored in RedisAI.
+```
+```
+redis> AI.SCRIPTEXECUTE redis_scripts{1} test_model_execute KEYS 1 {1} LIST_INPUTS 1 tf_mul{1} OUTPUTS 1 y{1}
+OK
+redis> AI.TENSORGET y{1} VALUES
+1) (float) 4
+2) (float) 9
+3) (float) 4
+4) (float) 9
+```
+
 !!! warning "Intermediate memory overhead"
     The execution of scripts may generate intermediate tensors that are not allocated by the Redis allocator, but by whatever allocator is used in the backends (which may act on main memory or GPU memory, depending on the device), thus not being limited by `maxmemory` configuration settings of Redis.
 

get_deps.sh

@@ -47,10 +47,17 @@ else
 	fi
 fi
 
-git submodule update --init --recursive || true
+#git submodule update --init --recursive || true
+
+if [ -f ${HERE}/opt/readies/bin/platform ]; then
+    OS=$(python3 $HERE/opt/readies/bin/platform --os)
+    ARCH=$(python3 $HERE/opt/readies/bin/platform --arch)
+else
+    OS=`uname -s | tr '[:upper:]' '[:lower:]'`
+    uname -m|grep aarch64 || ARCH=x64
+    uname -m|grep x86 || ARCH=arm64v8
+fi
 
-OS=$(python3 $HERE/opt/readies/bin/platform --os)
-ARCH=$(python3 $HERE/opt/readies/bin/platform --arch)
 
 # avoid wget warnings on macOS
 [[ $OS == macos ]] && export LC_ALL=en_US.UTF-8
@@ -235,7 +242,7 @@ if [[ $WITH_PT != 0 ]]; then
 		LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz
 
 		if [[ $PT_REPACK == 1 ]]; then
-			PT_VERSION=$PT_VERSION GPU=$GPU $HERE/opt/build/libtorch/repack.sh
+			PT_VERSION=$PT_VERSION GPU=$GPU OS=${OS} ARCH=${ARCH} $HERE/opt/build/libtorch/repack.sh
 		else
 			LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE
 

opt/build/libtorch/repack.sh

@@ -6,7 +6,7 @@ set -e
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 ROOT=$HERE/../../..
-. $ROOT/opt/readies/shibumi/functions
+#. $ROOT/opt/readies/shibumi/functions
 ROOT=$(realpath $ROOT)
 
 if [[ "$1" == "cpu" || $CPU == 1 ]]; then
@@ -24,8 +24,13 @@ else
 	fi
 fi
 
-OS=$(python3 $ROOT/opt/readies/bin/platform --os)
-ARCH=$(python3 $ROOT/opt/readies/bin/platform --arch)
+# set them internally or externally
+if [ -z ${OS} ]; then
+    OS=$(python3 $ROOT/opt/readies/bin/platform --os)
+fi
+if [ -z ${ARCH} ]; then
+    ARCH=$(python3 $ROOT/opt/readies/bin/platform --arch)
+fi
 
 TARGET_DIR=$ROOT/deps/$OS-$ARCH-$DEVICE
 

src/backends/backedns_api.h

@@ -17,6 +17,7 @@
 #include "redismodule.h"
 #include "config/config.h"
 #include "execution/background_workers.h"
+#include "execution/execution_contexts/modelRun_ctx.h"
 
 static bool _ValidateFuncExists(RedisModuleCtx *ctx, void *func_ptr, const char *func_name,
                                 const char *backend_name, const char *path) {
@@ -40,6 +41,7 @@ static bool _ValidateFuncExists(RedisModuleCtx *ctx, void *func_ptr, const char
  */
 int RAI_ExportFunc(const char *func_name, void **targetFuncPtr) {
 
+    // Retrieve info from RedisAI internals.
     if (strcmp("GetThreadId", func_name) == 0) {
         *targetFuncPtr = BGWorker_GetThreadId;
     } else if (strcmp("GetNumThreadsPerQueue", func_name) == 0) {
@@ -48,6 +50,40 @@ int RAI_ExportFunc(const char *func_name, void **targetFuncPtr) {
         *targetFuncPtr = Config_GetModelExecutionTimeout;
     } else if (strcmp("GetThreadsCount", func_name) == 0) {
         *targetFuncPtr = BGWorker_GetThreadsCount;
+
+        // Export RedisAI low level API functions.
+    } else if (strcmp("RedisAI_InitError", func_name) == 0) {
+        *targetFuncPtr = RAI_InitError;
+    } else if (strcmp("RedisAI_FreeError", func_name) == 0) {
+        *targetFuncPtr = RAI_FreeError;
+    } else if (strcmp("RedisAI_GetError", func_name) == 0) {
+        *targetFuncPtr = RAI_GetError;
+    } else if (strcmp("RedisAI_TensorCreateFromDLTensor", func_name) == 0) {
+        *targetFuncPtr = RAI_TensorCreateFromDLTensor;
+    } else if (strcmp("RedisAI_TensorGetDLTensor", func_name) == 0) {
+        *targetFuncPtr = RAI_TensorGetDLTensor;
+    } else if (strcmp("RedisAI_TensorGetShallowCopy", func_name) == 0) {
+        *targetFuncPtr = RAI_TensorGetShallowCopy;
+    } else if (strcmp("RedisAI_TensorFree", func_name) == 0) {
+        *targetFuncPtr = RAI_TensorFree;
+    } else if (strcmp("RedisAI_GetModelFromKeyspace", func_name) == 0) {
+        *targetFuncPtr = RAI_GetModelFromKeyspace;
+    } else if (strcmp("RedisAI_ModelRunCtxCreate", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxCreate;
+    } else if (strcmp("RedisAI_ModelRunCtxAddInput", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxAddInput;
+    } else if (strcmp("RedisAI_ModelRunCtxNumOutputs", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxNumOutputs;
+    } else if (strcmp("RedisAI_ModelRunCtxAddOutput", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxAddOutput;
+    } else if (strcmp("RedisAI_ModelRunCtxOutputTensor", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxOutputTensor;
+    } else if (strcmp("RedisAI_ModelRunCtxFree", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRunCtxFree;
+    } else if (strcmp("RedisAI_ModelRun", func_name) == 0) {
+        *targetFuncPtr = RAI_ModelRun;
+
+        // Export RedisModule API functions.
     } else {
         return RedisModule_GetApi(func_name, targetFuncPtr);
     }
@@ -244,15 +280,15 @@ int RAI_LoadBackend_Torch(RedisModuleCtx *ctx, const char *path) {
 
     RAI_LoadedBackend backend = {0}; // Initialize all the callbacks to NULL.
 
-    int (*init_backend)(int (*)(const char *, void *));
-    init_backend = (int (*)(int (*)(const char *, void *)))(unsigned long)dlsym(
+    int (*init_backend)(int (*)(const char *, void **));
+    init_backend = (int (*)(int (*)(const char *, void **)))(unsigned long)dlsym(
         handle, "RAI_InitBackendTorch");
     if (!_ValidateFuncExists(ctx, init_backend, "RAI_InitBackendTorch", "TORCH", path)) {
         goto error;
     }
-    // Here we use the input callback to export functions from Redis to the backend,
-    // by setting the backend's function pointers to the corresponding functions in Redis.
-    init_backend(RedisModule_GetApi);
+    // Here we use the input callback to export functions from Redis and Redis AI to the backend,
+    // by setting the backend's function pointers to the corresponding functions in Redis/RedisAI.
+    init_backend(RAI_ExportFunc);
 
     backend.model_create =
         (RAI_Model * (*)(RAI_Backend, const char *, RAI_ModelOpts, const char *, size_t,

src/backends/backends.c

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <stdint.h>
+#include "redismodule.h"
+
+#ifdef BACKENDS_API_EXTERN
+#define BACKENDS_API extern
+#endif
+
+#ifndef BACKENDS_API
+#define BACKENDS_API
+#endif
+
+typedef struct RAI_Tensor RAI_Tensor;
+typedef struct RAI_Model RAI_Model;
+typedef struct RAI_ModelRunCtx RAI_ModelRunCtx;
+typedef struct RAI_Error RAI_Error;
+
+/**
+ * @return The internal id of RedisAI current working thread.
+ * id range is {0, ..., <threads_count>-1}. If this is called from a non
+ * RedisAI BG thread, return -1.
+ */
+BACKENDS_API long (*RedisAI_GetThreadId)(void);
+
+/**
+ * @return The number of working threads in RedisAI. This number should be
+ * equal to the number of threads per queue (load time config) * number of devices
+ * registered in RedisAI (a new device is registered if a model is set to run on
+ * this device in AI.MODELSTORE command.
+ */
+BACKENDS_API uintptr_t (*RedisAI_GetThreadsCount)(void);
+
+/**
+ * @return The number of working threads per device queue (load time config).
+ */
+BACKENDS_API long long (*RedisAI_GetNumThreadsPerQueue)(void);
+
+/**
+ * @return The maximal number of milliseconds that a model run session should run
+ * before it is terminated forcefully (load time config).
+ * Currently supported only fo onnxruntime backend.
+ */
+BACKENDS_API long long (*RedisAI_GetModelExecutionTimeout)(void);
+
+/**
+ * The following functions are part of RedisAI low level API (the full low level
+ * API is defined in redisai.h). For every function below named "RedisAI_X", its
+ * implementation can be found under the name "RAI_X" in RedisAI header files.
+ */
+
+BACKENDS_API int (*RedisAI_InitError)(RAI_Error **err);
+BACKENDS_API void (*RedisAI_FreeError)(RAI_Error *err);
+BACKENDS_API const char *(*RedisAI_GetError)(RAI_Error *err);
+
+BACKENDS_API RAI_Tensor *(*RedisAI_TensorCreateFromDLTensor)(DLManagedTensor *dl_tensor);
+BACKENDS_API DLTensor *(*RedisAI_TensorGetDLTensor)(RAI_Tensor *tensor);
+BACKENDS_API RAI_Tensor *(*RedisAI_TensorGetShallowCopy)(RAI_Tensor *t);
+BACKENDS_API void (*RedisAI_TensorFree)(RAI_Tensor *tensor);
+
+BACKENDS_API RAI_ModelRunCtx *(*RedisAI_ModelRunCtxCreate)(RAI_Model *model);
+BACKENDS_API int (*RedisAI_GetModelFromKeyspace)(RedisModuleCtx *ctx, RedisModuleString *keyName,
+                                                 RAI_Model **model, int mode, RAI_Error *err);
+BACKENDS_API int (*RedisAI_ModelRunCtxAddInput)(RAI_ModelRunCtx *mctx, const char *inputName,
+                                                RAI_Tensor *inputTensor);
+BACKENDS_API int (*RedisAI_ModelRunCtxAddOutput)(RAI_ModelRunCtx *mctx, const char *outputName);
+BACKENDS_API size_t (*RedisAI_ModelRunCtxNumOutputs)(RAI_ModelRunCtx *mctx);
+BACKENDS_API RAI_Tensor *(*RedisAI_ModelRunCtxOutputTensor)(RAI_ModelRunCtx *mctx, size_t index);
+BACKENDS_API void (*RedisAI_ModelRunCtxFree)(RAI_ModelRunCtx *mctx);
+BACKENDS_API int (*RedisAI_ModelRun)(RAI_ModelRunCtx **mctx, long long n, RAI_Error *err);