Add scripts to build all package manually

.gitignore

@@ -0,0 +1,2 @@
+scripts/dist
+*.deb

scripts/README.md

@@ -0,0 +1,25 @@
+# Scripts for creating custom memgraph packages and docker images
+
+Run `{script}.sh -h` to figure the details and how to start building.
+
+## Notes
+
+* If you need builder image for any supported operating system, please take a
+  look under `memgraph/release/package/run.sh`. Under the
+  `memgraph/release/package/` there are Dockerfiles to build builder
+  containers.
+* Lab is a private repo so this build is unavailable to public
+  (`docker_image_platform.sh`). You can still build Memgraph + MAGE and use
+  downloaded Lab with your image.
+
+## Loading custom docker image
+
+Docker image can be loaded with (it takes some time):
+```
+docker load -i <IMAGE_NAME>.tar.gz
+```
+and then used for example like this:
+```
+docker run -it --rm -p 7687:7687 --name <NAME> <IMAGE_NAME>
+```
+Platform image has a specific run command, please refer to the root of this repo.

scripts/build_memgraph_native.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# NOTE: -u is not an option here because toolchain activate fails (a better
+# ZSH_NAME check required).
+set -eo pipefail
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+MGPLAT_TOOLCHAIN_ROOT="${MGPLAT_TOOLCHAIN_ROOT:-/opt/toolchain-v4}"
+MGPLAT_MG_ROOT="${MGPLAT_MG_ROOT:-$DIR/../mage/cpp/memgraph}"
+MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}"
+MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}"
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  MGPLAT_CORES="${MGPLAT_CORES:-$(nproc)}"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  MGPLAT_CORES="${MGPLAT_CORES:-$(sysctl -n hw.physicalcpu)}"
+else
+  MGPLAT_CORES="${MGPLAT_CORES:-8}"
+fi
+declare -A MGPLAT_CPACK
+MGPLAT_CPACK[ubuntu]="cpack -G DEB --config ../CPackConfig.cmake"
+MGPLAT_CPACK[debian]="cpack -G DEB --config ../CPackConfig.cmake"
+MGPLAT_DIST_BINARY="$DIR/dist/binary"
+print_help() {
+  echo -e "Builds memgraph on natively supported operating systems."
+  echo -e "Can be used natively or under Docker."
+  echo -e ""
+  echo -e "Env vars:"
+  echo -e "  MGPLAT_TOOLCHAIN_ROOT -> root directory of the toolchain"
+  echo -e "  MGPLAT_MG_ROOT        -> root directory of memgraph"
+  echo -e "  MGPLAT_MG_TAG         -> git ref/branch of memgraph to build"
+  echo -e "  MGPLAT_MG_BUILD_TYPE  -> Debug|Release|RelWithDebInfo"
+  echo -e "  MGPLAT_CORES          -> number of cores to build memgraph"
+  echo -e ""
+  echo -e "How to run?"
+  echo -e " $0 [build|copy_binary|copy_package|cleanup]"
+  exit 1
+}
+
+pull_memgraph_and_os() {
+  cd "$MGPLAT_MG_ROOT"
+  git checkout "$MGPLAT_MG_TAG"
+  git pull origin "$MGPLAT_MG_TAG"
+  # shellcheck disable=SC1091
+  source "$MGPLAT_MG_ROOT/environment/util.sh"
+  # Required to get the underlying opearting system
+  OPERATING_SYSTEM_FAMILY="$(operating_system | cut -d "-" -f 1)"
+}
+
+build() {
+  pull_memgraph_and_os
+  if [ "$(architecture)" = "arm64" ] || [ "$(architecture)" = "aarch64" ]; then
+    OS_SCRIPT="$MGPLAT_MG_ROOT/environment/os/$(operating_system)-arm.sh"
+  else
+    OS_SCRIPT="$MGPLAT_MG_ROOT/environment/os/$(operating_system).sh"
+  fi
+  if [ "$EUID" -eq 0 ]; then # sudo or root -> it should be possible to just install deps
+    "$OS_SCRIPT" install TOOLCHAIN_RUN_DEPS
+    "$OS_SCRIPT" install MEMGRAPH_BUILD_DEPS
+  else # regular user -> privilege escalation required to install the system level deps
+    sudo "$OS_SCRIPT" install TOOLCHAIN_RUN_DEPS
+    sudo "$OS_SCRIPT" install MEMGRAPH_BUILD_DEPS
+  fi
+  # shellcheck disable=SC1091
+  source "$MGPLAT_TOOLCHAIN_ROOT/activate"
+  ./init
+  mkdir -p build && cd build
+  if [ "$(architecture)" = "arm64" ] || [ "$(architecture)" = "aarch64" ]; then
+    cmake -DCMAKE_BUILD_TYPE="$MGPLAT_MG_BUILD_TYPE" -DMG_ARCH="ARM64" ..
+  else
+    cmake -DCMAKE_BUILD_TYPE="$MGPLAT_MG_BUILD_TYPE" ..
+  fi
+  make -j"$MGPLAT_CORES" && make -j"$MGPLAT_CORES" mgconsole
+  mkdir -p output && cd output
+  ${MGPLAT_CPACK[$OPERATING_SYSTEM_FAMILY]}
+}
+
+cleanup() {
+  cd "$MGPLAT_MG_ROOT"
+  rm -rf ./build/*
+  ./libs/cleanup.sh
+}
+
+copy_binary() {
+  if [ ! -f "$MGPLAT_MG_ROOT/build/memgraph" ]; then
+    echo "Unable to find memgraph under the build folder"
+    exit 1
+  fi
+  binary_name="$(basename $(readlink "$MGPLAT_MG_ROOT"/build/memgraph))"
+  cp -L "$MGPLAT_MG_ROOT/build/memgraph" "$MGPLAT_DIST_BINARY/$binary_name"
+}
+
+copy_package() {
+  # NOTE: dist/package is mounted -> root is required -> think about better way
+  sudo cp "$MGPLAT_MG_ROOT"/build/output/memgraph* "$DIR/dist/package"
+}
+
+if [ "$#" == 0 ]; then
+  # if we source this script to get access to the env vars.
+  echo "NOTE: running $0 with 0 args -> pass"
+else
+  case "$1" in
+    build)
+      build
+    ;;
+    copy_binary)
+      copy_binary
+    ;;
+    copy_package)
+      copy_package
+    ;;
+    cleanup)
+      clean
+    ;;
+    *)
+      print_help
+    ;;
+  esac
+fi

scripts/docker_image_mage.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -eo pipefail
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+MAGE_DIR="$DIR/../mage"
+print_help() {
+  echo -e "Builds memgraph mage Docker image."
+  echo -e ""
+  echo -e "How to run?"
+  echo -e "  $0 [-h|build src_package_path image_name]"
+  exit 1
+}
+
+build() {
+  src_package="$1"
+  image_name="$2"
+  package_file="$(basename $src_package)"
+  mage_package_file="memgraph-${package_file#memgraph_}"
+  package_file_name="${package_file%.*}"
+  target_arch="${package_file_name#memgraph_}"
+  arch_suffix="${target_arch##*_}"
+  cp "$src_package" \
+     "$MAGE_DIR/$mage_package_file"
+  cd ${MAGE_DIR}
+  docker buildx build --target prod --platform="linux/$arch_suffix" -t "$image_name" --build-arg TARGETARCH="$target_arch" -f "$MAGE_DIR/Dockerfile.release" .
+  mkdir -p "$DIR/dist/docker"
+  docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz"
+}
+
+if [ "$#" == 0 ]; then
+  print_help
+else
+  case "$1" in
+    build)
+      if [ "$#" -ne 3 ]; then
+        print_help
+      fi
+      build "$2" "$3"
+    ;;
+    *)
+      print_help
+    ;;
+  esac
+fi

scripts/docker_image_memgraph.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -eo pipefail
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker"
+print_help() {
+  echo -e "Builds memgraph Docker image."
+  echo -e ""
+  echo -e "How to run?"
+  echo -e "  $0 [-h|build src_package_path]"
+  exit 1
+}
+
+build() {
+  src_package="$1"
+  package_file="$(basename $src_package)"
+  package_file_no_prefix="${package_file#memgraph_}"
+  memgraph_version="${package_file_no_prefix%-1*}"
+  dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')"
+  $MEMGRAPH_DOCKER_DIR/package_docker "$src_package"
+  mkdir -p "$DIR/dist/docker"
+  cp "$MEMGRAPH_DOCKER_DIR/memgraph-${dockerize_memgraph_version}-docker.tar.gz" "$DIR/dist/docker"
+}
+
+if [ "$#" == 0 ]; then
+  print_help
+else
+  case "$1" in
+    build)
+      if [ "$#" -ne 2 ]; then
+        print_help
+      fi
+      build "$2"
+    ;;
+    *)
+      print_help
+    ;;
+  esac
+fi

scripts/docker_image_platform.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eo pipefail
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+MGPLAT_ROOT="$DIR/../"
+MGPLAT_GHA_PAT_TOKEN="${MGPLAT_GHA_PAT_TOKEN:-github_personal_access_token}"
+print_help() {
+  echo -e "Builds memgraph platform Docker image."
+  echo -e ""
+  echo -e "Env vars:"
+  echo -e "  MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package"
+  echo -e ""
+  echo -e "How to run?"
+  echo -e "  $0 [-h|build src_package_path image_name]"
+  exit 1
+}
+
+# TODO(gitbuda): An option to build wihout mage.
+build() {
+  src_package="$1"
+  image_name="$2"
+  package_file="$(basename $src_package)"
+  platform_package_file="memgraph-${package_file#memgraph_}"
+  package_file_name="${package_file%.*}"
+  target_arch="${package_file_name#memgraph_}"
+  arch_suffix="${target_arch##*_}"
+  cp "$src_package" \
+     "$MGPLAT_ROOT/$platform_package_file"
+  cd "$MGPLAT_ROOT"
+  docker buildx build --platform="linux/$arch_suffix" -t ${image_name} \
+    --build-arg TARGETARCH="$target_arch" \
+    --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \
+    -f Dockerfile .
+  mkdir -p "$DIR/dist/docker"
+  docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz"
+}
+
+if [ "$#" == 0 ]; then
+  print_help
+else
+  case "$1" in
+    build)
+      if [ "$#" -ne 3 ]; then
+        print_help
+      fi
+      build "$2" "$3"
+    ;;
+    *)
+      print_help
+    ;;
+  esac
+fi

scripts/pack_memgraph_via_docker.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -eo pipefail
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# NOTE: The builder container image defines for which operating system Memgraph will be built.
+MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-10}"
+MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}"
+MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}"
+MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}"
+MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}"
+MGPLAT_DIST_BINARY="$DIR/dist/binary"
+print_help() {
+  echo -e "Builds memgraph binary and package via Docker build container."
+  echo -e ""
+  echo -e "Env vars:"
+  echo -e "  MGPLAT_CNT_IMAGE     -> Docker image used to build and pack memgraph"
+  echo -e "  MGPLAT_CNT_NAME      -> the name of builder Docker container"
+  echo -e "  MGPLAT_CNT_MG_ROOT   -> memgraph root directory inside the container"
+  echo -e "  MGPLAT_MG_TAG        -> git ref/branch of memgraph to build"
+  echo -e "  MGPLAT_MG_BUILD_TYPE -> Debug|Release|RelWithDebInfo"
+  echo -e ""
+  echo -e "How to run?"
+  echo -e "  $0 [pack|cleanup|copy_binary]"
+  exit 1
+}
+
+docker_run () {
+  cnt_name="$1"
+  cnt_image="$2"
+  if [ ! "$(docker ps -q -f name="$cnt_name")" ]; then
+      if [ "$(docker ps -aq -f status=exited -f name="$cnt_name")" ]; then
+          echo "Cleanup of the old exited container..."
+          docker rm "$cnt_name"
+      fi
+      docker run -d \
+        -v "$DIR/..:/platform" \
+        -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \
+        --network host --name "$cnt_name" "$cnt_image"
+  fi
+  echo "The $cnt_image container is active under $cnt_name name!"
+}
+
+docker_stop_rm() {
+  docker stop "$MGPLAT_CNT_NAME"
+  docker rm "$MGPLAT_CNT_NAME"
+}
+
+docker_exec() {
+  cnt_cmd="$1"
+  docker exec -it "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd"
+}
+
+build_pack() {
+  cd "$DIR"
+  # shellcheck disable=SC1091
+  source build_memgraph_native.sh
+  mkdir -p dist/binary
+  docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE"
+  docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/"
+  docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT"
+  mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT"
+  mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG"
+  mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE"
+  docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build"
+}
+
+cleanup() {
+  docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*"
+  docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh"
+}
+
+copy_binary() {
+  cnt_cmd="echo \$(readlink $MGPLAT_CNT_MG_ROOT/build/memgraph)"
+  cnt_binary_path=$(docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd")
+  binary_name="$(basename $cnt_binary_path)"
+  src_cnt_binary_path="$MGPLAT_CNT_NAME:$cnt_binary_path"
+  docker cp -L "$src_cnt_binary_path" "$DIR/dist/binary/"
+}
+
+if [ "$#" == 0 ]; then
+  build_pack
+else
+  case "$1" in
+    pack)
+      build_pack
+    ;;
+    # NOTE: The output package might be deleted as well (from our mounted dir).
+    cleanup)
+      cleanup
+    ;;
+    # Useful if you need memgraph binary for a specific operating system, e.g.
+    # Debian 10 binary to run under Jepsen
+    copy_binary)
+      copy_binary
+    ;;
+    *)
+      print_help
+    ;;
+  esac
+fi