diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8876e2a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +scripts/dist +*.deb diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..2e2a2ec --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,25 @@ +# Scripts for creating custom memgraph packages and docker images + +Run `{script}.sh -h` to figure the details and how to start building. + +## Notes + +* If you need builder image for any supported operating system, please take a + look under `memgraph/release/package/run.sh`. Under the + `memgraph/release/package/` there are Dockerfiles to build builder + containers. +* Lab is a private repo so this build is unavailable to public + (`docker_image_platform.sh`). You can still build Memgraph + MAGE and use + downloaded Lab with your image. + +## Loading custom docker image + +Docker image can be loaded with (it takes some time): +``` +docker load -i .tar.gz +``` +and then used for example like this: +``` +docker run -it --rm -p 7687:7687 --name +``` +Platform image has a specific run command, please refer to the root of this repo. diff --git a/scripts/build_memgraph_native.sh b/scripts/build_memgraph_native.sh new file mode 100755 index 0000000..f68e0b8 --- /dev/null +++ b/scripts/build_memgraph_native.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# NOTE: -u is not an option here because toolchain activate fails (a better +# ZSH_NAME check required). +set -eo pipefail +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +MGPLAT_TOOLCHAIN_ROOT="${MGPLAT_TOOLCHAIN_ROOT:-/opt/toolchain-v4}" +MGPLAT_MG_ROOT="${MGPLAT_MG_ROOT:-$DIR/../mage/cpp/memgraph}" +MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" +MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + MGPLAT_CORES="${MGPLAT_CORES:-$(nproc)}" +elif [[ "$OSTYPE" == "darwin"* ]]; then + MGPLAT_CORES="${MGPLAT_CORES:-$(sysctl -n hw.physicalcpu)}" +else + MGPLAT_CORES="${MGPLAT_CORES:-8}" +fi +declare -A MGPLAT_CPACK +MGPLAT_CPACK[ubuntu]="cpack -G DEB --config ../CPackConfig.cmake" +MGPLAT_CPACK[debian]="cpack -G DEB --config ../CPackConfig.cmake" +MGPLAT_DIST_BINARY="$DIR/dist/binary" +print_help() { + echo -e "Builds memgraph on natively supported operating systems." + echo -e "Can be used natively or under Docker." + echo -e "" + echo -e "Env vars:" + echo -e " MGPLAT_TOOLCHAIN_ROOT -> root directory of the toolchain" + echo -e " MGPLAT_MG_ROOT -> root directory of memgraph" + echo -e " MGPLAT_MG_TAG -> git ref/branch of memgraph to build" + echo -e " MGPLAT_MG_BUILD_TYPE -> Debug|Release|RelWithDebInfo" + echo -e " MGPLAT_CORES -> number of cores to build memgraph" + echo -e "" + echo -e "How to run?" + echo -e " $0 [build|copy_binary|copy_package|cleanup]" + exit 1 +} + +pull_memgraph_and_os() { + cd "$MGPLAT_MG_ROOT" + git checkout "$MGPLAT_MG_TAG" + git pull origin "$MGPLAT_MG_TAG" + # shellcheck disable=SC1091 + source "$MGPLAT_MG_ROOT/environment/util.sh" + # Required to get the underlying opearting system + OPERATING_SYSTEM_FAMILY="$(operating_system | cut -d "-" -f 1)" +} + +build() { + pull_memgraph_and_os + if [ "$(architecture)" = "arm64" ] || [ "$(architecture)" = "aarch64" ]; then + OS_SCRIPT="$MGPLAT_MG_ROOT/environment/os/$(operating_system)-arm.sh" + else + OS_SCRIPT="$MGPLAT_MG_ROOT/environment/os/$(operating_system).sh" + fi + if [ "$EUID" -eq 0 ]; then # sudo or root -> it should be possible to just install deps + "$OS_SCRIPT" install TOOLCHAIN_RUN_DEPS + "$OS_SCRIPT" install MEMGRAPH_BUILD_DEPS + else # regular user -> privilege escalation required to install the system level deps + sudo "$OS_SCRIPT" install TOOLCHAIN_RUN_DEPS + sudo "$OS_SCRIPT" install MEMGRAPH_BUILD_DEPS + fi + # shellcheck disable=SC1091 + source "$MGPLAT_TOOLCHAIN_ROOT/activate" + ./init + mkdir -p build && cd build + if [ "$(architecture)" = "arm64" ] || [ "$(architecture)" = "aarch64" ]; then + cmake -DCMAKE_BUILD_TYPE="$MGPLAT_MG_BUILD_TYPE" -DMG_ARCH="ARM64" .. + else + cmake -DCMAKE_BUILD_TYPE="$MGPLAT_MG_BUILD_TYPE" .. + fi + make -j"$MGPLAT_CORES" && make -j"$MGPLAT_CORES" mgconsole + mkdir -p output && cd output + ${MGPLAT_CPACK[$OPERATING_SYSTEM_FAMILY]} +} + +cleanup() { + cd "$MGPLAT_MG_ROOT" + rm -rf ./build/* + ./libs/cleanup.sh +} + +copy_binary() { + if [ ! -f "$MGPLAT_MG_ROOT/build/memgraph" ]; then + echo "Unable to find memgraph under the build folder" + exit 1 + fi + binary_name="$(basename $(readlink "$MGPLAT_MG_ROOT"/build/memgraph))" + cp -L "$MGPLAT_MG_ROOT/build/memgraph" "$MGPLAT_DIST_BINARY/$binary_name" +} + +copy_package() { + # NOTE: dist/package is mounted -> root is required -> think about better way + sudo cp "$MGPLAT_MG_ROOT"/build/output/memgraph* "$DIR/dist/package" +} + +if [ "$#" == 0 ]; then + # if we source this script to get access to the env vars. + echo "NOTE: running $0 with 0 args -> pass" +else + case "$1" in + build) + build + ;; + copy_binary) + copy_binary + ;; + copy_package) + copy_package + ;; + cleanup) + clean + ;; + *) + print_help + ;; + esac +fi diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh new file mode 100755 index 0000000..a57b71b --- /dev/null +++ b/scripts/docker_image_mage.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -eo pipefail +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +MAGE_DIR="$DIR/../mage" +print_help() { + echo -e "Builds memgraph mage Docker image." + echo -e "" + echo -e "How to run?" + echo -e " $0 [-h|build src_package_path image_name]" + exit 1 +} + +build() { + src_package="$1" + image_name="$2" + package_file="$(basename $src_package)" + mage_package_file="memgraph-${package_file#memgraph_}" + package_file_name="${package_file%.*}" + target_arch="${package_file_name#memgraph_}" + arch_suffix="${target_arch##*_}" + cp "$src_package" \ + "$MAGE_DIR/$mage_package_file" + cd ${MAGE_DIR} + docker buildx build --target prod --platform="linux/$arch_suffix" -t "$image_name" --build-arg TARGETARCH="$target_arch" -f "$MAGE_DIR/Dockerfile.release" . + mkdir -p "$DIR/dist/docker" + docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" +} + +if [ "$#" == 0 ]; then + print_help +else + case "$1" in + build) + if [ "$#" -ne 3 ]; then + print_help + fi + build "$2" "$3" + ;; + *) + print_help + ;; + esac +fi diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh new file mode 100755 index 0000000..2c39217 --- /dev/null +++ b/scripts/docker_image_memgraph.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -eo pipefail +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker" +print_help() { + echo -e "Builds memgraph Docker image." + echo -e "" + echo -e "How to run?" + echo -e " $0 [-h|build src_package_path]" + exit 1 +} + +build() { + src_package="$1" + package_file="$(basename $src_package)" + package_file_no_prefix="${package_file#memgraph_}" + memgraph_version="${package_file_no_prefix%-1*}" + dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')" + $MEMGRAPH_DOCKER_DIR/package_docker "$src_package" + mkdir -p "$DIR/dist/docker" + cp "$MEMGRAPH_DOCKER_DIR/memgraph-${dockerize_memgraph_version}-docker.tar.gz" "$DIR/dist/docker" +} + +if [ "$#" == 0 ]; then + print_help +else + case "$1" in + build) + if [ "$#" -ne 2 ]; then + print_help + fi + build "$2" + ;; + *) + print_help + ;; + esac +fi diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh new file mode 100755 index 0000000..15054c8 --- /dev/null +++ b/scripts/docker_image_platform.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -eo pipefail +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +MGPLAT_ROOT="$DIR/../" +MGPLAT_GHA_PAT_TOKEN="${MGPLAT_GHA_PAT_TOKEN:-github_personal_access_token}" +print_help() { + echo -e "Builds memgraph platform Docker image." + echo -e "" + echo -e "Env vars:" + echo -e " MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package" + echo -e "" + echo -e "How to run?" + echo -e " $0 [-h|build src_package_path image_name]" + exit 1 +} + +# TODO(gitbuda): An option to build wihout mage. +build() { + src_package="$1" + image_name="$2" + package_file="$(basename $src_package)" + platform_package_file="memgraph-${package_file#memgraph_}" + package_file_name="${package_file%.*}" + target_arch="${package_file_name#memgraph_}" + arch_suffix="${target_arch##*_}" + cp "$src_package" \ + "$MGPLAT_ROOT/$platform_package_file" + cd "$MGPLAT_ROOT" + docker buildx build --platform="linux/$arch_suffix" -t ${image_name} \ + --build-arg TARGETARCH="$target_arch" \ + --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ + -f Dockerfile . + mkdir -p "$DIR/dist/docker" + docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" +} + +if [ "$#" == 0 ]; then + print_help +else + case "$1" in + build) + if [ "$#" -ne 3 ]; then + print_help + fi + build "$2" "$3" + ;; + *) + print_help + ;; + esac +fi diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh new file mode 100755 index 0000000..6bdae82 --- /dev/null +++ b/scripts/pack_memgraph_via_docker.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -eo pipefail +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# NOTE: The builder container image defines for which operating system Memgraph will be built. +MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-10}" +MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" +MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" +MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" +MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" +MGPLAT_DIST_BINARY="$DIR/dist/binary" +print_help() { + echo -e "Builds memgraph binary and package via Docker build container." + echo -e "" + echo -e "Env vars:" + echo -e " MGPLAT_CNT_IMAGE -> Docker image used to build and pack memgraph" + echo -e " MGPLAT_CNT_NAME -> the name of builder Docker container" + echo -e " MGPLAT_CNT_MG_ROOT -> memgraph root directory inside the container" + echo -e " MGPLAT_MG_TAG -> git ref/branch of memgraph to build" + echo -e " MGPLAT_MG_BUILD_TYPE -> Debug|Release|RelWithDebInfo" + echo -e "" + echo -e "How to run?" + echo -e " $0 [pack|cleanup|copy_binary]" + exit 1 +} + +docker_run () { + cnt_name="$1" + cnt_image="$2" + if [ ! "$(docker ps -q -f name="$cnt_name")" ]; then + if [ "$(docker ps -aq -f status=exited -f name="$cnt_name")" ]; then + echo "Cleanup of the old exited container..." + docker rm "$cnt_name" + fi + docker run -d \ + -v "$DIR/..:/platform" \ + -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ + --network host --name "$cnt_name" "$cnt_image" + fi + echo "The $cnt_image container is active under $cnt_name name!" +} + +docker_stop_rm() { + docker stop "$MGPLAT_CNT_NAME" + docker rm "$MGPLAT_CNT_NAME" +} + +docker_exec() { + cnt_cmd="$1" + docker exec -it "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd" +} + +build_pack() { + cd "$DIR" + # shellcheck disable=SC1091 + source build_memgraph_native.sh + mkdir -p dist/binary + docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" + docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" + docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" + mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" + mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" + mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" + docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build" +} + +cleanup() { + docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" + docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" +} + +copy_binary() { + cnt_cmd="echo \$(readlink $MGPLAT_CNT_MG_ROOT/build/memgraph)" + cnt_binary_path=$(docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd") + binary_name="$(basename $cnt_binary_path)" + src_cnt_binary_path="$MGPLAT_CNT_NAME:$cnt_binary_path" + docker cp -L "$src_cnt_binary_path" "$DIR/dist/binary/" +} + +if [ "$#" == 0 ]; then + build_pack +else + case "$1" in + pack) + build_pack + ;; + # NOTE: The output package might be deleted as well (from our mounted dir). + cleanup) + cleanup + ;; + # Useful if you need memgraph binary for a specific operating system, e.g. + # Debian 10 binary to run under Jepsen + copy_binary) + copy_binary + ;; + *) + print_help + ;; + esac +fi