From ba38a0dd5d458e59cfc20f54c07a44ceffb4adbe Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:30:00 +0100 Subject: [PATCH 01/67] Change default builder image --- scripts/pack_memgraph_via_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 6bdae82..72f9472 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -3,7 +3,7 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # NOTE: The builder container image defines for which operating system Memgraph will be built. -MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-10}" +MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-11}" MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" From d32d5a4823344c83c8a841d911f7a7ebb4100c74 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:32:10 +0100 Subject: [PATCH 02/67] Add daily build workflow --- .github/workflows/daily-platform-build.yml | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/daily-platform-build.yml diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml new file mode 100644 index 0000000..aab18ef --- /dev/null +++ b/.github/workflows/daily-platform-build.yml @@ -0,0 +1,55 @@ +name: Daily Platform Build + +on: + pull_request: + workflow_dispatch: + inputs: + build_type: + type: choice + description: "Build type" + default: 'ReleaseWithDebug' + options: + - Release + - ReleaseWithDebug + - Debug + toolchain_version: + type: choice + description: "Toolchain version" + default: 'v4' + options: + - v4 + - v5 + +jobs: + build_AMDx64: + name: "Build memgraph plaftorm AMDx64" + runs-on: [self-hosted, docker, X64] + env: + TARGET_OS: "debian-11" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Build memgraph" + run: | + cd platform/scripts + ./pack_memgraph_via_docker.sh pack + ./pack_memgraph_via_docker.sh copy_binary + ./pack_memgraph_via_docker.sh cleanup + env: + MGPLAT_CNT_IMAGE="memgraph/memgraph-builder:$TOOLCHAIN_VERSION_$TARGET_OS" + MGPLAT_CNT_NAME="mgbuilder_$TARGET_OS" + MGPLAT_CNT_MG_ROOT="/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG="master" + MGPLAT_MG_BUILD_TYPE=${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} + + - name: "Upload memgraph binary" + uses: actions/upload-artifact@v3 + with: + name: memgraph-debian-11.deb + path: platform/scripts/dist/binary/memgraph From 78fafa9726c6e27752f1bc1b8b4c0a90cca2f6f5 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:33:46 +0100 Subject: [PATCH 03/67] Fix environment definition --- .github/workflows/daily-platform-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index aab18ef..30ae745 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -42,11 +42,11 @@ jobs: ./pack_memgraph_via_docker.sh copy_binary ./pack_memgraph_via_docker.sh cleanup env: - MGPLAT_CNT_IMAGE="memgraph/memgraph-builder:$TOOLCHAIN_VERSION_$TARGET_OS" - MGPLAT_CNT_NAME="mgbuilder_$TARGET_OS" - MGPLAT_CNT_MG_ROOT="/platform/mage/cpp/memgraph" - MGPLAT_MG_TAG="master" - MGPLAT_MG_BUILD_TYPE=${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:$TOOLCHAIN_VERSION_$TARGET_OS" + MGPLAT_CNT_NAME: "mgbuilder_$TARGET_OS" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG: "master" + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 From 9986d440c9574498449b25f878689b2e0dd0e249 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:35:23 +0100 Subject: [PATCH 04/67] Fix path to script folder --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 30ae745..7c1d7b8 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -37,7 +37,7 @@ jobs: - name: "Build memgraph" run: | - cd platform/scripts + cd scripts ./pack_memgraph_via_docker.sh pack ./pack_memgraph_via_docker.sh copy_binary ./pack_memgraph_via_docker.sh cleanup @@ -52,4 +52,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph-debian-11.deb - path: platform/scripts/dist/binary/memgraph + path: scripts/dist/binary/memgraph From a76993073e32cbe02a35bc1abbf5d044bc7a974b Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:39:27 +0100 Subject: [PATCH 05/67] Fix using environment variables --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 7c1d7b8..6f8b321 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -42,8 +42,8 @@ jobs: ./pack_memgraph_via_docker.sh copy_binary ./pack_memgraph_via_docker.sh cleanup env: - MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:$TOOLCHAIN_VERSION_$TARGET_OS" - MGPLAT_CNT_NAME: "mgbuilder_$TARGET_OS" + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" + MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" MGPLAT_MG_TAG: "master" MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} From f766df6d875bfd093da6f03c8feef2d1c9591839 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 14:55:46 +0100 Subject: [PATCH 06/67] Add debug output --- scripts/pack_memgraph_via_docker.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 72f9472..aaec63e 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -56,12 +56,19 @@ build_pack() { source build_memgraph_native.sh mkdir -p dist/binary docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" + echo "--1--" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" + echo "--2--" docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" + echo "--3--" mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" + echo "--4--" mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" + echo "--5--" mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" + echo "--6--" docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build" + echo "--7--" } cleanup() { From 59acc5b08e205ade042b8ddc7ec04ceeabf92630 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 21 Nov 2023 15:00:11 +0100 Subject: [PATCH 07/67] Remove -it flag from docker exec commands --- scripts/pack_memgraph_via_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index aaec63e..c8b58e8 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -47,7 +47,7 @@ docker_stop_rm() { docker_exec() { cnt_cmd="$1" - docker exec -it "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd" + docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd" } build_pack() { From 9d13ca310c796d2768946057ff0400a79414254a Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 00:28:35 +0100 Subject: [PATCH 08/67] Run worklfow without cleanup --- .github/workflows/daily-platform-build.yml | 4 ++-- scripts/pack_memgraph_via_docker.sh | 12 +++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 6f8b321..328c697 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -23,7 +23,7 @@ on: jobs: build_AMDx64: name: "Build memgraph plaftorm AMDx64" - runs-on: [self-hosted, docker, X64] + runs-on: [self-hosted, Docker, X64] env: TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} @@ -40,7 +40,7 @@ jobs: cd scripts ./pack_memgraph_via_docker.sh pack ./pack_memgraph_via_docker.sh copy_binary - ./pack_memgraph_via_docker.sh cleanup + # ./pack_memgraph_via_docker.sh cleanup env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index c8b58e8..2fb3ed3 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -56,24 +56,18 @@ build_pack() { source build_memgraph_native.sh mkdir -p dist/binary docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" - echo "--1--" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" - echo "--2--" docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" - echo "--3--" mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" - echo "--4--" mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" - echo "--5--" mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" - echo "--6--" docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build" - echo "--7--" } cleanup() { - docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" - docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" + # docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" + # docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" + docker_stop_rm $MGPLAT_CNT_NAME } copy_binary() { From ff77f19b474fd1de80665420dbf8f0543f6c886f Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 15:52:56 +0100 Subject: [PATCH 09/67] Add mage and platform build --- .github/workflows/daily-platform-build.yml | 115 +++++++++++++++++++-- scripts/docker_image_mage.sh | 27 +++-- scripts/docker_image_platform.sh | 28 +++-- 3 files changed, 133 insertions(+), 37 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 328c697..0bb211b 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -19,14 +19,25 @@ on: options: - v4 - v5 + memgraph_tag: + description: "Memgraph version (choose master to use the latest source)" + default: 'master' + mage_tag: + description: "Mage version (choose main to use the latest source)" + default: 'main' + lab_tag: + description: "Lab version (choose dev to use the latest source)" + default: 'dev' jobs: - build_AMDx64: - name: "Build memgraph plaftorm AMDx64" + build_amd64: + name: "Build memgraph plaftorm amd64" runs-on: [self-hosted, Docker, X64] env: + ARCHITECTURE: "amd64" TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -35,21 +46,113 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git checkout ${{ github.event.inputs.memgraph_tag }} && cd ../.. + git checkout${{ github.event.inputs.mage_tag }} && cd .. + cd lab && git checkout ${{ github.event.inputs.lab_tag }} && cd .. + - name: "Build memgraph" run: | cd scripts ./pack_memgraph_via_docker.sh pack ./pack_memgraph_via_docker.sh copy_binary - # ./pack_memgraph_via_docker.sh cleanup + ./pack_memgraph_via_docker.sh cleanup env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" - MGPLAT_MG_TAG: "master" + MGPLAT_MG_TAG: ${{ github.event.inputs.memgraph_tag }} MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 with: - name: memgraph-debian-11.deb - path: scripts/dist/binary/memgraph + name: memgraph_${{ env.TARGET_OS }}.deb + path: ${{ env.MG_BINARY_PATH }} + + - name: "Build Mage" + run: | + cd scripts + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }}} + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + + - name: "Build memgraph platform" + run: | + cd scripts + ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + + build_arm64: + name: "Build memgraph plaftorm arm64" + runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + env: + ARCHITECTURE: "arm64" + TARGET_OS: "debian-11-arm" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git checkout ${{ github.event.inputs.memgraph_tag }} && cd ../.. + git checkout${{ github.event.inputs.mage_tag }} && cd .. + cd lab && git checkout ${{ github.event.inputs.lab_tag }} && cd .. + + - name: "Build memgraph" + run: | + cd scripts + ./pack_memgraph_via_docker.sh pack + ./pack_memgraph_via_docker.sh copy_binary + ./pack_memgraph_via_docker.sh cleanup + env: + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" + MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG: ${{ github.event.inputs.memgraph_tag }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} + + - name: "Upload memgraph binary" + uses: actions/upload-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + path: ${{ env.MG_BINARY_PATH }} + + - name: "Build Mage" + run: | + git checkout main + cd scripts + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }}} + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + + - name: "Build memgraph platform" + run: | + cd scripts + ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index a57b71b..fa95f96 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -7,24 +7,21 @@ print_help() { echo -e "Builds memgraph mage Docker image." echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name]" + echo -e " $0 [-h|build src_package_path image_name target_arch]" exit 1 } build() { - src_package="$1" - image_name="$2" - package_file="$(basename $src_package)" - mage_package_file="memgraph-${package_file#memgraph_}" - package_file_name="${package_file%.*}" - target_arch="${package_file_name#memgraph_}" - arch_suffix="${target_arch##*_}" - cp "$src_package" \ - "$MAGE_DIR/$mage_package_file" - cd ${MAGE_DIR} - docker buildx build --target prod --platform="linux/$arch_suffix" -t "$image_name" --build-arg TARGETARCH="$target_arch" -f "$MAGE_DIR/Dockerfile.release" . + src_package=$1 + image_name=$2 + target_arch=$3 + mage_package_file="memgraph-$target_arch.deb" + cp $src_package \ + $MAGE_DIR/$mage_package_file + cd $MAGE_DIR + docker buildx build --target prod --platform="linux/$target_arch" -t $image_name -f Dockerfile.release . mkdir -p "$DIR/dist/docker" - docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" + docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } if [ "$#" == 0 ]; then @@ -32,10 +29,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 3 ]; then + if [ "$#" -ne 4 ]; then print_help fi - build "$2" "$3" + build $2 $3 $4 ;; *) print_help diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 15054c8..4d66908 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -11,28 +11,24 @@ print_help() { echo -e " MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name]" + echo -e " $0 [-h|build src_package_path image_name target_arch]" exit 1 } # TODO(gitbuda): An option to build wihout mage. build() { - src_package="$1" - image_name="$2" - package_file="$(basename $src_package)" - platform_package_file="memgraph-${package_file#memgraph_}" - package_file_name="${package_file%.*}" - target_arch="${package_file_name#memgraph_}" - arch_suffix="${target_arch##*_}" - cp "$src_package" \ - "$MGPLAT_ROOT/$platform_package_file" - cd "$MGPLAT_ROOT" - docker buildx build --platform="linux/$arch_suffix" -t ${image_name} \ - --build-arg TARGETARCH="$target_arch" \ + src_package=$1 + image_name=$2 + target_arch=$3 + platform_package_file="memgraph-$target_arch.deb" + cp $src_package \ + $MGPLAT_ROOT/$platform_package_file + cd $MGPLAT_ROOT + docker buildx build --platform="linux/$target_arch" -t ${image_name} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ -f Dockerfile . mkdir -p "$DIR/dist/docker" - docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" + docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } if [ "$#" == 0 ]; then @@ -40,10 +36,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 3 ]; then + if [ "$#" -ne 4 ]; then print_help fi - build "$2" "$3" + build $2 $3 $4 ;; *) print_help From 40ac8b5ecaeb45052c06d38a6ad0bbe78f6c5135 Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 16:02:03 +0100 Subject: [PATCH 10/67] Fix env variables --- .github/workflows/daily-platform-build.yml | 36 +++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 0bb211b..95e3abf 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -29,6 +29,13 @@ on: description: "Lab version (choose dev to use the latest source)" default: 'dev' +env: + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} + jobs: build_amd64: name: "Build memgraph plaftorm amd64" @@ -36,8 +43,6 @@ jobs: env: ARCHITECTURE: "amd64" TARGET_OS: "debian-11" - TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -48,9 +53,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ github.event.inputs.memgraph_tag }} && cd ../.. - git checkout${{ github.event.inputs.mage_tag }} && cd .. - cd lab && git checkout ${{ github.event.inputs.lab_tag }} && cd .. + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. + git checkout${{ env.MAGE_TAG }} && cd .. + cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - name: "Build memgraph" run: | @@ -62,7 +67,7 @@ jobs: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" - MGPLAT_MG_TAG: ${{ github.event.inputs.memgraph_tag }} + MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} - name: "Upload memgraph binary" @@ -74,7 +79,7 @@ jobs: - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }}} + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -86,6 +91,8 @@ jobs: run: | cd scripts ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 @@ -99,8 +106,6 @@ jobs: env: ARCHITECTURE: "arm64" TARGET_OS: "debian-11-arm" - TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -111,9 +116,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ github.event.inputs.memgraph_tag }} && cd ../.. - git checkout${{ github.event.inputs.mage_tag }} && cd .. - cd lab && git checkout ${{ github.event.inputs.lab_tag }} && cd .. + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. + git checkout${{ env.MAGE_TAG }} && cd .. + cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - name: "Build memgraph" run: | @@ -125,7 +130,7 @@ jobs: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" - MGPLAT_MG_TAG: ${{ github.event.inputs.memgraph_tag }} + MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} - name: "Upload memgraph binary" @@ -136,9 +141,8 @@ jobs: - name: "Build Mage" run: | - git checkout main cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }}} + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -150,6 +154,8 @@ jobs: run: | cd scripts ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 From c4a8226e98dbdc0812d6be4a191ca3b9bfb6c42e Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 16:03:40 +0100 Subject: [PATCH 11/67] Fix missing bracket --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 95e3abf..f803b68 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -79,7 +79,7 @@ jobs: - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -142,7 +142,7 @@ jobs: - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE } ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 From e82176cc097969e48d8a1ed326705ef3a1f9ad3e Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 16:05:05 +0100 Subject: [PATCH 12/67] Fix typ --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index f803b68..82f9fab 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -54,7 +54,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. - git checkout${{ env.MAGE_TAG }} && cd .. + git checkout ${{ env.MAGE_TAG }} && cd .. cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - name: "Build memgraph" @@ -117,7 +117,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. - git checkout${{ env.MAGE_TAG }} && cd .. + git checkout ${{ env.MAGE_TAG }} && cd .. cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - name: "Build memgraph" From 2774a3b37c9a27640f6e927a2cc7b330bcad876c Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 16:40:13 +0100 Subject: [PATCH 13/67] Fix mismatched env variables --- .github/workflows/daily-platform-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 82f9fab..d6f8257 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -74,12 +74,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_BINARY_PATH }} + path: ${{ env.MG_BINARY_TARGET_PATH }} - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_BINARY_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -90,7 +90,7 @@ jobs: - name: "Build memgraph platform" run: | cd scripts - ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_platform ${{ env.MG_BINARY_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -137,12 +137,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_BINARY_PATH }} + path: ${{ env.MG_BINARY_TARGET_PATH }} - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_BINARY_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -153,7 +153,7 @@ jobs: - name: "Build memgraph platform" run: | cd scripts - ./docker_image_platform ${{ env.MG_BINARY_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_platform ${{ env.MG_BINARY_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} From 24d3bb0ae7016dbf297079556bdc207cfdaadd13 Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 20:14:56 +0100 Subject: [PATCH 14/67] Add command to rename memgraph deb pacakge after build --- .github/workflows/daily-platform-build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index d6f8257..6fba443 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -31,7 +31,7 @@ on: env: TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_BINARY_TARGET_PATH: "scripts/dist/binary/memgraph" + MG_PACKAGE_TARGET_PATH: "scripts/dist/binary/memgraph.deb" MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} @@ -61,8 +61,8 @@ jobs: run: | cd scripts ./pack_memgraph_via_docker.sh pack - ./pack_memgraph_via_docker.sh copy_binary ./pack_memgraph_via_docker.sh cleanup + mv dist/pacakge/memgraph*.deb dist/pacakge/memgraph.deb env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" @@ -74,12 +74,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_BINARY_TARGET_PATH }} + path: ${{ env.MG_PACKAGE_TARGET_PATH }} - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -90,7 +90,7 @@ jobs: - name: "Build memgraph platform" run: | cd scripts - ./docker_image_platform ${{ env.MG_BINARY_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -124,8 +124,8 @@ jobs: run: | cd scripts ./pack_memgraph_via_docker.sh pack - ./pack_memgraph_via_docker.sh copy_binary ./pack_memgraph_via_docker.sh cleanup + mv dist/pacakge/memgraph*.deb dist/pacakge/memgraph.deb env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" @@ -137,12 +137,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_BINARY_TARGET_PATH }} + path: ${{ env.MG_PACKAGE_TARGET_PATH }} - name: "Build Mage" run: | cd scripts - ./docker_image_mage.sh build ${{ env.MG_BINARY_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -153,7 +153,7 @@ jobs: - name: "Build memgraph platform" run: | cd scripts - ./docker_image_platform ${{ env.MG_BINARY_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} From 51eb422c19a894a9831b46eba39b7404b6d8975b Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 21:17:24 +0100 Subject: [PATCH 15/67] Fix pacakge typo ... --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 6fba443..9970084 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -62,7 +62,7 @@ jobs: cd scripts ./pack_memgraph_via_docker.sh pack ./pack_memgraph_via_docker.sh cleanup - mv dist/pacakge/memgraph*.deb dist/pacakge/memgraph.deb + mv dist/package/memgraph*.deb dist/package/memgraph.deb env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" @@ -125,7 +125,7 @@ jobs: cd scripts ./pack_memgraph_via_docker.sh pack ./pack_memgraph_via_docker.sh cleanup - mv dist/pacakge/memgraph*.deb dist/pacakge/memgraph.deb + mv dist/package/memgraph*.deb dist/package/memgraph.deb env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" From 746688481d13af0b3c57fbbf986731c73d9ad909 Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 23:43:47 +0100 Subject: [PATCH 16/67] Add copy_package and use it in workflow --- .github/workflows/daily-platform-build.yml | 1 + scripts/pack_memgraph_via_docker.sh | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 9970084..069fd41 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -124,6 +124,7 @@ jobs: run: | cd scripts ./pack_memgraph_via_docker.sh pack + ./pack_memgraph_via_docker.sh copy_package ./pack_memgraph_via_docker.sh cleanup mv dist/package/memgraph*.deb dist/package/memgraph.deb env: diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 2fb3ed3..0dc026b 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -9,6 +9,7 @@ MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" MGPLAT_DIST_BINARY="$DIR/dist/binary" +MGPLAT_DIST_PACKAGE="$DIR/dist/package" print_help() { echo -e "Builds memgraph binary and package via Docker build container." echo -e "" @@ -20,7 +21,7 @@ print_help() { echo -e " MGPLAT_MG_BUILD_TYPE -> Debug|Release|RelWithDebInfo" echo -e "" echo -e "How to run?" - echo -e " $0 [pack|cleanup|copy_binary]" + echo -e " $0 [pack|cleanup|copy_binary|copy_package]" exit 1 } @@ -33,8 +34,6 @@ docker_run () { docker rm "$cnt_name" fi docker run -d \ - -v "$DIR/..:/platform" \ - -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ --network host --name "$cnt_name" "$cnt_image" fi echo "The $cnt_image container is active under $cnt_name name!" @@ -70,6 +69,11 @@ cleanup() { docker_stop_rm $MGPLAT_CNT_NAME } +copy_package() { + src_cnt_package_path="$MGPLAT_CNT_NAME:$MGPLAT_CNT_MG_ROOT/build/output/." + docker cp $src_cnt_package_path $MGPLAT_DIST_PACKAGE +} + copy_binary() { cnt_cmd="echo \$(readlink $MGPLAT_CNT_MG_ROOT/build/memgraph)" cnt_binary_path=$(docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd") @@ -94,6 +98,9 @@ else copy_binary) copy_binary ;; + copy_package) + copy_package + ;; *) print_help ;; From 117eed0f5dbf9a971ef0e59423b4c18c4a231e74 Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 22 Nov 2023 23:46:00 +0100 Subject: [PATCH 17/67] Add volume --- scripts/pack_memgraph_via_docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 0dc026b..7544ac0 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -34,6 +34,7 @@ docker_run () { docker rm "$cnt_name" fi docker run -d \ + -v "$DIR/..:/platform" \ --network host --name "$cnt_name" "$cnt_image" fi echo "The $cnt_image container is active under $cnt_name name!" From 58425ac5d986e192c2ec42c80b4714148fe33c01 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 07:33:45 +0100 Subject: [PATCH 18/67] Add command to create dist/package --- scripts/pack_memgraph_via_docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 7544ac0..a51006a 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -55,6 +55,7 @@ build_pack() { # shellcheck disable=SC1091 source build_memgraph_native.sh mkdir -p dist/binary + mkdir -p dist/package docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" From af619049cc2b3f35ebc93dbe001f19dc799f2f51 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 08:40:07 +0100 Subject: [PATCH 19/67] Fix package target path --- .github/workflows/daily-platform-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 069fd41..84914e9 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -31,14 +31,14 @@ on: env: TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_PACKAGE_TARGET_PATH: "scripts/dist/binary/memgraph.deb" + MG_PACKAGE_TARGET_PATH: "scripts/dist/package/memgraph.deb" MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} jobs: build_amd64: - name: "Build memgraph plaftorm amd64" + name: "Build memgraph plaftorm for amd64" runs-on: [self-hosted, Docker, X64] env: ARCHITECTURE: "amd64" @@ -61,6 +61,7 @@ jobs: run: | cd scripts ./pack_memgraph_via_docker.sh pack + ./pack_memgraph_via_docker.sh copy_package ./pack_memgraph_via_docker.sh cleanup mv dist/package/memgraph*.deb dist/package/memgraph.deb env: @@ -101,7 +102,7 @@ jobs: path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz build_arm64: - name: "Build memgraph plaftorm arm64" + name: "Build memgraph plaftorm for arm64" runs-on: [self-hosted, DockerMgBuild, ARM64, strange] env: ARCHITECTURE: "arm64" From 08b7df29df8a13eadb0b4b8a3b2d1ab96a93c732 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 09:16:23 +0100 Subject: [PATCH 20/67] Fix package path --- .github/workflows/daily-platform-build.yml | 30 +++++++++------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 84914e9..66d09e3 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -59,11 +59,10 @@ jobs: - name: "Build memgraph" run: | - cd scripts - ./pack_memgraph_via_docker.sh pack - ./pack_memgraph_via_docker.sh copy_package - ./pack_memgraph_via_docker.sh cleanup - mv dist/package/memgraph*.deb dist/package/memgraph.deb + ./scripts/pack_memgraph_via_docker.sh pack + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup + mv scripts/dist/package/memgraph*.deb ${{ env.MG_PACKAGE_TARGET_PATH }} env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" @@ -79,8 +78,7 @@ jobs: - name: "Build Mage" run: | - cd scripts - ./docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -90,8 +88,7 @@ jobs: - name: "Build memgraph platform" run: | - cd scripts - ./docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -123,11 +120,10 @@ jobs: - name: "Build memgraph" run: | - cd scripts - ./pack_memgraph_via_docker.sh pack - ./pack_memgraph_via_docker.sh copy_package - ./pack_memgraph_via_docker.sh cleanup - mv dist/package/memgraph*.deb dist/package/memgraph.deb + ./scripts/pack_memgraph_via_docker.sh pack + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup + mv scripts/dist/package/memgraph*.deb ${{ env.MG_PACKAGE_TARGET_PATH }} env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" @@ -143,8 +139,7 @@ jobs: - name: "Build Mage" run: | - cd scripts - ./docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 @@ -154,8 +149,7 @@ jobs: - name: "Build memgraph platform" run: | - cd scripts - ./docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} From feb402840033d88b2f44fd27291e8684f8749892 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 10:57:09 +0100 Subject: [PATCH 21/67] Fix script call for building platform --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index 66d09e3..ace9c80 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -88,7 +88,7 @@ jobs: - name: "Build memgraph platform" run: | - ./scripts/docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform.sh ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -149,7 +149,7 @@ jobs: - name: "Build memgraph platform" run: | - ./scripts/docker_image_platform ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform.sh ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} From 56db8ef943dffc58b934285b2b743946a45e93b3 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 12:36:32 +0100 Subject: [PATCH 22/67] Fix docker_image_platform command call --- .github/workflows/daily-platform-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml index ace9c80..aebcc60 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-platform-build.yml @@ -88,7 +88,7 @@ jobs: - name: "Build memgraph platform" run: | - ./scripts/docker_image_platform.sh ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -149,7 +149,7 @@ jobs: - name: "Build memgraph platform" run: | - ./scripts/docker_image_platform.sh ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} From 95806b67fd33d979c7702e906f91d3dd9c68a7d0 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 16:58:51 +0100 Subject: [PATCH 23/67] Add back volume for dist/pacakage --- scripts/pack_memgraph_via_docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index a51006a..25b9698 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -35,6 +35,7 @@ docker_run () { fi docker run -d \ -v "$DIR/..:/platform" \ + -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ --network host --name "$cnt_name" "$cnt_image" fi echo "The $cnt_image container is active under $cnt_name name!" From 4723bdc932fdce0b6e3bdde751789872c9289165 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 16:59:10 +0100 Subject: [PATCH 24/67] Add support for builidng without mage --- scripts/docker_image_platform.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 4d66908..09952fd 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -11,7 +11,7 @@ print_help() { echo -e " MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name target_arch]" + echo -e " $0 [-h|build src_package_path image_name target_arch [--no-mage]]" exit 1 } @@ -24,9 +24,19 @@ build() { cp $src_package \ $MGPLAT_ROOT/$platform_package_file cd $MGPLAT_ROOT - docker buildx build --platform="linux/$target_arch" -t ${image_name} \ - --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ - -f Dockerfile . + if [ "$#" -eq 5 ]; then + if [[ "$4" == "--no-mage" ]]; then + docker buildx build --platform="linux/$target_arch" -t ${image_name} \ + --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ + -f memgraph_and_lab.Dockerfile . + else + print_help + fi + else + docker buildx build --platform="linux/$target_arch" -t ${image_name} \ + --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ + -f Dockerfile . + fi mkdir -p "$DIR/dist/docker" docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } @@ -36,7 +46,7 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 4 ]; then + if [ "$#" -lt 4 || "$#" -gt 5 ]; then print_help fi build $2 $3 $4 From c45802425a30dec8b836b276a0b8e138ce736b5b Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 16:59:50 +0100 Subject: [PATCH 25/67] Add parallel building --- ...latform-build.yml => daily-builds-amd.yml} | 148 ++++++++++-------- 1 file changed, 87 insertions(+), 61 deletions(-) rename .github/workflows/{daily-platform-build.yml => daily-builds-amd.yml} (56%) diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-builds-amd.yml similarity index 56% rename from .github/workflows/daily-platform-build.yml rename to .github/workflows/daily-builds-amd.yml index aebcc60..a9637b2 100644 --- a/.github/workflows/daily-platform-build.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -1,4 +1,4 @@ -name: Daily Platform Build +name: Daily builds for amd64 on: pull_request: @@ -30,19 +30,17 @@ on: default: 'dev' env: - TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_PACKAGE_TARGET_PATH: "scripts/dist/package/memgraph.deb" - MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + ARCHITECTURE: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} + TARGET_OS: "debian-11" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} jobs: - build_amd64: - name: "Build memgraph plaftorm for amd64" + build_memgraph: + name: "Build memgraph" runs-on: [self-hosted, Docker, X64] - env: - ARCHITECTURE: "amd64" - TARGET_OS: "debian-11" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -51,44 +49,55 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} - - name: "Checkout correct versions in submodules" - run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. - git checkout ${{ env.MAGE_TAG }} && cd .. - cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - - name: "Build memgraph" run: | ./scripts/pack_memgraph_via_docker.sh pack - ./scripts/pack_memgraph_via_docker.sh copy_package ./scripts/pack_memgraph_via_docker.sh cleanup - mv scripts/dist/package/memgraph*.deb ${{ env.MG_PACKAGE_TARGET_PATH }} env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} - MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_PACKAGE_TARGET_PATH }} + path: scripts/dist/package/memgraph*.deb - - name: "Build Mage" + build_memgraph_platform: + name: "Build memgraph plaftorm" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + env: + DOCKER_IMAGE_NAME: "memgraph-platform_${{ env.ARCHITECTURE }}" + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" run: | - ./scripts/docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. + git checkout ${{ env.MAGE_TAG }} && cd .. + cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - - name: "Upload mage docker image" - uses: actions/upload-artifact@v3 + - name: Download artifacts + uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph_${{ env.TARGET_OS }}.deb + path: . - name: "Build memgraph platform" run: | - ./scripts/docker_image_platform.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + ${{ env.DOCKER_IMAGE_NAME }} \ + ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -96,14 +105,14 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/${{ env.DOCKER_IMAGE_NAME }}.tar.gz - build_arm64: - name: "Build memgraph plaftorm for arm64" - runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + build_memgraph_platform_without mage: + name: "Build memgraph plaftorm without mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] env: - ARCHITECTURE: "arm64" - TARGET_OS: "debian-11-arm" + DOCKER_IMAGE_NAME: "memgraph-platform_${{ env.ARCHITECTURE }}" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -114,47 +123,64 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. - git checkout ${{ env.MAGE_TAG }} && cd .. cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - - name: "Build memgraph" + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + path: . + + - name: "Build memgraph platform" run: | - ./scripts/pack_memgraph_via_docker.sh pack - ./scripts/pack_memgraph_via_docker.sh copy_package - ./scripts/pack_memgraph_via_docker.sh cleanup - mv scripts/dist/package/memgraph*.deb ${{ env.MG_PACKAGE_TARGET_PATH }} + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + ${{ env.DOCKER_IMAGE_NAME }} \ + ${{ env.ARCHITECTURE }} \ + --no-mage env: - MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" - MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" - MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" - MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} - MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'ReleaseWithDebug' }} + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - - name: "Upload memgraph binary" + - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb - path: ${{ env.MG_PACKAGE_TARGET_PATH }} + name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/${{ DOCKER_IMAGE_NAME }}.tar.gz - - name: "Build Mage" + build_mage: + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + env: + DOCKER_IMAGE_NAME: "mage_${{ env.ARCHITECTURE }}" + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" run: | - ./scripts/docker_image_mage.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} mage_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - - name: "Upload mage docker image" - uses: actions/upload-artifact@v3 + - name: Download artifacts + uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph_${{ env.TARGET_OS }}.deb + path: . - - name: "Build memgraph platform" + - name: "Build Mage" run: | - ./scripts/docker_image_platform.sh build ${{ env.MG_PACKAGE_TARGET_PATH }} memgraph-platform_${{ env.ARCHITECTURE }} ${{ env.ARCHITECTURE }} - env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + ${{ env.DOCKER_IMAGE_NAME }} \ + ${{ env.ARCHITECTURE }} - - name: "Upload memgraph platform docker image" + - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/${{ env.DOCKER_IMAGE_NAME }}.tar.gz From f2579b23267725ec370ecf7de4d55c826b27dca8 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 17:08:19 +0100 Subject: [PATCH 26/67] Fix typos --- .github/workflows/daily-builds-amd.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index a9637b2..93738e4 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -70,8 +70,6 @@ jobs: name: "Build memgraph plaftorm" needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] - env: - DOCKER_IMAGE_NAME: "memgraph-platform_${{ env.ARCHITECTURE }}" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -96,7 +94,7 @@ jobs: run: | ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - ${{ env.DOCKER_IMAGE_NAME }} \ + memgraph-platform_${{ env.ARCHITECTURE }} \ ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -105,14 +103,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/${{ env.DOCKER_IMAGE_NAME }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz build_memgraph_platform_without mage: name: "Build memgraph plaftorm without mage" needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] - env: - DOCKER_IMAGE_NAME: "memgraph-platform_${{ env.ARCHITECTURE }}" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -135,7 +131,7 @@ jobs: run: | ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - ${{ env.DOCKER_IMAGE_NAME }} \ + memgraph-platform_${{ env.ARCHITECTURE }} \ ${{ env.ARCHITECTURE }} \ --no-mage env: @@ -145,14 +141,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/${{ DOCKER_IMAGE_NAME }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz build_mage: name: "Build mage" needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] - env: - DOCKER_IMAGE_NAME: "mage_${{ env.ARCHITECTURE }}" steps: - name: "Checkout repository and submodules" uses: actions/checkout@v3 @@ -175,12 +169,12 @@ jobs: - name: "Build Mage" run: | ./scripts/docker_image_mage.sh build \ + mage_${{ env.ARCHITECTURE }} \ memgraph_${{ env.TARGET_OS }}.deb \ - ${{ env.DOCKER_IMAGE_NAME }} \ ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/${{ env.DOCKER_IMAGE_NAME }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz From 0ee767d137f4e8c3fc032efa205b603c0f2e76d7 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 17:09:19 +0100 Subject: [PATCH 27/67] Fix typo --- .github/workflows/daily-builds-amd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 93738e4..361339e 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -105,7 +105,7 @@ jobs: name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - build_memgraph_platform_without mage: + build_memgraph_platform_without_mage: name: "Build memgraph plaftorm without mage" needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] From c29544fd8ae25bc84adc49af3da16dbb8fd2aaf8 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 17:45:26 +0100 Subject: [PATCH 28/67] Fix artifact download --- .github/workflows/daily-builds-amd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 361339e..babd3a5 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -88,7 +88,7 @@ jobs: uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: . + path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build memgraph platform" run: | @@ -125,7 +125,7 @@ jobs: uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: . + path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build memgraph platform" run: | @@ -164,7 +164,7 @@ jobs: uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: . + path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build Mage" run: | From 0ae3c203c16b981d5ae1714144de8277158c0c40 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 23 Nov 2023 19:05:42 +0100 Subject: [PATCH 29/67] Fix artifact donwload --- .github/workflows/daily-builds-amd.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index babd3a5..e3abc7d 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -84,14 +84,14 @@ jobs: git checkout ${{ env.MAGE_TAG }} && cd .. cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - - name: Download artifacts + - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build memgraph platform" run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ memgraph-platform_${{ env.ARCHITECTURE }} \ @@ -121,14 +121,14 @@ jobs: run: | cd lab && git checkout ${{ env.LAB_TAG }} && cd .. - - name: Download artifacts + - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build memgraph platform" run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ memgraph-platform_${{ env.ARCHITECTURE }} \ @@ -160,14 +160,14 @@ jobs: cd mage && git checkout ${{ env.MAGE_TAG }} cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - - name: Download artifacts + - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - path: ./memgraph_${{ env.TARGET_OS }}.deb - name: "Build Mage" run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_mage.sh build \ mage_${{ env.ARCHITECTURE }} \ memgraph_${{ env.TARGET_OS }}.deb \ From 32b51a6210d54eff6e0fc7c46c9b82a583ed982a Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 11:50:37 +0100 Subject: [PATCH 30/67] Change checkout order --- .github/workflows/daily-builds-amd.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index e3abc7d..e83a145 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -71,6 +71,7 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: + - name: - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -80,9 +81,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && cd ../.. - git checkout ${{ env.MAGE_TAG }} && cd .. - cd lab && git checkout ${{ env.LAB_TAG }} && cd .. + cd lab && git checkout ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -119,7 +120,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && cd .. + cd lab && git checkout ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 From b190dc935cd7e0cb170dbfb5a0730c4f26e4c72e Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 11:52:09 +0100 Subject: [PATCH 31/67] Fix typo --- .github/workflows/daily-builds-amd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index e83a145..3ac8d01 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -71,7 +71,6 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: - - name: - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: From 2fc13d1b2cf569f522f34e746e1b2424ed748fd8 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 14:15:34 +0100 Subject: [PATCH 32/67] Fix mage build, test only memgraph and mage build --- .github/workflows/daily-builds-amd.yml | 158 ++++++++++++------------- scripts/docker_image_platform.sh | 11 +- 2 files changed, 84 insertions(+), 85 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 3ac8d01..0f3bfbf 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -65,84 +65,84 @@ jobs: with: name: memgraph_${{ env.TARGET_OS }}.deb path: scripts/dist/package/memgraph*.deb - - build_memgraph_platform: - name: "Build memgraph plaftorm" - needs: [build_memgraph] - runs-on: [self-hosted, Docker, X64] - steps: - - name: "Checkout repository and submodules" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.PAT }} - - - name: "Checkout correct versions in submodules" - run: | - cd lab && git checkout ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - - - name: "Download memgraph package" - uses: actions/download-artifact@v3 - with: - name: memgraph_${{ env.TARGET_OS }}.deb - - - name: "Build memgraph platform" - run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} - env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - - - name: "Upload memgraph platform docker image" - uses: actions/upload-artifact@v3 - with: - name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - - build_memgraph_platform_without_mage: - name: "Build memgraph plaftorm without mage" - needs: [build_memgraph] - runs-on: [self-hosted, Docker, X64] - steps: - - name: "Checkout repository and submodules" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.PAT }} - - - name: "Checkout correct versions in submodules" - run: | - cd lab && git checkout ${{ env.LAB_TAG }} - - - name: "Download memgraph package" - uses: actions/download-artifact@v3 - with: - name: memgraph_${{ env.TARGET_OS }}.deb - - - name: "Build memgraph platform" - run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} \ - --no-mage - env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - - - name: "Upload memgraph platform docker image" - uses: actions/upload-artifact@v3 - with: - name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - + # + # build_memgraph_platform: + # name: "Build memgraph plaftorm" + # needs: [build_memgraph] + # runs-on: [self-hosted, Docker, X64] + # steps: + # - name: "Checkout repository and submodules" + # uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + # submodules: recursive + # token: ${{ secrets.PAT }} + # + # - name: "Checkout correct versions in submodules" + # run: | + # cd lab && git checkout ${{ env.LAB_TAG }} + # cd ../mage && git checkout ${{ env.MAGE_TAG }} + # cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + # + # - name: "Download memgraph package" + # uses: actions/download-artifact@v3 + # with: + # name: memgraph_${{ env.TARGET_OS }}.deb + # + # - name: "Build memgraph platform" + # run: | + # mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + # ./scripts/docker_image_platform.sh build \ + # memgraph_${{ env.TARGET_OS }}.deb \ + # memgraph-platform_${{ env.ARCHITECTURE }} \ + # ${{ env.ARCHITECTURE }} + # env: + # MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + # + # - name: "Upload memgraph platform docker image" + # uses: actions/upload-artifact@v3 + # with: + # name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + # path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + # + # build_memgraph_platform_without_mage: + # name: "Build memgraph plaftorm without mage" + # needs: [build_memgraph] + # runs-on: [self-hosted, Docker, X64] + # steps: + # - name: "Checkout repository and submodules" + # uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + # submodules: recursive + # token: ${{ secrets.PAT }} + # + # - name: "Checkout correct versions in submodules" + # run: | + # cd lab && git checkout ${{ env.LAB_TAG }} + # + # - name: "Download memgraph package" + # uses: actions/download-artifact@v3 + # with: + # name: memgraph_${{ env.TARGET_OS }}.deb + # + # - name: "Build memgraph platform" + # run: | + # mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + # ./scripts/docker_image_platform.sh build \ + # memgraph_${{ env.TARGET_OS }}.deb \ + # memgraph-platform_${{ env.ARCHITECTURE }} \ + # ${{ env.ARCHITECTURE }} \ + # --no-mage + # env: + # MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + # + # - name: "Upload memgraph platform docker image" + # uses: actions/upload-artifact@v3 + # with: + # name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + # path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + # build_mage: name: "Build mage" needs: [build_memgraph] @@ -169,8 +169,8 @@ jobs: run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_mage.sh build \ - mage_${{ env.ARCHITECTURE }} \ memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }} \ ${{ env.ARCHITECTURE }} - name: "Upload mage docker image" diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 09952fd..1f77030 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -26,17 +26,16 @@ build() { cd $MGPLAT_ROOT if [ "$#" -eq 5 ]; then if [[ "$4" == "--no-mage" ]]; then - docker buildx build --platform="linux/$target_arch" -t ${image_name} \ - --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ - -f memgraph_and_lab.Dockerfile . + dockerfile="memgraph_and_lab.Dockerfile" else print_help fi else - docker buildx build --platform="linux/$target_arch" -t ${image_name} \ - --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ - -f Dockerfile . + dockerfile="Dockerfile" fi + docker buildx build --platform="linux/$target_arch" -t ${image_name} \ + --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ + -f ${dockerfile} . mkdir -p "$DIR/dist/docker" docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } From 81adff283777bd976bc69b95c620e42f460501cd Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 17:25:18 +0100 Subject: [PATCH 33/67] Make platform build use prebuilt mage image --- .github/workflows/daily-builds-amd.yml | 162 +++++++++++++------------ Dockerfile | 47 +++---- scripts/docker_image_platform.sh | 4 +- 3 files changed, 110 insertions(+), 103 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 0f3bfbf..a611c8c 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -65,84 +65,90 @@ jobs: with: name: memgraph_${{ env.TARGET_OS }}.deb path: scripts/dist/package/memgraph*.deb - # - # build_memgraph_platform: - # name: "Build memgraph plaftorm" - # needs: [build_memgraph] - # runs-on: [self-hosted, Docker, X64] - # steps: - # - name: "Checkout repository and submodules" - # uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - # submodules: recursive - # token: ${{ secrets.PAT }} - # - # - name: "Checkout correct versions in submodules" - # run: | - # cd lab && git checkout ${{ env.LAB_TAG }} - # cd ../mage && git checkout ${{ env.MAGE_TAG }} - # cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - # - # - name: "Download memgraph package" - # uses: actions/download-artifact@v3 - # with: - # name: memgraph_${{ env.TARGET_OS }}.deb - # - # - name: "Build memgraph platform" - # run: | - # mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - # ./scripts/docker_image_platform.sh build \ - # memgraph_${{ env.TARGET_OS }}.deb \ - # memgraph-platform_${{ env.ARCHITECTURE }} \ - # ${{ env.ARCHITECTURE }} - # env: - # MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - # - # - name: "Upload memgraph platform docker image" - # uses: actions/upload-artifact@v3 - # with: - # name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - # path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - # - # build_memgraph_platform_without_mage: - # name: "Build memgraph plaftorm without mage" - # needs: [build_memgraph] - # runs-on: [self-hosted, Docker, X64] - # steps: - # - name: "Checkout repository and submodules" - # uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - # submodules: recursive - # token: ${{ secrets.PAT }} - # - # - name: "Checkout correct versions in submodules" - # run: | - # cd lab && git checkout ${{ env.LAB_TAG }} - # - # - name: "Download memgraph package" - # uses: actions/download-artifact@v3 - # with: - # name: memgraph_${{ env.TARGET_OS }}.deb - # - # - name: "Build memgraph platform" - # run: | - # mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - # ./scripts/docker_image_platform.sh build \ - # memgraph_${{ env.TARGET_OS }}.deb \ - # memgraph-platform_${{ env.ARCHITECTURE }} \ - # ${{ env.ARCHITECTURE }} \ - # --no-mage - # env: - # MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - # - # - name: "Upload memgraph platform docker image" - # uses: actions/upload-artifact@v3 - # with: - # name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - # path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - # + + build_memgraph_platform: + name: "Build memgraph plaftorm" + needs: [build_memgraph, build_mage] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git checkout ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: mage_${{ env.TARGET_OS }}.tar.gz + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + docker load < mage_${{ env.TARGET_OS }}.tar.gz + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + + build_memgraph_platform_without_mage: + name: "Build memgraph plaftorm without mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git checkout ${{ env.LAB_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} \ + --no-mage + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + build_mage: name: "Build mage" needs: [build_memgraph] diff --git a/Dockerfile b/Dockerfile index dd89448..c0e8a5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,30 +30,30 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ################################################################################################################################################### - +FROM mage_${TARGETARCH}:latest as mage-dev # MAGE -FROM base as mage-dev - -WORKDIR /mage -COPY mage /mage - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - -ENV PATH="/root/.cargo/bin:${PATH}" - -RUN python3 -m pip install --upgrade pip \ - && python3 -m pip install --default-timeout=1000 -r /mage/python/requirements.txt - -RUN python3 -m pip --default-timeout=1000 --no-cache-dir install torch-sparse torch-cluster torch-spline-conv \ - torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html - -RUN python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ - -# DGL build from source -RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git \ - && cd dgl && mkdir build && cd build && cmake .. \ - && make -j4 && cd ../python && python3 setup.py install - +# FROM base as mage-dev +# +# WORKDIR /mage +# COPY mage /mage +# +# RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +# +# ENV PATH="/root/.cargo/bin:${PATH}" +# +# RUN python3 -m pip install --upgrade pip \ +# && python3 -m pip install --default-timeout=1000 -r /mage/python/requirements.txt +# +# RUN python3 -m pip --default-timeout=1000 --no-cache-dir install torch-sparse torch-cluster torch-spline-conv \ +# torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html +# +# RUN python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ +# +# # DGL build from source +# RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git \ +# && cd dgl && mkdir build && cd build && cmake .. \ +# && make -j4 && cd ../python && python3 setup.py install +# ################################################################################################################################################### # Lab: Build backend @@ -91,6 +91,7 @@ RUN cd frontend && npm run build:production ################################################################################################################################################### + FROM base as final # Copy the backend artifacts diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 1f77030..bb6b352 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -24,15 +24,15 @@ build() { cp $src_package \ $MGPLAT_ROOT/$platform_package_file cd $MGPLAT_ROOT + dockerfile="Dockerfile" if [ "$#" -eq 5 ]; then if [[ "$4" == "--no-mage" ]]; then dockerfile="memgraph_and_lab.Dockerfile" else print_help fi - else - dockerfile="Dockerfile" fi + echo "-----Using dockerfile $dockerfile-----" docker buildx build --platform="linux/$target_arch" -t ${image_name} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ -f ${dockerfile} . From bf9707084d9db43cf1cb4ed0ec892a5a935817bf Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 19:17:58 +0100 Subject: [PATCH 34/67] Fix --no-mage flag --- scripts/docker_image_platform.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index bb6b352..7d91c64 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -25,8 +25,8 @@ build() { $MGPLAT_ROOT/$platform_package_file cd $MGPLAT_ROOT dockerfile="Dockerfile" - if [ "$#" -eq 5 ]; then - if [[ "$4" == "--no-mage" ]]; then + if [[ "$#" -eq 4 ]]; then + if [ "$4" == "--no-mage" ]; then dockerfile="memgraph_and_lab.Dockerfile" else print_help @@ -40,15 +40,15 @@ build() { docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } -if [ "$#" == 0 ]; then +if [[ "$#" -eq 0 ]]; then print_help else case "$1" in build) - if [ "$#" -lt 4 || "$#" -gt 5 ]; then + if [[ "$#" -lt 4 || "$#" -gt 5 ]]; then print_help fi - build $2 $3 $4 + build $2 $3 $4 $5 ;; *) print_help From f8241ff0366d3d1bb5c60a62f6f75c3c53bd706b Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 19:18:17 +0100 Subject: [PATCH 35/67] Fix mage artifact download --- .github/workflows/daily-builds-amd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index a611c8c..68e582a 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -92,12 +92,12 @@ jobs: - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.TARGET_OS }}.tar.gz + name: mage_${{ env.ARCHITECTURE }}.tar.gz - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.TARGET_OS }}.tar.gz + docker load < mage_${{ env.ARCHITECTURE }}.tar.gz ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ memgraph-platform_${{ env.ARCHITECTURE }} \ From 59938a988d974f4b703a611ba73d45dc4e362280 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 20:24:41 +0100 Subject: [PATCH 36/67] Fix crashing when using a machine after build memghraph job --- .github/workflows/daily-builds-amd.yml | 2 +- scripts/pack_memgraph_via_docker.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 68e582a..6054cea 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -52,7 +52,7 @@ jobs: - name: "Build memgraph" run: | ./scripts/pack_memgraph_via_docker.sh pack - ./scripts/pack_memgraph_via_docker.sh cleanup + echo ${{ secrets.HOST_PASSWORD }} | sudo -S ./scripts/pack_memgraph_via_docker.sh cleanup env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 25b9698..3c64ebb 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -70,6 +70,8 @@ cleanup() { # docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" # docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" docker_stop_rm $MGPLAT_CNT_NAME + # NOTE: Run cleanup as root or with sudo; sudo ./pack_memgraph_via_docker.sh cleanup + rm -rf "$DIR/dist/package/*" } copy_package() { From 7be55a9602d0bfd098dcd5a7e9697eac836c5383 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 21:20:48 +0100 Subject: [PATCH 37/67] Push to trigger workflow From 750326e8abc86f1cc21a89806a0a84e12d013b15 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 21:22:22 +0100 Subject: [PATCH 38/67] Add on push --- .github/workflows/daily-builds-amd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 6054cea..6f8532a 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -1,6 +1,7 @@ name: Daily builds for amd64 on: + push: pull_request: workflow_dispatch: inputs: From 7cfc9b9f46a00b55b80cba1cc6acbf84458df452 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 22:33:19 +0100 Subject: [PATCH 39/67] Fix bind mount --- .github/workflows/daily-builds-amd.yml | 3 ++- scripts/pack_memgraph_via_docker.sh | 29 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 6f8532a..3eddf7e 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -53,7 +53,8 @@ jobs: - name: "Build memgraph" run: | ./scripts/pack_memgraph_via_docker.sh pack - echo ${{ secrets.HOST_PASSWORD }} | sudo -S ./scripts/pack_memgraph_via_docker.sh cleanup + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 3c64ebb..c257732 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -6,6 +6,7 @@ DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-11}" MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" +MGPLAT_CNT_OUTPUT_DIR="/memgraph/build/output" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" MGPLAT_DIST_BINARY="$DIR/dist/binary" @@ -33,10 +34,9 @@ docker_run () { echo "Cleanup of the old exited container..." docker rm "$cnt_name" fi - docker run -d \ - -v "$DIR/..:/platform" \ - -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ - --network host --name "$cnt_name" "$cnt_image" + docker run -d --network host --name "$cnt_name" "$cnt_image" + # -v "$DIR/..:/platform" \ + # -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ fi echo "The $cnt_image container is active under $cnt_name name!" } @@ -55,36 +55,35 @@ build_pack() { cd "$DIR" # shellcheck disable=SC1091 source build_memgraph_native.sh - mkdir -p dist/binary - mkdir -p dist/package docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" + docker cp "$MGPLAT_CNT_MG_ROOT" "$MGPLAT_CNT_NAME:/" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" - docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" - mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" + docker_exec "git config --global --add safe.directory /memgraph" + mg_root="MGPLAT_MG_ROOT=/memgraph" mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build" } cleanup() { - # docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" - # docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" docker_stop_rm $MGPLAT_CNT_NAME - # NOTE: Run cleanup as root or with sudo; sudo ./pack_memgraph_via_docker.sh cleanup - rm -rf "$DIR/dist/package/*" } copy_package() { - src_cnt_package_path="$MGPLAT_CNT_NAME:$MGPLAT_CNT_MG_ROOT/build/output/." - docker cp $src_cnt_package_path $MGPLAT_DIST_PACKAGE + mkdir -p $MGPLAT_DIST_PACKAGE + last_package_name=$(docker exec "$MGPLAT_CNT_NAME" bash -c "cd $MGPLAT_CNT_OUTPUT_DIR && ls -t memgraph* | head -1") + package_host_destination="$MGPLAT_DIST_PACKAGE/$last_package_name" + docker cp "$MGPLAT_CNT_NAME:$MGPLAT_CNT_OUTPUT_DIR/$last_package_name" "$package_host_destination" + echo "Package $last_package_name saved to $package_host_destination." } copy_binary() { + mkdir -p $MGPLAT_DIST_BINARY cnt_cmd="echo \$(readlink $MGPLAT_CNT_MG_ROOT/build/memgraph)" cnt_binary_path=$(docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd") binary_name="$(basename $cnt_binary_path)" src_cnt_binary_path="$MGPLAT_CNT_NAME:$cnt_binary_path" - docker cp -L "$src_cnt_binary_path" "$DIR/dist/binary/" + docker cp -L "$src_cnt_binary_path" "$MGPLAT_DIST_BINARY" } if [ "$#" == 0 ]; then From bef18a0906f1ba5fcee4d17c72f5b4d3889a3c45 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 22:42:47 +0100 Subject: [PATCH 40/67] Add fix --- .github/workflows/daily-builds-amd.yml | 1 - scripts/pack_memgraph_via_docker.sh | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 3eddf7e..fcc69bb 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -1,7 +1,6 @@ name: Daily builds for amd64 on: - push: pull_request: workflow_dispatch: inputs: diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index c257732..ae9bf0f 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -5,8 +5,9 @@ DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # NOTE: The builder container image defines for which operating system Memgraph will be built. MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-11}" MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" -MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" +MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/memgraph}" MGPLAT_CNT_OUTPUT_DIR="/memgraph/build/output" +MGPLAT_HOST_MG_ROOT="${MGPLAT_HOST_MG_ROOT:-$DIR/../mage/cpp/memgraph}" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" MGPLAT_DIST_BINARY="$DIR/dist/binary" @@ -56,10 +57,10 @@ build_pack() { # shellcheck disable=SC1091 source build_memgraph_native.sh docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" - docker cp "$MGPLAT_CNT_MG_ROOT" "$MGPLAT_CNT_NAME:/" + docker cp "$MGPLAT_HOST_MG_ROOT" "$MGPLAT_CNT_NAME:$MGPLAT_CNT_MG_ROOT" + docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" - docker_exec "git config --global --add safe.directory /memgraph" - mg_root="MGPLAT_MG_ROOT=/memgraph" + mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" docker_exec "$mg_root $mg_build_type $mg_tag /build_memgraph_native.sh build" From 9f7e9baba83b93dc0ef2c9580c03eaabda4fcd0e Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 22:44:46 +0100 Subject: [PATCH 41/67] Add fix --- .github/workflows/daily-builds-amd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index fcc69bb..dc12be7 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -57,7 +57,7 @@ jobs: env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" - MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_CNT_MG_ROOT: "/memgraph" MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} From 68c291d2b033213076a3476a9f9d3e01127ba43f Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 27 Nov 2023 23:08:56 +0100 Subject: [PATCH 42/67] Fix cnt mg root --- .github/workflows/daily-builds-amd.yml | 2 +- scripts/pack_memgraph_via_docker.sh | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index dc12be7..fcc69bb 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -57,7 +57,7 @@ jobs: env: MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" - MGPLAT_CNT_MG_ROOT: "/memgraph" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index ae9bf0f..919a7a9 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -5,9 +5,8 @@ DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # NOTE: The builder container image defines for which operating system Memgraph will be built. MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-11}" MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" -MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/memgraph}" -MGPLAT_CNT_OUTPUT_DIR="/memgraph/build/output" -MGPLAT_HOST_MG_ROOT="${MGPLAT_HOST_MG_ROOT:-$DIR/../mage/cpp/memgraph}" +MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" +MGPLAT_CNT_OUTPUT_DIR="$MGPLAT_CNT_MG_ROOT/build/output" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" MGPLAT_DIST_BINARY="$DIR/dist/binary" @@ -57,7 +56,7 @@ build_pack() { # shellcheck disable=SC1091 source build_memgraph_native.sh docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" - docker cp "$MGPLAT_HOST_MG_ROOT" "$MGPLAT_CNT_NAME:$MGPLAT_CNT_MG_ROOT" + docker cp "$DIR/.." "$MGPLAT_CNT_NAME:/platform" docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" From 86498301fc7ec5bf5a0c8bd486c8aceae148d241 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 28 Nov 2023 01:34:31 +0100 Subject: [PATCH 43/67] Add arm builds --- .github/workflows/daily-builds-arm.yml | 187 +++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 .github/workflows/daily-builds-arm.yml diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml new file mode 100644 index 0000000..1170e00 --- /dev/null +++ b/.github/workflows/daily-builds-arm.yml @@ -0,0 +1,187 @@ +name: Daily builds for arm64 + +on: + pull_request: + workflow_dispatch: + inputs: + build_type: + type: choice + description: "Build type" + default: 'ReleaseWithDebug' + options: + - Release + - ReleaseWithDebug + - Debug + toolchain_version: + type: choice + description: "Toolchain version" + default: 'v4' + options: + - v4 + - v5 + memgraph_tag: + description: "Memgraph version (choose master to use the latest source)" + default: 'master' + mage_tag: + description: "Mage version (choose main to use the latest source)" + default: 'main' + lab_tag: + description: "Lab version (choose dev to use the latest source)" + default: 'dev' + +env: + ARCHITECTURE: "arm64" + LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} + TARGET_OS: "debian-11-arm" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + +jobs: + build_memgraph: + name: "Build memgraph" + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Build memgraph" + run: | + ./scripts/pack_memgraph_via_docker.sh pack + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup + env: + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" + MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} + + - name: "Upload memgraph binary" + uses: actions/upload-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + path: scripts/dist/package/memgraph*.deb + + build_memgraph_platform: + name: "Build memgraph plaftorm" + needs: [build_memgraph, build_mage] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git checkout ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + docker load < mage_${{ env.ARCHITECTURE }}.tar.gz + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + + build_memgraph_platform_without_mage: + name: "Build memgraph plaftorm without mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git checkout ${{ env.LAB_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_platform.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} \ + --no-mage + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + + build_mage: + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build Mage" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz From e63a4a171beceabf45d6fcbc629614084347e5b2 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 28 Nov 2023 12:03:20 +0100 Subject: [PATCH 44/67] Change default build type to RelWithDebInfo --- .github/workflows/daily-builds-amd.yml | 6 +++--- .github/workflows/daily-builds-arm.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index fcc69bb..6fa014b 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -7,10 +7,10 @@ on: build_type: type: choice description: "Build type" - default: 'ReleaseWithDebug' + default: 'RelWithDebInfo' options: - Release - - ReleaseWithDebug + - RelWithDebInfo - Debug toolchain_version: type: choice @@ -59,7 +59,7 @@ jobs: MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} - MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'RelWithDebInfo' }} - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 1170e00..81a7117 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -7,7 +7,7 @@ on: build_type: type: choice description: "Build type" - default: 'ReleaseWithDebug' + default: 'RelWithDebInfo' options: - Release - ReleaseWithDebug @@ -59,7 +59,7 @@ jobs: MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} - MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'RelWithDebInfo' }} - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 From 536a6f39c7c04959116acdbad4f79e55fa7733d3 Mon Sep 17 00:00:00 2001 From: Deda Date: Tue, 28 Nov 2023 15:57:53 +0100 Subject: [PATCH 45/67] Add mage without ML and optional builds when running manually --- .github/workflows/daily-builds-amd.yml | 141 ++++++++++++++++++++---- .github/workflows/daily-builds-arm.yml | 143 +++++++++++++++++++++---- scripts/docker_image_mage.sh | 14 ++- scripts/docker_image_platform.sh | 1 - 4 files changed, 256 insertions(+), 43 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 6fa014b..4070eee 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -28,6 +28,22 @@ on: lab_tag: description: "Lab version (choose dev to use the latest source)" default: 'dev' + build_mage: + description: "Build full mage image" + type: boolean + default: true + build_mage_no_ml: + description: "Build mage image without ML libraries" + type: boolean + default: true + build_memgraph_platform: + description: "Build memgraph plaftorm image" + type: boolean + default: true + build_memgraph_platform_without_mage: + description: "Build memgraph plaftorm without mage" + type: boolean + default: true env: ARCHITECTURE: "amd64" @@ -67,7 +83,83 @@ jobs: name: memgraph_${{ env.TARGET_OS }}.deb path: scripts/dist/package/memgraph*.deb + build_mage: + if: ${{ github.event.inputs.build_mage != 'false' }} + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build mage" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + + build_mage_no_ml: + if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} + name: "Build mage without ML libraries" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build mage" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }}_no-ML \ + ${{ env.ARCHITECTURE }} \ + --no-ml + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + build_memgraph_platform: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} name: "Build memgraph plaftorm" needs: [build_memgraph, build_mage] runs-on: [self-hosted, Docker, X64] @@ -112,9 +204,10 @@ jobs: name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - build_memgraph_platform_without_mage: - name: "Build memgraph plaftorm without mage" - needs: [build_memgraph] + build_memgraph_platform_no_ml: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm without ML libraries" + needs: [build_memgraph, build_mage_no_ml] runs-on: [self-hosted, Docker, X64] steps: - name: "Checkout repository and submodules" @@ -127,31 +220,39 @@ jobs: - name: "Checkout correct versions in submodules" run: | cd lab && git checkout ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + docker load < mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} \ - --no-mage + memgraph-platform_${{ env.ARCHITECTURE }}_no-ML \ + ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz - build_mage: - name: "Build mage" + build_memgraph_platform_without_mage: + if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} + name: "Build memgraph plaftorm without mage" needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: @@ -164,24 +265,26 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - - name: "Build Mage" + - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ + ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} \ + --no-mage + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - - name: "Upload mage docker image" + - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 81a7117..a567a26 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -10,7 +10,7 @@ on: default: 'RelWithDebInfo' options: - Release - - ReleaseWithDebug + - RelWithDebInfo - Debug toolchain_version: type: choice @@ -28,6 +28,22 @@ on: lab_tag: description: "Lab version (choose dev to use the latest source)" default: 'dev' + build_mage: + description: "Build full mage image" + type: boolean + default: true + build_mage_no_ml: + description: "Build mage image without ML libraries" + type: boolean + default: true + build_memgraph_platform: + description: "Build memgraph plaftorm image" + type: boolean + default: true + build_memgraph_platform_without_mage: + description: "Build memgraph plaftorm without mage" + type: boolean + default: true env: ARCHITECTURE: "arm64" @@ -67,7 +83,83 @@ jobs: name: memgraph_${{ env.TARGET_OS }}.deb path: scripts/dist/package/memgraph*.deb + build_mage: + if: ${{ github.event.inputs.build_mage != 'false' }} + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build mage" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + + build_mage_no_ml: + if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} + name: "Build mage without ML libraries" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: memgraph_${{ env.TARGET_OS }}.deb + + - name: "Build mage" + run: | + mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + ./scripts/docker_image_mage.sh build \ + memgraph_${{ env.TARGET_OS }}.deb \ + mage_${{ env.ARCHITECTURE }}_no-ML \ + ${{ env.ARCHITECTURE }} \ + --no-ml + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + build_memgraph_platform: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} name: "Build memgraph plaftorm" needs: [build_memgraph, build_mage] runs-on: [self-hosted, Docker, ARM64] @@ -112,9 +204,10 @@ jobs: name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - build_memgraph_platform_without_mage: - name: "Build memgraph plaftorm without mage" - needs: [build_memgraph] + build_memgraph_platform_no_ml: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm without ML libraries" + needs: [build_memgraph, build_mage_no_ml] runs-on: [self-hosted, Docker, ARM64] steps: - name: "Checkout repository and submodules" @@ -127,31 +220,39 @@ jobs: - name: "Checkout correct versions in submodules" run: | cd lab && git checkout ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb + docker load < mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} \ - --no-mage + memgraph-platform_${{ env.ARCHITECTURE }}_no-ML \ + ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz - build_mage: - name: "Build mage" + build_memgraph_platform_without_mage: + if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} + name: "Build memgraph plaftorm without mage" needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: @@ -164,24 +265,26 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 with: name: memgraph_${{ env.TARGET_OS }}.deb - - name: "Build Mage" + - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ + ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + memgraph-platform_${{ env.ARCHITECTURE }} \ + ${{ env.ARCHITECTURE }} \ + --no-mage + env: + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} - - name: "Upload mage docker image" + - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index fa95f96..9a8f0c8 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -19,7 +19,15 @@ build() { cp $src_package \ $MAGE_DIR/$mage_package_file cd $MAGE_DIR - docker buildx build --target prod --platform="linux/$target_arch" -t $image_name -f Dockerfile.release . + dockerfile="Dockerfile.release" + if [[ "$#" -eq 4 ]]; then + if [ "$4" == "--no-ml" ]; then + dockerfile="Dockerfile.no_ML" + else + print_help + fi + fi + docker buildx build --target prod --platform="linux/$target_arch" -t $image_name -f $dockerfile . mkdir -p "$DIR/dist/docker" docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" } @@ -29,10 +37,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 4 ]; then + if [[ "$#" -lt 4 || "$#" -gt 5 ]]; then print_help fi - build $2 $3 $4 + build $2 $3 $4 $5 ;; *) print_help diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 7d91c64..47a8c22 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -32,7 +32,6 @@ build() { print_help fi fi - echo "-----Using dockerfile $dockerfile-----" docker buildx build --platform="linux/$target_arch" -t ${image_name} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ -f ${dockerfile} . From 53578edef3e5cee36e265a2058eabed2d00fec61 Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 29 Nov 2023 10:59:59 +0100 Subject: [PATCH 46/67] Add files to test no ML build for mage --- .github/workflows/daily-builds-amd.yml | 7 +++ .github/workflows/daily-builds-arm.yml | 7 +++ Dockerfile | 26 +------- Dockerfile.no_ML | 86 ++++++++++++++++++++++++++ requirements_no_ml.txt | 13 ++++ 5 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 Dockerfile.no_ML create mode 100644 requirements_no_ml.txt diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 4070eee..90f5ee7 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -1,6 +1,7 @@ name: Daily builds for amd64 on: + #NOTE: Remove pull_request: before merge pull_request: workflow_dispatch: inputs: @@ -138,6 +139,12 @@ jobs: cd mage && git checkout ${{ env.MAGE_TAG }} cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + #NOTE: Remove this step before merge + - name: "Move no ML files into mage directory" + run: | + mv Dockerfile.no_ML mage/ + mv requirements_no_ml.txt mage/python/ + - name: "Download memgraph package" uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index a567a26..c5c7b3a 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -1,6 +1,7 @@ name: Daily builds for arm64 on: + #NOTE: Remove pull_request: before merge pull_request: workflow_dispatch: inputs: @@ -138,6 +139,12 @@ jobs: cd mage && git checkout ${{ env.MAGE_TAG }} cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + #NOTE: Remove this step before merge + - name: "Move no ML files into mage directory" + run: | + mv Dockerfile.no_ML mage/ + mv requirements_no_ml.txt mage/python/ + - name: "Download memgraph package" uses: actions/download-artifact@v3 with: diff --git a/Dockerfile b/Dockerfile index c0e8a5a..7087f84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,30 +30,10 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ################################################################################################################################################### -FROM mage_${TARGETARCH}:latest as mage-dev + # MAGE -# FROM base as mage-dev -# -# WORKDIR /mage -# COPY mage /mage -# -# RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -# -# ENV PATH="/root/.cargo/bin:${PATH}" -# -# RUN python3 -m pip install --upgrade pip \ -# && python3 -m pip install --default-timeout=1000 -r /mage/python/requirements.txt -# -# RUN python3 -m pip --default-timeout=1000 --no-cache-dir install torch-sparse torch-cluster torch-spline-conv \ -# torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html -# -# RUN python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ -# -# # DGL build from source -# RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git \ -# && cd dgl && mkdir build && cd build && cmake .. \ -# && make -j4 && cd ../python && python3 setup.py install -# +FROM mage_${TARGETARCH}:latest as mage-dev + ################################################################################################################################################### # Lab: Build backend diff --git a/Dockerfile.no_ML b/Dockerfile.no_ML new file mode 100644 index 0000000..347e4b0 --- /dev/null +++ b/Dockerfile.no_ML @@ -0,0 +1,86 @@ +ARG PY_VERSION_DEFAULT=3.9 + +FROM debian:bullseye as base + +USER root + +ARG TARGETARCH +ARG PY_VERSION_DEFAULT +ENV PY_VERSION ${PY_VERSION_DEFAULT} + +#essentials for production/dev +RUN apt-get update && apt-get install -y \ + libcurl4 `memgraph` \ + libpython${PY_VERSION} `memgraph` \ + libssl-dev `memgraph` \ + openssl `memgraph` \ + build-essential `mage-memgraph` \ + cmake `mage-memgraph` \ + curl `mage-memgraph` \ + g++ `mage-memgraph` \ + python3 `mage-memgraph` \ + python3-pip `mage-memgraph` \ + python3-setuptools `mage-memgraph` \ + python3-dev `mage-memgraph` \ + clang `mage-memgraph` \ + git `mage-memgraph` \ + unixodbc-dev `mage-memgraph` \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY memgraph-${TARGETARCH}.deb . + +RUN dpkg -i memgraph-${TARGETARCH}.deb && rm memgraph-${TARGETARCH}.deb + +ENV LD_LIBRARY_PATH /usr/lib/memgraph/query_modules + +# Memgraph listens for Bolt Protocol on this port by default. +EXPOSE 7687 + +FROM base as dev + +WORKDIR /mage +COPY . /mage + + +#MAGE +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ + && export PATH="/root/.cargo/bin:${PATH}" \ + && python3 -m pip install -r /mage/python/requirements_no_ml.txt \ + && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ \ + && chown -R memgraph: /mage/e2e + + +USER memgraph +ENTRYPOINT ["/usr/lib/memgraph/memgraph"] +CMD [""] + + + +FROM base as prod + +USER root +ENTRYPOINT [] +ARG PY_VERSION_DEFAULT +ENV PY_VERSION ${PY_VERSION_DEFAULT} + +#copy modules +COPY --from=dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/ + +#copy python build +COPY --from=dev /usr/local/lib/python${PY_VERSION}/ /usr/local/lib/python${PY_VERSION}/ + +#copy e2e tests +COPY --from=dev /mage/e2e/ /mage/e2e/ + +RUN mv /mage/e2e /e2e/ \ + && rm -rf /mage/* \ + && mv /e2e/ /mage/ \ + && chown -R memgraph: /mage/e2e \ + && export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}" \ + && apt-get -y --purge autoremove clang git curl python3-pip python3-dev cmake build-essential \ + && apt-get clean + +USER memgraph +ENTRYPOINT ["/usr/lib/memgraph/memgraph"] +CMD [""] diff --git a/requirements_no_ml.txt b/requirements_no_ml.txt new file mode 100644 index 0000000..65f2dde --- /dev/null +++ b/requirements_no_ml.txt @@ -0,0 +1,13 @@ +gekko==0.2.8 +networkx==2.6.2 +python-Levenshtein==0.12.1 +elasticsearch==8.4.3 +gensim==4.0.0 +six==1.16.0 +igraph==0.10.2 +scikit-learn==0.24.2 +gqlalchemy==1.4.1 +mysql-connector-python==8.0.32 +oracledb==1.2.2 +pyodbc==4.0.35 +defusedxml==0.7.1 From bf32e6e17439e491646da762149a46279bcd4fec Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 29 Nov 2023 11:35:35 +0100 Subject: [PATCH 47/67] Fix repository name issue --- .github/workflows/daily-builds-amd.yml | 16 ++++++++-------- .github/workflows/daily-builds-arm.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 90f5ee7..5846542 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -155,15 +155,15 @@ jobs: mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_mage.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }}_no-ML \ + mage_${{ env.ARCHITECTURE }}_no-ml \ ${{ env.ARCHITECTURE }} \ --no-ml - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz build_memgraph_platform: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -238,15 +238,15 @@ jobs: - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + docker load < mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }}_no-ML \ + memgraph-platform_${{ env.ARCHITECTURE }}_no-ml \ ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -254,8 +254,8 @@ jobs: - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz build_memgraph_platform_without_mage: if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index c5c7b3a..31ad28a 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -155,15 +155,15 @@ jobs: mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb ./scripts/docker_image_mage.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }}_no-ML \ + mage_${{ env.ARCHITECTURE }}_no-ml \ ${{ env.ARCHITECTURE }} \ --no-ml - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz build_memgraph_platform: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -238,15 +238,15 @@ jobs: - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - name: "Build memgraph platform" run: | mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}_no-ML.tar.gz + docker load < mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz ./scripts/docker_image_platform.sh build \ memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }}_no-ML \ + memgraph-platform_${{ env.ARCHITECTURE }}_no-ml \ ${{ env.ARCHITECTURE }} env: MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} @@ -254,8 +254,8 @@ jobs: - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ML.tar.gz + name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz + path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz build_memgraph_platform_without_mage: if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} From a68635972bf50954e0976dd43b6edeb34e8a3114 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 30 Nov 2023 16:42:45 +0100 Subject: [PATCH 48/67] Update scripts and workflows --- .github/workflows/daily-builds-amd.yml | 105 +++++++++++++------------ .github/workflows/daily-builds-arm.yml | 105 +++++++++++++------------ Dockerfile | 3 +- scripts/docker_image_mage.sh | 57 ++++++++++---- scripts/docker_image_memgraph.sh | 26 +++++- scripts/docker_image_platform.sh | 77 +++++++++++++----- 6 files changed, 234 insertions(+), 139 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 5846542..478bd01 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -47,12 +47,24 @@ on: default: true env: - ARCHITECTURE: "amd64" + TARGET_ARCH: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_PACKAGE_PATH: "memgraph_${{ env.TARGET_OS }}.deb" + MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" + MAGE_TAR: "mage_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_TAR: "mage_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" + MGPLAT_TAR: "memgraph-platform_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} jobs: build_memgraph: @@ -81,7 +93,7 @@ jobs: - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} path: scripts/dist/package/memgraph*.deb build_mage: @@ -105,21 +117,20 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build mage" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build + env: + CLEANUP: "true" - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MAGE_TAR }} + path: scripts/dist/docker/${{ env.MAGE_TAR }} build_mage_no_ml: if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} @@ -148,22 +159,22 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build mage" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }}_no-ml \ - ${{ env.ARCHITECTURE }} \ - --no-ml + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build --no-ml + env: + MAGE_IMAGE_NAME: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} + CLEANUP: "true" - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MAGE_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MAGE_NO_ML_TAR }} build_memgraph_platform: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -187,29 +198,25 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MAGE_TAR }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}.tar.gz - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_TAR }} env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MGPLAT_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_TAR }} build_memgraph_platform_no_ml: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -233,29 +240,27 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MAGE_NO_ML_TAR }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }}_no-ml \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_NO_ML_TAR }} env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_ML_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_ML_TAR }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MGPLAT_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_ML_TAR }} build_memgraph_platform_without_mage: if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} @@ -277,21 +282,19 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} \ - --no-mage + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --no-mage env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} + MGPLAT_TAR: $${{ env.MGPLAT_NO_MAGE_TAR }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MGPLAT_NO_MAGE_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_MAGE_TAR }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 31ad28a..e82bd52 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -47,12 +47,24 @@ on: default: true env: - ARCHITECTURE: "arm64" + TARGET_ARCH: "arm64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_PACKAGE_PATH: "memgraph_${{ env.TARGET_OS }}.deb" + MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" + MAGE_TAR: "mage_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_TAR: "mage_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" + MGPLAT_TAR: "memgraph-platform_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} jobs: build_memgraph: @@ -81,7 +93,7 @@ jobs: - name: "Upload memgraph binary" uses: actions/upload-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} path: scripts/dist/package/memgraph*.deb build_mage: @@ -105,21 +117,20 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build mage" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build + env: + CLEANUP: "true" - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MAGE_TAR }} + path: scripts/dist/docker/${{ env.MAGE_TAR }} build_mage_no_ml: if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} @@ -148,22 +159,22 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build mage" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_mage.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - mage_${{ env.ARCHITECTURE }}_no-ml \ - ${{ env.ARCHITECTURE }} \ - --no-ml + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build --no-ml + env: + MAGE_IMAGE_NAME: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} + CLEANUP: "true" - name: "Upload mage docker image" uses: actions/upload-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - path: scripts/dist/docker/mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MAGE_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MAGE_NO_ML_TAR }} build_memgraph_platform: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -187,29 +198,25 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MAGE_TAR }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}.tar.gz - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_TAR }} env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MGPLAT_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_TAR }} build_memgraph_platform_no_ml: if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} @@ -233,29 +240,27 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Download mage package" uses: actions/download-artifact@v3 with: - name: mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MAGE_NO_ML_TAR }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - docker load < mage_${{ env.ARCHITECTURE }}_no-ml.tar.gz - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }}_no-ml \ - ${{ env.ARCHITECTURE }} + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_NO_ML_TAR }} env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_ML_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_ML_TAR }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}_no-ml.tar.gz + name: ${{ env.MGPLAT_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_ML_TAR }} build_memgraph_platform_without_mage: if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} @@ -277,21 +282,19 @@ jobs: - name: "Download memgraph package" uses: actions/download-artifact@v3 with: - name: memgraph_${{ env.TARGET_OS }}.deb + name: ${{ env.MG_PACKAGE_PATH }} - name: "Build memgraph platform" run: | - mv memgraph_*.deb memgraph_${{ env.TARGET_OS }}.deb - ./scripts/docker_image_platform.sh build \ - memgraph_${{ env.TARGET_OS }}.deb \ - memgraph-platform_${{ env.ARCHITECTURE }} \ - ${{ env.ARCHITECTURE }} \ - --no-mage + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --no-mage env: - MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} + MGPLAT_TAR: $${{ env.MGPLAT_NO_MAGE_TAR }} + CLEANUP: "true" - name: "Upload memgraph platform docker image" uses: actions/upload-artifact@v3 with: - name: memgraph-platform-no-mage_${{ env.ARCHITECTURE }}.tar.gz - path: scripts/dist/docker/memgraph-platform_${{ env.ARCHITECTURE }}.tar.gz + name: ${{ env.MGPLAT_NO_MAGE_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_MAGE_TAR }} diff --git a/Dockerfile b/Dockerfile index 7087f84..1d9a9d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,8 @@ RUN apt-get update && apt-get install -y \ ################################################################################################################################################### # MAGE -FROM mage_${TARGETARCH}:latest as mage-dev +ARG MAGE_IMAGE +FROM ${MAGE_IMAGE} as mage-dev ################################################################################################################################################### diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index 9a8f0c8..d87f9c1 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -2,34 +2,61 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -MAGE_DIR="$DIR/../mage" +MAGE_ROOT="$DIR/../mage" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MAGE_ROOT/memgraph.deb}" +MAGE_IMAGE="${MAGE_IMAGE:-memgraph/mage:latest}" +MAGE_TAR="${MAGE_TAR:-mage_$TARGET_ARCH.tar.gz}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph mage Docker image." + echo -e "" + echo -e "Env vars:" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MAGE_IMAGE -> Name for the resulting docker image" + echo -e " MAGE_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name target_arch]" + echo -e " $0 [-h|build [--no-ml]]" exit 1 } build() { - src_package=$1 - image_name=$2 - target_arch=$3 - mage_package_file="memgraph-$target_arch.deb" - cp $src_package \ - $MAGE_DIR/$mage_package_file - cd $MAGE_DIR dockerfile="Dockerfile.release" - if [[ "$#" -eq 4 ]]; then - if [ "$4" == "--no-ml" ]; then + if [[ "$#" -eq 1 ]]; then + if [ "$1" == "--no-ml" ]; then dockerfile="Dockerfile.no_ML" else print_help fi fi - docker buildx build --target prod --platform="linux/$target_arch" -t $image_name -f $dockerfile . + + mage_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PAKCAGE_PATH \ + $MAGE_ROOT/$mage_package_file + cd $MAGE_ROOT + + docker buildx build \ + --target prod \ + --platform="linux/$TARGET_ARCH" \ + -t $MAGE_IMAGE \ + -f $dockerfile . mkdir -p "$DIR/dist/docker" - docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" + + mage_tar=$MAGE_TAR + mage_tar_ext=${MAGE_TAR#*.} + echo $mage_tar_ext + if [[ "$mage_tar_ext" != "tar.gz" ]]; then + mage_tar="$MAGE_TAR.tar.gz" + fi + docker save $MAGE_IMAGE | gzip -f > "$DIR/dist/docker/$mage_tar" + + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MAGE_IMAGE + fi } if [ "$#" == 0 ]; then @@ -37,10 +64,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [[ "$#" -lt 4 || "$#" -gt 5 ]]; then + if [[ "$#" -gt 2 ]]; then print_help fi - build $2 $3 $4 $5 + build $2 ;; *) print_help diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh index 2c39217..0d9b8ac 100755 --- a/scripts/docker_image_memgraph.sh +++ b/scripts/docker_image_memgraph.sh @@ -3,6 +3,12 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker" +MAGE_ROOT="$DIR/../mage" +MEMGRAPH_TARGET_ARCH="${MEMGRAPH_TARGET_ARCH:-amd64}" +MEMGRAPH_PACKAGE_PATH="${MEMGRAPH_PACKAGE_PATH:-$DIR/memgraph.deb}" +MEMGRAPH_IMAGE_NAME="${MEMGRAPH_IMAGE_NAME:-mage_$MEMGRAPH_TARGET_ARCH}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph Docker image." echo -e "" @@ -11,14 +17,30 @@ print_help() { exit 1 } +# memgraph_2.12.0+8~72d47fc3b-1_amd64.deb + build() { - src_package="$1" + src_package="memgraph_2.12.0+8~72d47fc3b-1_amd64.deb" + package_file="$(basename $src_package)" + package_file_no_prefix="${package_file#memgraph_}" + memgraph_version="${package_file_no_prefix%-1*}" - dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')" + + dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')" + + echo $package_file + echo $package_file_no_prefix + echo $memgraph_version + echo $dockerize_memgraph_version + + exit 0 + $MEMGRAPH_DOCKER_DIR/package_docker "$src_package" + mkdir -p "$DIR/dist/docker" + cp "$MEMGRAPH_DOCKER_DIR/memgraph-${dockerize_memgraph_version}-docker.tar.gz" "$DIR/dist/docker" } diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 47a8c22..fdfedb2 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -2,41 +2,80 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -MGPLAT_ROOT="$DIR/../" +MGPLAT_ROOT="$DIR/.." MGPLAT_GHA_PAT_TOKEN="${MGPLAT_GHA_PAT_TOKEN:-github_personal_access_token}" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MGPLAT_ROOT/memgraph.deb}" +MGPLAT_IMAGE="${MGPLAT_IMAGE:-memgraph-platform_$TARGET_ARCH}" +MGPLAT_TAR="${MGPLAT_TAR:-memgraph-platform_$TARGET_ARCH.tar.gz}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph platform Docker image." echo -e "" echo -e "Env vars:" echo -e " MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MGPLAT_IMAGE -> Name for the resulting docker image" + echo -e " MGPLAT_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build, also the tar.gz mage image if --mage-from-tar is passed (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name target_arch [--no-mage]]" + echo -e " $0 [-h|build --mage-from-tar mage_tar_path|build --mage-from-image mage_image_with_tag|build --mage-from-src|build --no-mage]" exit 1 } -# TODO(gitbuda): An option to build wihout mage. build() { - src_package=$1 - image_name=$2 - target_arch=$3 - platform_package_file="memgraph-$target_arch.deb" - cp $src_package \ - $MGPLAT_ROOT/$platform_package_file - cd $MGPLAT_ROOT dockerfile="Dockerfile" - if [[ "$#" -eq 4 ]]; then - if [ "$4" == "--no-mage" ]; then + case "$1" in + --no-mage) dockerfile="memgraph_and_lab.Dockerfile" - else + ;; + --mage-from-src) + dockerfile="mage_from_src.Dockerfile" + ;; + --mage-from-image) + mage_image=$2 + ;; + --mage-from-tar) + docker_load_out=$(docker load < $2) + mage_image=${docker_load_out#Loaded image:} + ;; + *) print_help - fi - fi - docker buildx build --platform="linux/$target_arch" -t ${image_name} \ + ;; + esac + + platform_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PACKAGE_PATH \ + $MGPLAT_ROOT/$platform_package_file + cd $MGPLAT_ROOT + docker buildx build \ + --platform="linux/$TARGET_ARCH" \ + -t ${MGPLAT_IMAGE} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ + --build-arg MAGE_IMAGE="${mage_image}" -f ${dockerfile} . mkdir -p "$DIR/dist/docker" - docker save $image_name | gzip -f > "$DIR/dist/docker/$image_name.tar.gz" + + mgplat_tar=$MGPLAT_TAR + mgplat_tar_ext=${MGPLAT_TAR#*.} + echo $mgplat_tar_ext + if [[ "$mgplat_tar_ext" != "tar.gz" ]]; then + mgplat_tar="$MGPLAT_TAR.tar.gz" + fi + docker save $MAGE_IMAGE | gzip -f > "$DIR/dist/docker/$mgplat_tar" + + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MGPLAT_IMAGE + if [[ "$1" == "--mage-from-tar" || "$1" == "--mage-from-image" ]]; then + docker image rm $mage_image + fi + if [[ "$1" == "--mage-from-tar" ]]; then + rm $2 + fi + fi } if [[ "$#" -eq 0 ]]; then @@ -44,10 +83,10 @@ if [[ "$#" -eq 0 ]]; then else case "$1" in build) - if [[ "$#" -lt 4 || "$#" -gt 5 ]]; then + if [[ "$#" -lt 2 || "$#" -gt 3 ]]; then print_help fi - build $2 $3 $4 $5 + build $2 $3 ;; *) print_help From 9704737a1088889f7d099151714be91ca61fe9d1 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 30 Nov 2023 16:57:59 +0100 Subject: [PATCH 49/67] Fix workflow envs --- .github/workflows/daily-builds-amd.yml | 12 ++++++------ .github/workflows/daily-builds-arm.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 478bd01..e90a390 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -53,17 +53,17 @@ env: MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_PACKAGE_PATH: "memgraph_${{ env.TARGET_OS }}.deb" + MG_PACKAGE_PATH: "memgraph_debian-11.deb" MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" - MAGE_TAR: "mage_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_TAR: "mage_amd64.tar.gz" MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" - MAGE_NO_ML_TAR: "mage_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_NO_ML_TAR: "mage_no-ml_amd64.tar.gz" MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" - MGPLAT_TAR: "memgraph-platform_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_TAR: "memgraph-platform_amd64.tar.gz" MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" - MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_amd64.tar.gz" MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" - MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_amd64.tar.gz" MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} jobs: diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index e82bd52..02f6483 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -53,17 +53,17 @@ env: MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} - MG_PACKAGE_PATH: "memgraph_${{ env.TARGET_OS }}.deb" + MG_PACKAGE_PATH: "memgraph_debian-11-arm.deb" MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" - MAGE_TAR: "mage_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_TAR: "mage_arm64.tar.gz" MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" - MAGE_NO_ML_TAR: "mage_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MAGE_NO_ML_TAR: "mage_no-ml_arm64.tar.gz" MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" - MGPLAT_TAR: "memgraph-platform_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_TAR: "memgraph-platform_arm64.tar.gz" MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" - MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_arm64.tar.gz" MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" - MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_${{ env.TARGET_ARCH }}.tar.gz" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_arm64.tar.gz" MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} jobs: From b91685ca46373f557d32ada29dde9a9f7ae20c1e Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 30 Nov 2023 18:37:33 +0100 Subject: [PATCH 50/67] Fix typos --- scripts/docker_image_mage.sh | 2 +- scripts/docker_image_platform.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index d87f9c1..60634eb 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -35,7 +35,7 @@ build() { fi mage_package_file="memgraph-$TARGET_ARCH.deb" - cp $MG_PAKCAGE_PATH \ + cp $MG_PACKAGE_PATH \ $MAGE_ROOT/$mage_package_file cd $MAGE_ROOT diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index fdfedb2..099dfd2 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -55,7 +55,7 @@ build() { --platform="linux/$TARGET_ARCH" \ -t ${MGPLAT_IMAGE} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ - --build-arg MAGE_IMAGE="${mage_image}" + --build-arg MAGE_IMAGE="${mage_image}" \ -f ${dockerfile} . mkdir -p "$DIR/dist/docker" From c4d7c4b0e640156919f9dd46441250c216678206 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 30 Nov 2023 19:26:43 +0100 Subject: [PATCH 51/67] Fix typo --- scripts/docker_image_platform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 099dfd2..874b9a8 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -65,7 +65,7 @@ build() { if [[ "$mgplat_tar_ext" != "tar.gz" ]]; then mgplat_tar="$MGPLAT_TAR.tar.gz" fi - docker save $MAGE_IMAGE | gzip -f > "$DIR/dist/docker/$mgplat_tar" + docker save $MGPLAT_IMAGE | gzip -f > "$DIR/dist/docker/$mgplat_tar" if [[ "$CLEANUP" == "true" ]]; then docker image rm $MGPLAT_IMAGE From 886a6c6d525ff2c5c3138d48af7f19216e8e4f7e Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 30 Nov 2023 23:36:03 +0100 Subject: [PATCH 52/67] Fix issues and typos --- .github/workflows/daily-builds-arm.yml | 15 +++++++++++++++ Dockerfile | 4 ++-- scripts/docker_image_platform.sh | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 02f6483..14bbfcb 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -109,6 +109,9 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + - name: "Checkout correct versions in submodules" run: | cd mage && git checkout ${{ env.MAGE_TAG }} @@ -145,6 +148,9 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + - name: "Checkout correct versions in submodules" run: | cd mage && git checkout ${{ env.MAGE_TAG }} @@ -189,6 +195,9 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + - name: "Checkout correct versions in submodules" run: | cd lab && git checkout ${{ env.LAB_TAG }} @@ -231,6 +240,9 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + - name: "Checkout correct versions in submodules" run: | cd lab && git checkout ${{ env.LAB_TAG }} @@ -275,6 +287,9 @@ jobs: submodules: recursive token: ${{ secrets.PAT }} + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + - name: "Checkout correct versions in submodules" run: | cd lab && git checkout ${{ env.LAB_TAG }} diff --git a/Dockerfile b/Dockerfile index 1d9a9d3..d4dee30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # For mage ARG PY_VERSION_DEFAULT=3.9 +ARG MAGE_IMAGE FROM debian:bullseye as base @@ -32,8 +33,7 @@ RUN apt-get update && apt-get install -y \ ################################################################################################################################################### # MAGE -ARG MAGE_IMAGE -FROM ${MAGE_IMAGE} as mage-dev +FROM $MAGE_IMAGE as mage-dev ################################################################################################################################################### diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 874b9a8..31dcc43 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -40,13 +40,13 @@ build() { ;; --mage-from-tar) docker_load_out=$(docker load < $2) - mage_image=${docker_load_out#Loaded image:} + mage_image=$(echo ${docker_load_out#Loaded image:} | tr -d '[:blank:]') ;; *) print_help ;; esac - + platform_package_file="memgraph-$TARGET_ARCH.deb" cp $MG_PACKAGE_PATH \ $MGPLAT_ROOT/$platform_package_file From aa4c9532fdd73188fefb22e4671464922d87c02f Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 1 Dec 2023 09:27:28 +0100 Subject: [PATCH 53/67] Fix typo that preveneted upoading platform without mage tar --- .github/workflows/daily-builds-amd.yml | 2 +- .github/workflows/daily-builds-arm.yml | 2 +- scripts/docker_image_platform.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index e90a390..c80694f 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -290,7 +290,7 @@ jobs: ./scripts/docker_image_platform.sh build --no-mage env: MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} - MGPLAT_TAR: $${{ env.MGPLAT_NO_MAGE_TAR }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_MAGE_TAR }} CLEANUP: "true" - name: "Upload memgraph platform docker image" diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 14bbfcb..4bacae8 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -305,7 +305,7 @@ jobs: ./scripts/docker_image_platform.sh build --no-mage env: MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} - MGPLAT_TAR: $${{ env.MGPLAT_NO_MAGE_TAR }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_MAGE_TAR }} CLEANUP: "true" - name: "Upload memgraph platform docker image" diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 31dcc43..6f885ad 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -61,10 +61,10 @@ build() { mgplat_tar=$MGPLAT_TAR mgplat_tar_ext=${MGPLAT_TAR#*.} - echo $mgplat_tar_ext if [[ "$mgplat_tar_ext" != "tar.gz" ]]; then mgplat_tar="$MGPLAT_TAR.tar.gz" fi + echo -e "Creating $mgplat_tar" docker save $MGPLAT_IMAGE | gzip -f > "$DIR/dist/docker/$mgplat_tar" if [[ "$CLEANUP" == "true" ]]; then From 7e44a5815080f1d4fef5b65c49b89f4e0f343e37 Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 1 Dec 2023 16:47:47 +0100 Subject: [PATCH 54/67] Fix mage no ml image name --- .github/workflows/daily-builds-amd.yml | 2 +- .github/workflows/daily-builds-arm.yml | 2 +- scripts/docker_image_mage.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index c80694f..653fcbc 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -166,7 +166,7 @@ jobs: mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} ./scripts/docker_image_mage.sh build --no-ml env: - MAGE_IMAGE_NAME: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_IMAGE: ${{ env.MAGE_NO_ML_IMAGE }} MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} CLEANUP: "true" diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 4bacae8..b25b7d1 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -172,7 +172,7 @@ jobs: mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} ./scripts/docker_image_mage.sh build --no-ml env: - MAGE_IMAGE_NAME: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_IMAGE: ${{ env.MAGE_NO_ML_IMAGE }} MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} CLEANUP: "true" diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index 60634eb..54d645d 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -53,7 +53,7 @@ build() { mage_tar="$MAGE_TAR.tar.gz" fi docker save $MAGE_IMAGE | gzip -f > "$DIR/dist/docker/$mage_tar" - + if [[ "$CLEANUP" == "true" ]]; then docker image rm $MAGE_IMAGE fi From 23fdac6824a2ff53eb7dda727d0ec1c77dd8bd95 Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 1 Dec 2023 17:07:42 +0100 Subject: [PATCH 55/67] Change mage image name to memgraph/memgraph-mage --- .github/workflows/daily-builds-amd.yml | 4 ++-- .github/workflows/daily-builds-arm.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 653fcbc..c71e781 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -54,9 +54,9 @@ env: TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11.deb" - MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" + MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" MAGE_TAR: "mage_amd64.tar.gz" - MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" MAGE_NO_ML_TAR: "mage_no-ml_amd64.tar.gz" MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" MGPLAT_TAR: "memgraph-platform_amd64.tar.gz" diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index b25b7d1..fc8cd95 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -54,9 +54,9 @@ env: TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11-arm.deb" - MAGE_IMAGE: "memgraph/mage:${{ github.run_id }}" + MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" MAGE_TAR: "mage_arm64.tar.gz" - MAGE_NO_ML_IMAGE: "memgraph/mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" MAGE_NO_ML_TAR: "mage_no-ml_arm64.tar.gz" MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" MGPLAT_TAR: "memgraph-platform_arm64.tar.gz" From 2035c47511a60a410c3f38cc71eca1f6e77014b0 Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 4 Dec 2023 16:05:32 +0100 Subject: [PATCH 56/67] Test mage fix --- .github/workflows/daily-builds-amd.yml | 2 +- .github/workflows/daily-builds-arm.yml | 2 +- scripts/pack_memgraph_via_docker.sh | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index c71e781..10a7ab3 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -49,7 +49,7 @@ on: env: TARGET_ARCH: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'fix-refactor-bug-rvalue-bind' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index fc8cd95..c14e43d 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -49,7 +49,7 @@ on: env: TARGET_ARCH: "arm64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'fix-refactor-bug-rvalue-bind' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 919a7a9..3e19829 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -35,8 +35,6 @@ docker_run () { docker rm "$cnt_name" fi docker run -d --network host --name "$cnt_name" "$cnt_image" - # -v "$DIR/..:/platform" \ - # -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ fi echo "The $cnt_image container is active under $cnt_name name!" } From 268be5296b709f266893355ee9aaf6a37a541502 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 7 Dec 2023 14:38:35 +0100 Subject: [PATCH 57/67] Add memgraph docker image build --- .github/workflows/daily-builds-amd.yml | 37 +++++++++++++++ .github/workflows/daily-builds-arm.yml | 37 +++++++++++++++ scripts/docker_image_mage.sh | 2 +- scripts/docker_image_memgraph.sh | 63 +++++++++++++++----------- 4 files changed, 111 insertions(+), 28 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 10a7ab3..6b271f3 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -54,6 +54,8 @@ env: TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11.deb" + MEMGRAPH_IMAGE: "memgraph/memgraph:${{ github.run_id }}" + MEMGRAPH_TAR: "memgraph_amd64.tar.gz" MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" MAGE_TAR: "mage_amd64.tar.gz" MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" @@ -96,6 +98,41 @@ jobs: name: ${{ env.MG_PACKAGE_PATH }} path: scripts/dist/package/memgraph*.deb + build_memgraph_image: + if: ${{ github.event.inputs.build_memgraph_image != 'false' }} + name: "Build memgraph image" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph image" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_memgraph.sh build + env: + CLEANUP: "true" + + - name: "Upload memgraph docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MEMGRAPH_TAR }} + path: scripts/dist/docker/${{ env.MEMGRAPH_TAR }} + build_mage: if: ${{ github.event.inputs.build_mage != 'false' }} name: "Build mage" diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index c14e43d..a989b92 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -54,6 +54,8 @@ env: TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11-arm.deb" + MEMGRAPH_IMAGE: "memgraph/memgraph:${{ github.run_id }}" + MEMGRAPH_TAR: "memgraph_arm64.tar.gz" MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" MAGE_TAR: "mage_arm64.tar.gz" MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" @@ -96,6 +98,41 @@ jobs: name: ${{ env.MG_PACKAGE_PATH }} path: scripts/dist/package/memgraph*.deb + build_memgraph_image: + if: ${{ github.event.inputs.build_memgraph_image != 'false' }} + name: "Build memgraph image" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph image" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_memgraph.sh build + env: + CLEANUP: "true" + + - name: "Upload memgraph docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MEMGRAPH_TAR }} + path: scripts/dist/docker/${{ env.MEMGRAPH_TAR }} + build_mage: if: ${{ github.event.inputs.build_mage != 'false' }} name: "Build mage" diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index 54d645d..bcab05b 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -5,7 +5,7 @@ DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MAGE_ROOT="$DIR/../mage" TARGET_ARCH="${TARGET_ARCH:-amd64}" MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MAGE_ROOT/memgraph.deb}" -MAGE_IMAGE="${MAGE_IMAGE:-memgraph/mage:latest}" +MAGE_IMAGE="${MAGE_IMAGE:-memgraph/memgraph-mage:latest}" MAGE_TAR="${MAGE_TAR:-mage_$TARGET_ARCH.tar.gz}" CLEANUP="${CLEANUP:-false}" diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh index 0d9b8ac..367b9e3 100755 --- a/scripts/docker_image_memgraph.sh +++ b/scripts/docker_image_memgraph.sh @@ -3,45 +3,54 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker" -MAGE_ROOT="$DIR/../mage" -MEMGRAPH_TARGET_ARCH="${MEMGRAPH_TARGET_ARCH:-amd64}" -MEMGRAPH_PACKAGE_PATH="${MEMGRAPH_PACKAGE_PATH:-$DIR/memgraph.deb}" -MEMGRAPH_IMAGE_NAME="${MEMGRAPH_IMAGE_NAME:-mage_$MEMGRAPH_TARGET_ARCH}" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$DIR/../memgraph.deb}" +MEMGRAPH_IMAGE="${MEMGRAPH_IMAGE:-memgraph/memgraph:latest}" +MEMGRAPH_TAR="${MEMGRAPH_TAR:-memgraph_$TARGET_ARCH.tar.gz}" CLEANUP="${CLEANUP:-false}" print_help() { echo -e "Builds memgraph Docker image." + echo -e "" + echo -e "Env vars:" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MEMGRAPH_IMAGE -> Name for the resulting docker image" + echo -e " MEMGRAPH_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path]" + echo -e " $0 [-h|build]" exit 1 } -# memgraph_2.12.0+8~72d47fc3b-1_amd64.deb - build() { - src_package="memgraph_2.12.0+8~72d47fc3b-1_amd64.deb" - - package_file="$(basename $src_package)" + dockerfile="memgraph_deb.dockerfile" - package_file_no_prefix="${package_file#memgraph_}" - - memgraph_version="${package_file_no_prefix%-1*}" - - dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')" + memgraph_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PACKAGE_PATH \ + $MEMGRAPH_DOCKER_DIR/$memgraph_package_file + cd $MEMGRAPH_DOCKER_DIR - echo $package_file - echo $package_file_no_prefix - echo $memgraph_version - echo $dockerize_memgraph_version + docker buildx build \ + --platform="linux/$TARGET_ARCH" \ + -tag $MEMGRAPH_IMAGE \ + --build-arg BINARY_NAME="memgraph-" \ + --build-arg EXTENSION="deb" + --file $dockerfile . + mkdir -p "$DIR/dist/docker" - exit 0 + memgraph_tar=$MEMGRAPH_TAR + memgraph_tar_ext=${MEMGRAPH_TAR#*.} + echo $memgraph_tar_ext + if [[ "$memgraph_tar_ext" != "tar.gz" ]]; then + memgraph_tar="$MEMGRAPH_TAR.tar.gz" + fi + docker save $MEMGRAPH_IMAGE | gzip -f > "$DIR/dist/docker/$memgraph_tar" - $MEMGRAPH_DOCKER_DIR/package_docker "$src_package" - - mkdir -p "$DIR/dist/docker" - - cp "$MEMGRAPH_DOCKER_DIR/memgraph-${dockerize_memgraph_version}-docker.tar.gz" "$DIR/dist/docker" + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MEMGRAPH_IMAGE + fi } if [ "$#" == 0 ]; then @@ -49,10 +58,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 2 ]; then + if [ "$#" -ne 1 ]; then print_help fi - build "$2" + build ;; *) print_help From a2fc676a0cd9f00a4af34a503529eb5ed5bdb79e Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 7 Dec 2023 16:11:28 +0100 Subject: [PATCH 58/67] Add step to clean working directory at the beginning of a job --- .github/workflows/daily-builds-amd.yml | 21 +++++++++++++++++++++ .github/workflows/daily-builds-arm.yml | 23 ++++++++++++++++++++++- scripts/docker_image_memgraph.sh | 5 +++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 6b271f3..cf8a7bf 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -73,6 +73,9 @@ jobs: name: "Build memgraph" runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -104,6 +107,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -139,6 +145,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -175,6 +184,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -219,6 +231,9 @@ jobs: needs: [build_memgraph, build_mage] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -261,6 +276,9 @@ jobs: needs: [build_memgraph, build_mage_no_ml] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -305,6 +323,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, X64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index a989b92..6c3fc46 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -73,6 +73,9 @@ jobs: name: "Build memgraph" runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -104,7 +107,10 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Checkout repository and submodules" + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: fetch-depth: 0 @@ -139,6 +145,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -178,6 +187,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -225,6 +237,9 @@ jobs: needs: [build_memgraph, build_mage] runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -270,6 +285,9 @@ jobs: needs: [build_memgraph, build_mage_no_ml] runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: @@ -317,6 +335,9 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: + - name: "Clean the working directory" + run: rm -rf * + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh index 367b9e3..ef7adde 100755 --- a/scripts/docker_image_memgraph.sh +++ b/scripts/docker_image_memgraph.sh @@ -3,8 +3,9 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker" +MEMGRAPH_ROOT="$DIR/../mage/cpp/memgraph" TARGET_ARCH="${TARGET_ARCH:-amd64}" -MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$DIR/../memgraph.deb}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MEMGRAPH_ROOT/memgraph.deb}" MEMGRAPH_IMAGE="${MEMGRAPH_IMAGE:-memgraph/memgraph:latest}" MEMGRAPH_TAR="${MEMGRAPH_TAR:-memgraph_$TARGET_ARCH.tar.gz}" CLEANUP="${CLEANUP:-false}" @@ -34,7 +35,7 @@ build() { docker buildx build \ --platform="linux/$TARGET_ARCH" \ - -tag $MEMGRAPH_IMAGE \ + --tag $MEMGRAPH_IMAGE \ --build-arg BINARY_NAME="memgraph-" \ --build-arg EXTENSION="deb" --file $dockerfile . From 82644a1dbe13e772daf6e6bb3db7e8a489b27f9f Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 7 Dec 2023 16:14:18 +0100 Subject: [PATCH 59/67] Fix indentation error --- .github/workflows/daily-builds-arm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 6c3fc46..61ca8ad 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -107,10 +107,10 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - - name: "Checkout repository and submodules" + - name: "Checkout repository and submodules" uses: actions/checkout@v3 with: fetch-depth: 0 From e9aa4e1c1c773fedf7419c1265fc27b2c295eb03 Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 7 Dec 2023 16:15:57 +0100 Subject: [PATCH 60/67] Fix indentation error --- .github/workflows/daily-builds-arm.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 61ca8ad..502079b 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -145,7 +145,7 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - name: "Checkout repository and submodules" @@ -187,7 +187,7 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - name: "Checkout repository and submodules" @@ -237,7 +237,7 @@ jobs: needs: [build_memgraph, build_mage] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - name: "Checkout repository and submodules" @@ -285,7 +285,7 @@ jobs: needs: [build_memgraph, build_mage_no_ml] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - name: "Checkout repository and submodules" @@ -335,7 +335,7 @@ jobs: needs: [build_memgraph] runs-on: [self-hosted, Docker, ARM64] steps: - - name: "Clean the working directory" + - name: "Clean the working directory" run: rm -rf * - name: "Checkout repository and submodules" From 997b8376e56cf817e9ca0191675c14202bc17adb Mon Sep 17 00:00:00 2001 From: Deda Date: Thu, 7 Dec 2023 17:11:15 +0100 Subject: [PATCH 61/67] Set main as deafult Mage tag --- .github/workflows/daily-builds-amd.yml | 2 +- .github/workflows/daily-builds-arm.yml | 2 +- scripts/docker_image_memgraph.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index cf8a7bf..d062296 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -49,7 +49,7 @@ on: env: TARGET_ARCH: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'fix-refactor-bug-rvalue-bind' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 502079b..3c8b3b1 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -49,7 +49,7 @@ on: env: TARGET_ARCH: "arm64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'fix-refactor-bug-rvalue-bind' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh index ef7adde..34966ab 100755 --- a/scripts/docker_image_memgraph.sh +++ b/scripts/docker_image_memgraph.sh @@ -37,7 +37,7 @@ build() { --platform="linux/$TARGET_ARCH" \ --tag $MEMGRAPH_IMAGE \ --build-arg BINARY_NAME="memgraph-" \ - --build-arg EXTENSION="deb" + --build-arg EXTENSION="deb" \ --file $dockerfile . mkdir -p "$DIR/dist/docker" From 690ef2e93f0d72016a68beb7061d81f2207269a1 Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 8 Dec 2023 12:25:28 +0100 Subject: [PATCH 62/67] Add git pull when checking out --- .github/workflows/daily-builds-amd.yml | 24 ++++++++++++------------ .github/workflows/daily-builds-arm.yml | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index d062296..878d083 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -157,8 +157,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -196,8 +196,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} #NOTE: Remove this step before merge - name: "Move no ML files into mage directory" @@ -243,9 +243,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -288,9 +288,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -335,7 +335,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 3c8b3b1..9d030d6 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -160,8 +160,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -202,8 +202,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} #NOTE: Remove this step before merge - name: "Move no ML files into mage directory" @@ -252,9 +252,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -300,9 +300,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -350,7 +350,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 From e2b7eae4eec52219450dd38c267602788884ef05 Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 8 Dec 2023 13:50:36 +0100 Subject: [PATCH 63/67] Fix env var typo --- .github/workflows/daily-builds-amd.yml | 24 ++++++++++++------------ .github/workflows/daily-builds-arm.yml | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 878d083..ceafc6d 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -157,8 +157,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -196,8 +196,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} #NOTE: Remove this step before merge - name: "Move no ML files into mage directory" @@ -243,9 +243,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -288,9 +288,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -335,7 +335,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 9d030d6..43ab173 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -160,8 +160,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -202,8 +202,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} #NOTE: Remove this step before merge - name: "Move no ML files into mage directory" @@ -252,9 +252,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -300,9 +300,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin {{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin {{ env.MEMGRAPH_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -350,7 +350,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin {{ env.LAB_TAG }} + cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 From 0b1ae52cc5a1fafd4ba3c8125c69ba1ee522b3ff Mon Sep 17 00:00:00 2001 From: Deda Date: Mon, 18 Dec 2023 12:27:57 +0100 Subject: [PATCH 64/67] Adjust build to merge of no-ml in mage --- .github/workflows/daily-builds-amd.yml | 6 -- .github/workflows/daily-builds-arm.yml | 6 -- Dockerfile.no_ML | 86 -------------------------- mage | 2 +- mage_from_src.Dockerfile | 0 requirements_no_ml.txt | 13 ---- 6 files changed, 1 insertion(+), 112 deletions(-) delete mode 100644 Dockerfile.no_ML create mode 100644 mage_from_src.Dockerfile delete mode 100644 requirements_no_ml.txt diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index ceafc6d..404f4f7 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -199,12 +199,6 @@ jobs: cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - #NOTE: Remove this step before merge - - name: "Move no ML files into mage directory" - run: | - mv Dockerfile.no_ML mage/ - mv requirements_no_ml.txt mage/python/ - - name: "Download memgraph package" uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 43ab173..239a4cf 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -205,12 +205,6 @@ jobs: cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - #NOTE: Remove this step before merge - - name: "Move no ML files into mage directory" - run: | - mv Dockerfile.no_ML mage/ - mv requirements_no_ml.txt mage/python/ - - name: "Download memgraph package" uses: actions/download-artifact@v3 with: diff --git a/Dockerfile.no_ML b/Dockerfile.no_ML deleted file mode 100644 index 347e4b0..0000000 --- a/Dockerfile.no_ML +++ /dev/null @@ -1,86 +0,0 @@ -ARG PY_VERSION_DEFAULT=3.9 - -FROM debian:bullseye as base - -USER root - -ARG TARGETARCH -ARG PY_VERSION_DEFAULT -ENV PY_VERSION ${PY_VERSION_DEFAULT} - -#essentials for production/dev -RUN apt-get update && apt-get install -y \ - libcurl4 `memgraph` \ - libpython${PY_VERSION} `memgraph` \ - libssl-dev `memgraph` \ - openssl `memgraph` \ - build-essential `mage-memgraph` \ - cmake `mage-memgraph` \ - curl `mage-memgraph` \ - g++ `mage-memgraph` \ - python3 `mage-memgraph` \ - python3-pip `mage-memgraph` \ - python3-setuptools `mage-memgraph` \ - python3-dev `mage-memgraph` \ - clang `mage-memgraph` \ - git `mage-memgraph` \ - unixodbc-dev `mage-memgraph` \ - --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY memgraph-${TARGETARCH}.deb . - -RUN dpkg -i memgraph-${TARGETARCH}.deb && rm memgraph-${TARGETARCH}.deb - -ENV LD_LIBRARY_PATH /usr/lib/memgraph/query_modules - -# Memgraph listens for Bolt Protocol on this port by default. -EXPOSE 7687 - -FROM base as dev - -WORKDIR /mage -COPY . /mage - - -#MAGE -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ - && export PATH="/root/.cargo/bin:${PATH}" \ - && python3 -m pip install -r /mage/python/requirements_no_ml.txt \ - && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ \ - && chown -R memgraph: /mage/e2e - - -USER memgraph -ENTRYPOINT ["/usr/lib/memgraph/memgraph"] -CMD [""] - - - -FROM base as prod - -USER root -ENTRYPOINT [] -ARG PY_VERSION_DEFAULT -ENV PY_VERSION ${PY_VERSION_DEFAULT} - -#copy modules -COPY --from=dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/ - -#copy python build -COPY --from=dev /usr/local/lib/python${PY_VERSION}/ /usr/local/lib/python${PY_VERSION}/ - -#copy e2e tests -COPY --from=dev /mage/e2e/ /mage/e2e/ - -RUN mv /mage/e2e /e2e/ \ - && rm -rf /mage/* \ - && mv /e2e/ /mage/ \ - && chown -R memgraph: /mage/e2e \ - && export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}" \ - && apt-get -y --purge autoremove clang git curl python3-pip python3-dev cmake build-essential \ - && apt-get clean - -USER memgraph -ENTRYPOINT ["/usr/lib/memgraph/memgraph"] -CMD [""] diff --git a/mage b/mage index 47344f3..8653871 160000 --- a/mage +++ b/mage @@ -1 +1 @@ -Subproject commit 47344f389a058ca0d95c568fa90f1a21942629f6 +Subproject commit 8653871913403006eb5d145de3e1af9f276bb9db diff --git a/mage_from_src.Dockerfile b/mage_from_src.Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/requirements_no_ml.txt b/requirements_no_ml.txt deleted file mode 100644 index 65f2dde..0000000 --- a/requirements_no_ml.txt +++ /dev/null @@ -1,13 +0,0 @@ -gekko==0.2.8 -networkx==2.6.2 -python-Levenshtein==0.12.1 -elasticsearch==8.4.3 -gensim==4.0.0 -six==1.16.0 -igraph==0.10.2 -scikit-learn==0.24.2 -gqlalchemy==1.4.1 -mysql-connector-python==8.0.32 -oracledb==1.2.2 -pyodbc==4.0.35 -defusedxml==0.7.1 From 810e2bf04b8673382454647c0b9884d040f44fef Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 22 Dec 2023 19:16:42 +0100 Subject: [PATCH 65/67] Run build for Ante --- .github/workflows/daily-builds-amd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 404f4f7..6a6b7ba 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -49,8 +49,8 @@ on: env: TARGET_ARCH: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} - MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'v2.13.0' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11.deb" From 320b6e104973953df62574a110006d8c72be6aac Mon Sep 17 00:00:00 2001 From: Deda Date: Wed, 27 Dec 2023 10:11:00 +0100 Subject: [PATCH 66/67] Add fetch before checkout --- .github/workflows/daily-builds-amd.yml | 24 ++++++++++++------------ .github/workflows/daily-builds-arm.yml | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 6a6b7ba..4bebc09 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -157,8 +157,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -196,8 +196,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -237,9 +237,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -282,9 +282,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -329,7 +329,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 239a4cf..0b43d52 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -49,7 +49,7 @@ on: env: TARGET_ARCH: "arm64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} @@ -119,7 +119,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage/cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage/cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -160,8 +160,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -202,8 +202,8 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -246,9 +246,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -294,9 +294,9 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - cd ../mage && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} - cd cpp/memgraph && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 @@ -344,7 +344,7 @@ jobs: - name: "Checkout correct versions in submodules" run: | - cd lab && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} - name: "Download memgraph package" uses: actions/download-artifact@v3 From 628c2f9fd7512106cb80801fee1ae66497c2c5a5 Mon Sep 17 00:00:00 2001 From: Deda Date: Fri, 29 Dec 2023 15:24:41 +0100 Subject: [PATCH 67/67] Test build with mage main branch --- .github/workflows/daily-builds-amd.yml | 3 ++- .github/workflows/daily-builds-arm.yml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml index 4bebc09..8fe1426 100644 --- a/.github/workflows/daily-builds-amd.yml +++ b/.github/workflows/daily-builds-amd.yml @@ -49,7 +49,8 @@ on: env: TARGET_ARCH: "amd64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + # MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'v2.13.0' }} TARGET_OS: "debian-11" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml index 0b43d52..6d63603 100644 --- a/.github/workflows/daily-builds-arm.yml +++ b/.github/workflows/daily-builds-arm.yml @@ -49,8 +49,9 @@ on: env: TARGET_ARCH: "arm64" LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} - MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} - MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'master' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + # MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'v2.13.0' }} TARGET_OS: "debian-11-arm" TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} MG_PACKAGE_PATH: "memgraph_debian-11-arm.deb"