diff --git a/.github/workflows/daily-platform-build-new.yml b/.github/workflows/daily-platform-build-new.yml new file mode 100644 index 0000000..71aa763 --- /dev/null +++ b/.github/workflows/daily-platform-build-new.yml @@ -0,0 +1,46 @@ +name: Daily Platform Build + +on: [pull_request, workflow_dispatch] + +jobs: + build_memgraph: + name: "Build memgraph Debian 11" + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + token: ${{ secrets.PAT }} + - name: "Build package" + run: | + git clone -b master https://github.com/memgraph/memgraph.git + cd memgraph + ./release/package/run.sh package debian-11 Release + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: memgraph-debian-11 + path: memgraph/build/output/debian-11/memgraph*.deb + + build_memgraph_arm: + name: "Build memgraph Debian 11 ARM" + runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + timeout-minutes: 120 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + token: ${{ secrets.PAT }} + - name: "Build package" + run: | + git clone -b master https://github.com/memgraph/memgraph.git + cd memgraph + ./release/package/run.sh package debian-11-arm Release + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: memgraph-debian-aarch64 + path: memgraph/build/output/debian-11-arm/memgraph*.deb diff --git a/.github/workflows/daily-platform-build.yml b/.github/workflows/daily-platform-build.yml new file mode 100644 index 0000000..5eace08 --- /dev/null +++ b/.github/workflows/daily-platform-build.yml @@ -0,0 +1,79 @@ +name: Daily Platform Build + +# on: [pull_request, workflow_dispatch] +on: [workflow_dispatch] + +jobs: + artifacts_build: + name: "Build artifacts" + runs-on: [self-hosted, Linux, X64, Diff, Ubuntu20.04] + env: + THREADS: 24 + PY_VERSION: "3.9" + + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + token: ${{ secrets.PAT }} + + - name: Build community binaries + run: | + git clone -b master https://github.com/memgraph/memgraph.git + + cd memgraph + + # Activate toolchain. + source /opt/toolchain-v4/activate + + # Initialize dependencies. + ./init + + # Build community binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. + make -j$THREADS && make -j$THREADS mgconsole + + mkdir -p output + cd output + + cpack -G DEB --config "../CPackConfig.cmake" + + cd ../../../ + + mkdir -p scripts/resources + + find memgraph/build/output -type f -name "*.deb" -exec mv -f {} scripts/resources/memgraph-daily-deb_amd64.deb \; + + rm -rf memgraph + + - name: "Upload DEB" + uses: actions/upload-artifact@v3 + with: + name: daily-deb + path: scripts/resources/memgraph*.deb + + # - name: Build MAGE image + # run: | + # cd scripts + + # ./build_mage_img.sh main daily-mage daily-deb + + # - name: "Upload MAGE image" + # uses: actions/upload-artifact@v3 + # with: + # name: daily-mage + # path: resources/daily-mage.tar.gz + + # - name: Build platform image + # run: | + # ./build_platform_img.sh master daily-platform daily-deb ${{ secrets.PAT }} + + # - name: "Upload platform image" + # uses: actions/upload-artifact@v3 + # with: + # name: daily-platform + # path: resources/daily-platform.tar.gz diff --git a/.gitignore b/.gitignore index 8876e2a..0645e90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +scripts/resources scripts/dist *.deb diff --git a/scripts/README.md b/scripts/README.md index 2e2a2ec..fcf7177 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,3 +1,72 @@ +# Scripts for creating custom docker images + +Each folder contains `.sh` script for creating either docker image or memgraph debian. Before running scripts, you need to change permissions on `build_` scripts with command: + +``` +sudo chmod 777 +``` + +Additional note, always check before running any scripts if your local user has the permission to read and write over the `resources` directory, which gets created in the `scripts` directory. + +## Creating Memgraph debian 11 package + +To create a Memgraph `.deb` file do the following steps: + +1. Run the `build_deb.sh` script as follows to create the deb from the GitHub repository: + + ```console + $ sudo ./build_deb.sh {branch_name} {deb_name} + ``` + - `{branch_name}`: any branch from the `memgraph` repo, e.g. `master` or `T1220-MG-properties-c++-api-bug`; for the script to pull and build `memgraph` from it + - `{deb_name}`: used to name the final `.deb` package file: `memgraph-{deb_name}_amd64.deb` + + +The script stores all `.deb` files in the `output_debian` folder. + +## Creating MAGE Docker image + +> ### This script releases docker images **only** for `amd64/linux` platform arhitectures. + +1. Store the memgraph Debian package file in the `memgraph_deb_files` folder with the exact name: `memgraph-{deb_name}_amd64.deb` and run the script with the following command: + + ``` + sudo ./build_mage_img.sh {branch_name} {image_name} {deb_name} + ``` + - `{branch_name}` - branch name, e.g. `main` + - `{deb_name}` - deb package filename + - `{image_name}` - for the final image name + +3. The docker image will be stored in the `output` folder in the `tar.gz` format, e.g. `{image_name}.tar.gz`. To check how to load the image, jump to the [Loading custom docker image](#loading-custom-docker-image) section. + + +## Creating Platform Docker image + +> ### Note: Lab is a private repo so this build is unavailable to public. You can still build Memgraph + MAGE above and use downloaded Lab with your image. + +> ### Docker image is released for platform `amd64/linux` arhitectures only. + +1. Make sure you added memgraph debian in `memgraph_deb_files`, configured `ssh` correctly to connect to `github` and run script with following command: + ``` + ./build_platform_img.sh {branch_name} {image_name} {deb_name} {github_pat_token} + + ``` + - `{branch_name}` - branch name of memgraph-platform, e.g. `main` + - `{image_name}` - name the final image name of memgraph platform + - `{deb_name}` - middle name of the deb file you provided in previous steps, e.g. memgraph-{deb_name}_amd64.deb + - `{github_pat_token}` - your github pat token for downloading Memgraph Lab + + +2. Platform docker image can be also built with only memgraph and lab wihout mage. run script with following command: + ``` + ./build_platform_img.sh main image_name deb_package_name github_pat_token true + + ``` + +Docker image will be stored in folder `output` in `tar.gz` format, for example `image_name.tar.gz`. + + +**NOTE** Deb packages need to be stored in format `memgraph-{deb_name}_amd64.deb`, for example `memgraph-feature_amd64.deb` and you start script with `feature`. + # Scripts for creating custom memgraph packages and docker images Run `{script}.sh -h` to figure the details and how to start building. @@ -15,6 +84,16 @@ Run `{script}.sh -h` to figure the details and how to start building. ## Loading custom docker image Docker image can be loaded with (it takes some time): + +``` +docker load < .tar.gz +``` + +and then used for example like this: + +``` +docker run -it --rm -p 7687:7687 --name +``` ``` docker load -i .tar.gz ``` diff --git a/scripts/_build_mg.sh b/scripts/_build_mg.sh new file mode 100644 index 0000000..20db605 --- /dev/null +++ b/scripts/_build_mg.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +git config --global --add safe.directory /memgraph + +source /opt/toolchain-v4/activate + +cd memgraph +./environment/os/debian-11.sh install MEMGRAPH_BUILD_DEPS +./init + +mkdir -p build +cd build + +cmake -DCMAKE_BUILD_TYPE=Release .. && make -j4 memgraph && make -j4 mgconsole + +cd output +cpack -G DEB --config "../CPackConfig.cmake" diff --git a/scripts/build_deb.sh b/scripts/build_deb.sh new file mode 100755 index 0000000..970ea55 --- /dev/null +++ b/scripts/build_deb.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +CURR_DIR="$PWD" +MEMGRAPH_DIR="$CURR_DIR/memgraph" + +cloned=true +branch_name=$1 +deb_name=$2 + +git clone -b $branch_name https://github.com/memgraph/memgraph.git + +mkdir -p resources + +docker pull memgraph/memgraph-builder +docker stop meme-build || true && docker rm meme-build || true + +docker run --name meme-build -itd -v $MEMGRAPH_DIR:/memgraph -v $CURR_DIR/resources/output:/memgraph/build/output --entrypoint bash memgraph/memgraph-builder +docker exec -i meme-build bash < $CURR_DIR/_build_mg.sh +docker stop meme-build +docker rm meme-build + +find $CURR_DIR/resources/output -type f -name "*.deb" -exec mv -f {} resources/memgraph-${deb_name}_amd64.deb \; + +if $cloned;then + rm -rf $MEMGRAPH_DIR +fi +rm -rf $CURR_DIR/resources/output + diff --git a/scripts/build_mage_img.sh b/scripts/build_mage_img.sh new file mode 100755 index 0000000..d6c1bd4 --- /dev/null +++ b/scripts/build_mage_img.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +CURR_DIR="$PWD" +MAGE_DIR="$CURR_DIR/mage" + +branch=$1 +image_name=$2 +deb_name=$3 + +git clone --recurse-submodules -b $1 https://github.com/memgraph/mage.git + +cp "${CURR_DIR}/resources/memgraph-${deb_name}_amd64.deb" "${MAGE_DIR}/memgraph-${deb_name}_amd64.deb" + +cd ${MAGE_DIR} + +docker buildx build --no-cache --target prod --platform=linux/amd64 -t ${image_name} --build-arg TARGETARCH=${deb_name}_amd64 -f ${MAGE_DIR}/Dockerfile.release . + +cd ${CURR_DIR} + +docker save ${image_name} | gzip -f > "${CURR_DIR}/resources/${image_name}.tar.gz" +docker rmi ${image_name} + +rm -rf ${image_name} +rm -rf ${MAGE_DIR} diff --git a/scripts/build_platform_img.sh b/scripts/build_platform_img.sh new file mode 100755 index 0000000..c2120ff --- /dev/null +++ b/scripts/build_platform_img.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +CURR_DIR="$PWD" +PLATFORM_DIR="${CURR_DIR}/memgraph-platform" + +branch=$1 +image_name=$2 +target_arch=$3 +token=$4 +memgraph_and_lab=${5-false} + +git clone --recurse-submodules -b $1 https://github.com/memgraph/memgraph-platform.git + +cp "${CURR_DIR}/resources/memgraph-${target_arch}_amd64.deb" "${PLATFORM_DIR}/memgraph-${target_arch}_amd64.deb" +cd ${PLATFORM_DIR} + +dockerfile=${PLATFORM_DIR}/Dockerfile + +if [ "${memgraph_and_lab}" == true ]; +then + dockerfile = ${PLATFORM_DIR}/memgraph_and_lab.Dockerfile +fi + +docker buildx build --platform=linux/amd64 -t ${image_name} --build-arg TARGETARCH=${target_arch}_amd64 --build-arg NPM_PACKAGE_TOKEN=${token} -f ${dockerfile} . + +cd ${CURR_DIR} + +docker save ${image_name} | gzip -f > "${CURR_DIR}/resources/${image_name}.tar.gz" +docker rmi ${image_name} + +rm -rf ${image_name} +rm -rf ${PLATFORM_DIR}