diff --git a/.github/workflows/daily-builds-amd.yml b/.github/workflows/daily-builds-amd.yml new file mode 100644 index 0000000..8fe1426 --- /dev/null +++ b/.github/workflows/daily-builds-amd.yml @@ -0,0 +1,353 @@ +name: Daily builds for amd64 + +on: + #NOTE: Remove pull_request: before merge + pull_request: + workflow_dispatch: + inputs: + build_type: + type: choice + description: "Build type" + default: 'RelWithDebInfo' + options: + - Release + - RelWithDebInfo + - Debug + toolchain_version: + type: choice + description: "Toolchain version" + default: 'v4' + options: + - v4 + - v5 + memgraph_tag: + description: "Memgraph version (choose master to use the latest source)" + default: 'master' + mage_tag: + description: "Mage version (choose main to use the latest source)" + default: 'main' + lab_tag: + description: "Lab version (choose dev to use the latest source)" + default: 'dev' + build_mage: + description: "Build full mage image" + type: boolean + default: true + build_mage_no_ml: + description: "Build mage image without ML libraries" + type: boolean + default: true + build_memgraph_platform: + description: "Build memgraph plaftorm image" + type: boolean + default: true + build_memgraph_platform_without_mage: + description: "Build memgraph plaftorm without mage" + type: boolean + default: true + +env: + TARGET_ARCH: "amd64" + LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + # MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'v2.13.0' }} + TARGET_OS: "debian-11" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_PACKAGE_PATH: "memgraph_debian-11.deb" + MEMGRAPH_IMAGE: "memgraph/memgraph:${{ github.run_id }}" + MEMGRAPH_TAR: "memgraph_amd64.tar.gz" + MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" + MAGE_TAR: "mage_amd64.tar.gz" + MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_TAR: "mage_no-ml_amd64.tar.gz" + MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" + MGPLAT_TAR: "memgraph-platform_amd64.tar.gz" + MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_amd64.tar.gz" + MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_amd64.tar.gz" + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + +jobs: + build_memgraph: + name: "Build memgraph" + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Build memgraph" + run: | + ./scripts/pack_memgraph_via_docker.sh pack + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup + env: + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" + MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'RelWithDebInfo' }} + + - name: "Upload memgraph binary" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + path: scripts/dist/package/memgraph*.deb + + build_memgraph_image: + if: ${{ github.event.inputs.build_memgraph_image != 'false' }} + name: "Build memgraph image" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph image" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_memgraph.sh build + env: + CLEANUP: "true" + + - name: "Upload memgraph docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MEMGRAPH_TAR }} + path: scripts/dist/docker/${{ env.MEMGRAPH_TAR }} + + build_mage: + if: ${{ github.event.inputs.build_mage != 'false' }} + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build mage" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build + env: + CLEANUP: "true" + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MAGE_TAR }} + path: scripts/dist/docker/${{ env.MAGE_TAR }} + + build_mage_no_ml: + if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} + name: "Build mage without ML libraries" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build mage" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build --no-ml + env: + MAGE_IMAGE: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} + CLEANUP: "true" + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MAGE_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MAGE_NO_ML_TAR }} + + build_memgraph_platform: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm" + needs: [build_memgraph, build_mage] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MAGE_TAR }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_TAR }} + env: + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_TAR }} + + build_memgraph_platform_no_ml: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm without ML libraries" + needs: [build_memgraph, build_mage_no_ml] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MAGE_NO_ML_TAR }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_NO_ML_TAR }} + env: + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_ML_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_ML_TAR }} + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_ML_TAR }} + + build_memgraph_platform_without_mage: + if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} + name: "Build memgraph plaftorm without mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, X64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --no-mage + env: + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_MAGE_TAR }} + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_NO_MAGE_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_MAGE_TAR }} diff --git a/.github/workflows/daily-builds-arm.yml b/.github/workflows/daily-builds-arm.yml new file mode 100644 index 0000000..6d63603 --- /dev/null +++ b/.github/workflows/daily-builds-arm.yml @@ -0,0 +1,368 @@ +name: Daily builds for arm64 + +on: + #NOTE: Remove pull_request: before merge + pull_request: + workflow_dispatch: + inputs: + build_type: + type: choice + description: "Build type" + default: 'RelWithDebInfo' + options: + - Release + - RelWithDebInfo + - Debug + toolchain_version: + type: choice + description: "Toolchain version" + default: 'v4' + options: + - v4 + - v5 + memgraph_tag: + description: "Memgraph version (choose master to use the latest source)" + default: 'master' + mage_tag: + description: "Mage version (choose main to use the latest source)" + default: 'main' + lab_tag: + description: "Lab version (choose dev to use the latest source)" + default: 'dev' + build_mage: + description: "Build full mage image" + type: boolean + default: true + build_mage_no_ml: + description: "Build mage image without ML libraries" + type: boolean + default: true + build_memgraph_platform: + description: "Build memgraph plaftorm image" + type: boolean + default: true + build_memgraph_platform_without_mage: + description: "Build memgraph plaftorm without mage" + type: boolean + default: true + +env: + TARGET_ARCH: "arm64" + LAB_TAG: ${{ github.event.inputs.lab_tag || 'dev' }} + MAGE_TAG: ${{ github.event.inputs.mage_tag || 'main' }} + # MAGE_TAG: ${{ github.event.inputs.mage_tag || 'add-k-shortest-path' }} + MEMGRAPH_TAG: ${{ github.event.inputs.memgraph_tag || 'v2.13.0' }} + TARGET_OS: "debian-11-arm" + TOOLCHAIN_VERSION: ${{ github.events.inputs.toolchain_version || 'v4' }} + MG_PACKAGE_PATH: "memgraph_debian-11-arm.deb" + MEMGRAPH_IMAGE: "memgraph/memgraph:${{ github.run_id }}" + MEMGRAPH_TAR: "memgraph_arm64.tar.gz" + MAGE_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}" + MAGE_TAR: "mage_arm64.tar.gz" + MAGE_NO_ML_IMAGE: "memgraph/memgraph-mage:${{ github.run_id }}-no-ml" + MAGE_NO_ML_TAR: "mage_no-ml_arm64.tar.gz" + MGPLAT_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}" + MGPLAT_TAR: "memgraph-platform_arm64.tar.gz" + MGPLAT_NO_MAGE_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-mage" + MGPLAT_NO_MAGE_TAR: "memgraph-platform_no-mage_arm64.tar.gz" + MGPLAT_NO_ML_IMAGE: "memgraph/memgraph-platform:${{ github.run_id }}-no-ml" + MGPLAT_NO_ML_TAR: "memgraph-platform_no-ml_arm64.tar.gz" + MGPLAT_GHA_PAT_TOKEN: ${{ secrets.PAT }} + +jobs: + build_memgraph: + name: "Build memgraph" + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Build memgraph" + run: | + ./scripts/pack_memgraph_via_docker.sh pack + ./scripts/pack_memgraph_via_docker.sh copy_package + ./scripts/pack_memgraph_via_docker.sh cleanup + env: + MGPLAT_CNT_IMAGE: "memgraph/memgraph-builder:${{ env.TOOLCHAIN_VERSION }}_${{ env.TARGET_OS }}" + MGPLAT_CNT_NAME: "mgbuilder_${{ env.TARGET_OS }}" + MGPLAT_CNT_MG_ROOT: "/platform/mage/cpp/memgraph" + MGPLAT_MG_TAG: ${{ env.MEMGRAPH_TAG }} + MGPLAT_MG_BUILD_TYPE: ${{ github.event.inputs.build_type || 'RelWithDebInfo' }} + + - name: "Upload memgraph binary" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + path: scripts/dist/package/memgraph*.deb + + build_memgraph_image: + if: ${{ github.event.inputs.build_memgraph_image != 'false' }} + name: "Build memgraph image" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Checkout correct versions in submodules" + run: | + cd mage/cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph image" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_memgraph.sh build + env: + CLEANUP: "true" + + - name: "Upload memgraph docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MEMGRAPH_TAR }} + path: scripts/dist/docker/${{ env.MEMGRAPH_TAR }} + + build_mage: + if: ${{ github.event.inputs.build_mage != 'false' }} + name: "Build mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build mage" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build + env: + CLEANUP: "true" + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MAGE_TAR }} + path: scripts/dist/docker/${{ env.MAGE_TAR }} + + build_mage_no_ml: + if: ${{ github.event.inputs.build_mage_no_ml != 'false' }} + name: "Build mage without ML libraries" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + + - name: "Checkout correct versions in submodules" + run: | + cd mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build mage" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_mage.sh build --no-ml + env: + MAGE_IMAGE: ${{ env.MAGE_NO_ML_IMAGE }} + MAGE_TAR: ${{ env.MAGE_NO_ML_TAR }} + CLEANUP: "true" + + - name: "Upload mage docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MAGE_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MAGE_NO_ML_TAR }} + + build_memgraph_platform: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm" + needs: [build_memgraph, build_mage] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MAGE_TAR }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_TAR }} + env: + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_TAR }} + + build_memgraph_platform_no_ml: + if: ${{ github.event.inputs.build_memgraph_platform != 'false' }} + name: "Build memgraph plaftorm without ML libraries" + needs: [build_memgraph, build_mage_no_ml] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + cd ../mage && git fetch --all && git checkout ${{ env.MAGE_TAG }} && git pull origin ${{ env.MAGE_TAG }} + cd cpp/memgraph && git fetch --all && git checkout ${{ env.MEMGRAPH_TAG }} && git pull origin ${{ env.MEMGRAPH_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Download mage package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MAGE_NO_ML_TAR }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --mage-from-tar ${{ env.MAGE_NO_ML_TAR }} + env: + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_ML_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_ML_TAR }} + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_NO_ML_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_ML_TAR }} + + build_memgraph_platform_without_mage: + if: ${{ github.event.inputs.build_memgraph_platform_without_mage != 'false' }} + name: "Build memgraph plaftorm without mage" + needs: [build_memgraph] + runs-on: [self-hosted, Docker, ARM64] + steps: + - name: "Clean the working directory" + run: rm -rf * + + - name: "Checkout repository and submodules" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT }} + + - name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos + run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db + + - name: "Checkout correct versions in submodules" + run: | + cd lab && git fetch --all && git checkout ${{ env.LAB_TAG }} && git pull origin ${{ env.LAB_TAG }} + + - name: "Download memgraph package" + uses: actions/download-artifact@v3 + with: + name: ${{ env.MG_PACKAGE_PATH }} + + - name: "Build memgraph platform" + run: | + mv memgraph_*.deb ${{ env.MG_PACKAGE_PATH }} + ./scripts/docker_image_platform.sh build --no-mage + env: + MGPLAT_IMAGE: ${{ env.MGPLAT_NO_MAGE_IMAGE }} + MGPLAT_TAR: ${{ env.MGPLAT_NO_MAGE_TAR }} + CLEANUP: "true" + + - name: "Upload memgraph platform docker image" + uses: actions/upload-artifact@v3 + with: + name: ${{ env.MGPLAT_NO_MAGE_TAR }} + path: scripts/dist/docker/${{ env.MGPLAT_NO_MAGE_TAR }} diff --git a/Dockerfile b/Dockerfile index f595615..0af80fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # For mage ARG PY_VERSION_DEFAULT=3.9 +ARG MAGE_IMAGE FROM debian:bullseye as base @@ -32,27 +33,7 @@ RUN apt-get update && apt-get install -y \ ################################################################################################################################################### # MAGE -FROM base as mage-dev - -WORKDIR /mage -COPY mage /mage - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - -ENV PATH="/root/.cargo/bin:${PATH}" - -RUN python3 -m pip install --upgrade pip \ - && python3 -m pip install --default-timeout=1000 -r /mage/python/requirements.txt - -RUN python3 -m pip --default-timeout=1000 --no-cache-dir install torch-sparse torch-cluster torch-spline-conv \ - torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html - -RUN python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ - -# DGL build from source -RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git \ - && cd dgl && mkdir build && cd build && cmake .. \ - && make -j4 && cd ../python && python3 setup.py install +FROM $MAGE_IMAGE as mage-dev ################################################################################################################################################### @@ -91,6 +72,7 @@ RUN cd frontend && npm run build:production ################################################################################################################################################### + FROM base as final # Copy the backend artifacts diff --git a/mage b/mage index 47344f3..8653871 160000 --- a/mage +++ b/mage @@ -1 +1 @@ -Subproject commit 47344f389a058ca0d95c568fa90f1a21942629f6 +Subproject commit 8653871913403006eb5d145de3e1af9f276bb9db diff --git a/mage_from_src.Dockerfile b/mage_from_src.Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/scripts/docker_image_mage.sh b/scripts/docker_image_mage.sh index a57b71b..bcab05b 100755 --- a/scripts/docker_image_mage.sh +++ b/scripts/docker_image_mage.sh @@ -2,29 +2,61 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -MAGE_DIR="$DIR/../mage" +MAGE_ROOT="$DIR/../mage" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MAGE_ROOT/memgraph.deb}" +MAGE_IMAGE="${MAGE_IMAGE:-memgraph/memgraph-mage:latest}" +MAGE_TAR="${MAGE_TAR:-mage_$TARGET_ARCH.tar.gz}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph mage Docker image." + echo -e "" + echo -e "Env vars:" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MAGE_IMAGE -> Name for the resulting docker image" + echo -e " MAGE_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name]" + echo -e " $0 [-h|build [--no-ml]]" exit 1 } build() { - src_package="$1" - image_name="$2" - package_file="$(basename $src_package)" - mage_package_file="memgraph-${package_file#memgraph_}" - package_file_name="${package_file%.*}" - target_arch="${package_file_name#memgraph_}" - arch_suffix="${target_arch##*_}" - cp "$src_package" \ - "$MAGE_DIR/$mage_package_file" - cd ${MAGE_DIR} - docker buildx build --target prod --platform="linux/$arch_suffix" -t "$image_name" --build-arg TARGETARCH="$target_arch" -f "$MAGE_DIR/Dockerfile.release" . + dockerfile="Dockerfile.release" + if [[ "$#" -eq 1 ]]; then + if [ "$1" == "--no-ml" ]; then + dockerfile="Dockerfile.no_ML" + else + print_help + fi + fi + + mage_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PACKAGE_PATH \ + $MAGE_ROOT/$mage_package_file + cd $MAGE_ROOT + + docker buildx build \ + --target prod \ + --platform="linux/$TARGET_ARCH" \ + -t $MAGE_IMAGE \ + -f $dockerfile . mkdir -p "$DIR/dist/docker" - docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" + + mage_tar=$MAGE_TAR + mage_tar_ext=${MAGE_TAR#*.} + echo $mage_tar_ext + if [[ "$mage_tar_ext" != "tar.gz" ]]; then + mage_tar="$MAGE_TAR.tar.gz" + fi + docker save $MAGE_IMAGE | gzip -f > "$DIR/dist/docker/$mage_tar" + + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MAGE_IMAGE + fi } if [ "$#" == 0 ]; then @@ -32,10 +64,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 3 ]; then + if [[ "$#" -gt 2 ]]; then print_help fi - build "$2" "$3" + build $2 ;; *) print_help diff --git a/scripts/docker_image_memgraph.sh b/scripts/docker_image_memgraph.sh index 2c39217..34966ab 100755 --- a/scripts/docker_image_memgraph.sh +++ b/scripts/docker_image_memgraph.sh @@ -3,23 +3,55 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MEMGRAPH_DOCKER_DIR="$DIR/../mage/cpp/memgraph/release/docker" +MEMGRAPH_ROOT="$DIR/../mage/cpp/memgraph" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MEMGRAPH_ROOT/memgraph.deb}" +MEMGRAPH_IMAGE="${MEMGRAPH_IMAGE:-memgraph/memgraph:latest}" +MEMGRAPH_TAR="${MEMGRAPH_TAR:-memgraph_$TARGET_ARCH.tar.gz}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph Docker image." + echo -e "" + echo -e "Env vars:" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MEMGRAPH_IMAGE -> Name for the resulting docker image" + echo -e " MEMGRAPH_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path]" + echo -e " $0 [-h|build]" exit 1 } build() { - src_package="$1" - package_file="$(basename $src_package)" - package_file_no_prefix="${package_file#memgraph_}" - memgraph_version="${package_file_no_prefix%-1*}" - dockerize_memgraph_version="$(echo "$memgraph_version" | sed 's/+/_/g' | sed 's/~/_/g')" - $MEMGRAPH_DOCKER_DIR/package_docker "$src_package" + dockerfile="memgraph_deb.dockerfile" + + memgraph_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PACKAGE_PATH \ + $MEMGRAPH_DOCKER_DIR/$memgraph_package_file + cd $MEMGRAPH_DOCKER_DIR + + docker buildx build \ + --platform="linux/$TARGET_ARCH" \ + --tag $MEMGRAPH_IMAGE \ + --build-arg BINARY_NAME="memgraph-" \ + --build-arg EXTENSION="deb" \ + --file $dockerfile . mkdir -p "$DIR/dist/docker" - cp "$MEMGRAPH_DOCKER_DIR/memgraph-${dockerize_memgraph_version}-docker.tar.gz" "$DIR/dist/docker" + + memgraph_tar=$MEMGRAPH_TAR + memgraph_tar_ext=${MEMGRAPH_TAR#*.} + echo $memgraph_tar_ext + if [[ "$memgraph_tar_ext" != "tar.gz" ]]; then + memgraph_tar="$MEMGRAPH_TAR.tar.gz" + fi + docker save $MEMGRAPH_IMAGE | gzip -f > "$DIR/dist/docker/$memgraph_tar" + + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MEMGRAPH_IMAGE + fi } if [ "$#" == 0 ]; then @@ -27,10 +59,10 @@ if [ "$#" == 0 ]; then else case "$1" in build) - if [ "$#" -ne 2 ]; then + if [ "$#" -ne 1 ]; then print_help fi - build "$2" + build ;; *) print_help diff --git a/scripts/docker_image_platform.sh b/scripts/docker_image_platform.sh index 15054c8..6f885ad 100755 --- a/scripts/docker_image_platform.sh +++ b/scripts/docker_image_platform.sh @@ -2,48 +2,91 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -MGPLAT_ROOT="$DIR/../" +MGPLAT_ROOT="$DIR/.." MGPLAT_GHA_PAT_TOKEN="${MGPLAT_GHA_PAT_TOKEN:-github_personal_access_token}" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +MG_PACKAGE_PATH="${MG_PACKAGE_PATH:-$MGPLAT_ROOT/memgraph.deb}" +MGPLAT_IMAGE="${MGPLAT_IMAGE:-memgraph-platform_$TARGET_ARCH}" +MGPLAT_TAR="${MGPLAT_TAR:-memgraph-platform_$TARGET_ARCH.tar.gz}" +CLEANUP="${CLEANUP:-false}" + print_help() { echo -e "Builds memgraph platform Docker image." echo -e "" echo -e "Env vars:" echo -e " MGPLAT_GHA_PAT_TOKEN -> Github PAT token to download Lab's NPM package" + echo -e " TARGET_ARCH -> Target architecture for the build (amd64/arm64)" + echo -e " MG_PACKAGE_PATH -> Path to the memgraph deb pacakage" + echo -e " MGPLAT_IMAGE -> Name for the resulting docker image" + echo -e " MGPLAT_TAR -> Name of the resulting .tar.gz of the image" + echo -e " CLEANUP -> Cleanup docker images created during build, also the tar.gz mage image if --mage-from-tar is passed (true/false)" echo -e "" echo -e "How to run?" - echo -e " $0 [-h|build src_package_path image_name]" + echo -e " $0 [-h|build --mage-from-tar mage_tar_path|build --mage-from-image mage_image_with_tag|build --mage-from-src|build --no-mage]" exit 1 } -# TODO(gitbuda): An option to build wihout mage. build() { - src_package="$1" - image_name="$2" - package_file="$(basename $src_package)" - platform_package_file="memgraph-${package_file#memgraph_}" - package_file_name="${package_file%.*}" - target_arch="${package_file_name#memgraph_}" - arch_suffix="${target_arch##*_}" - cp "$src_package" \ - "$MGPLAT_ROOT/$platform_package_file" - cd "$MGPLAT_ROOT" - docker buildx build --platform="linux/$arch_suffix" -t ${image_name} \ - --build-arg TARGETARCH="$target_arch" \ + dockerfile="Dockerfile" + case "$1" in + --no-mage) + dockerfile="memgraph_and_lab.Dockerfile" + ;; + --mage-from-src) + dockerfile="mage_from_src.Dockerfile" + ;; + --mage-from-image) + mage_image=$2 + ;; + --mage-from-tar) + docker_load_out=$(docker load < $2) + mage_image=$(echo ${docker_load_out#Loaded image:} | tr -d '[:blank:]') + ;; + *) + print_help + ;; + esac + + platform_package_file="memgraph-$TARGET_ARCH.deb" + cp $MG_PACKAGE_PATH \ + $MGPLAT_ROOT/$platform_package_file + cd $MGPLAT_ROOT + docker buildx build \ + --platform="linux/$TARGET_ARCH" \ + -t ${MGPLAT_IMAGE} \ --build-arg NPM_PACKAGE_TOKEN="${MGPLAT_GHA_PAT_TOKEN}" \ - -f Dockerfile . + --build-arg MAGE_IMAGE="${mage_image}" \ + -f ${dockerfile} . mkdir -p "$DIR/dist/docker" - docker save ${image_name} | gzip -f > "$DIR/dist/docker/${image_name}.tar.gz" + + mgplat_tar=$MGPLAT_TAR + mgplat_tar_ext=${MGPLAT_TAR#*.} + if [[ "$mgplat_tar_ext" != "tar.gz" ]]; then + mgplat_tar="$MGPLAT_TAR.tar.gz" + fi + echo -e "Creating $mgplat_tar" + docker save $MGPLAT_IMAGE | gzip -f > "$DIR/dist/docker/$mgplat_tar" + + if [[ "$CLEANUP" == "true" ]]; then + docker image rm $MGPLAT_IMAGE + if [[ "$1" == "--mage-from-tar" || "$1" == "--mage-from-image" ]]; then + docker image rm $mage_image + fi + if [[ "$1" == "--mage-from-tar" ]]; then + rm $2 + fi + fi } -if [ "$#" == 0 ]; then +if [[ "$#" -eq 0 ]]; then print_help else case "$1" in build) - if [ "$#" -ne 3 ]; then + if [[ "$#" -lt 2 || "$#" -gt 3 ]]; then print_help fi - build "$2" "$3" + build $2 $3 ;; *) print_help diff --git a/scripts/pack_memgraph_via_docker.sh b/scripts/pack_memgraph_via_docker.sh index 6bdae82..3e19829 100755 --- a/scripts/pack_memgraph_via_docker.sh +++ b/scripts/pack_memgraph_via_docker.sh @@ -3,12 +3,14 @@ set -eo pipefail DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # NOTE: The builder container image defines for which operating system Memgraph will be built. -MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-10}" +MGPLAT_CNT_IMAGE="${MGPLAT_CNT_IMAGE:-memgraph/memgraph-builder:v4_debian-11}" MGPLAT_CNT_NAME="${MGPLAT_CNT_NAME:-mgbuild_builder}" MGPLAT_CNT_MG_ROOT="${MGPLAT_CNT_MG_ROOT:-/platform/mage/cpp/memgraph}" +MGPLAT_CNT_OUTPUT_DIR="$MGPLAT_CNT_MG_ROOT/build/output" MGPLAT_MG_TAG="${MGPLAT_MG_TAG:-master}" MGPLAT_MG_BUILD_TYPE="${MGPLAT_MG_BUILD_TYPE:-RelWithDebInfo}" MGPLAT_DIST_BINARY="$DIR/dist/binary" +MGPLAT_DIST_PACKAGE="$DIR/dist/package" print_help() { echo -e "Builds memgraph binary and package via Docker build container." echo -e "" @@ -20,7 +22,7 @@ print_help() { echo -e " MGPLAT_MG_BUILD_TYPE -> Debug|Release|RelWithDebInfo" echo -e "" echo -e "How to run?" - echo -e " $0 [pack|cleanup|copy_binary]" + echo -e " $0 [pack|cleanup|copy_binary|copy_package]" exit 1 } @@ -32,10 +34,7 @@ docker_run () { echo "Cleanup of the old exited container..." docker rm "$cnt_name" fi - docker run -d \ - -v "$DIR/..:/platform" \ - -v "$DIR/dist/package:$MGPLAT_CNT_MG_ROOT/build/output" \ - --network host --name "$cnt_name" "$cnt_image" + docker run -d --network host --name "$cnt_name" "$cnt_image" fi echo "The $cnt_image container is active under $cnt_name name!" } @@ -47,17 +46,17 @@ docker_stop_rm() { docker_exec() { cnt_cmd="$1" - docker exec -it "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd" + docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd" } build_pack() { cd "$DIR" # shellcheck disable=SC1091 source build_memgraph_native.sh - mkdir -p dist/binary docker_run "$MGPLAT_CNT_NAME" "$MGPLAT_CNT_IMAGE" - docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" + docker cp "$DIR/.." "$MGPLAT_CNT_NAME:/platform" docker_exec "git config --global --add safe.directory $MGPLAT_CNT_MG_ROOT" + docker cp "$DIR/build_memgraph_native.sh" "$MGPLAT_CNT_NAME:/" mg_root="MGPLAT_MG_ROOT=$MGPLAT_CNT_MG_ROOT" mg_tag="MGPLAT_MG_TAG=$MGPLAT_MG_TAG" mg_build_type="MGPLAT_MG_BUILD_TYPE=$MGPLAT_MG_BUILD_TYPE" @@ -65,16 +64,24 @@ build_pack() { } cleanup() { - docker_exec "rm -rf $MGPLAT_CNT_MG_ROOT/build/*" - docker_exec "$MGPLAT_CNT_MG_ROOT/libs/cleanup.sh" + docker_stop_rm $MGPLAT_CNT_NAME +} + +copy_package() { + mkdir -p $MGPLAT_DIST_PACKAGE + last_package_name=$(docker exec "$MGPLAT_CNT_NAME" bash -c "cd $MGPLAT_CNT_OUTPUT_DIR && ls -t memgraph* | head -1") + package_host_destination="$MGPLAT_DIST_PACKAGE/$last_package_name" + docker cp "$MGPLAT_CNT_NAME:$MGPLAT_CNT_OUTPUT_DIR/$last_package_name" "$package_host_destination" + echo "Package $last_package_name saved to $package_host_destination." } copy_binary() { + mkdir -p $MGPLAT_DIST_BINARY cnt_cmd="echo \$(readlink $MGPLAT_CNT_MG_ROOT/build/memgraph)" cnt_binary_path=$(docker exec "$MGPLAT_CNT_NAME" bash -c "$cnt_cmd") binary_name="$(basename $cnt_binary_path)" src_cnt_binary_path="$MGPLAT_CNT_NAME:$cnt_binary_path" - docker cp -L "$src_cnt_binary_path" "$DIR/dist/binary/" + docker cp -L "$src_cnt_binary_path" "$MGPLAT_DIST_BINARY" } if [ "$#" == 0 ]; then @@ -93,6 +100,9 @@ else copy_binary) copy_binary ;; + copy_package) + copy_package + ;; *) print_help ;;