ci: Refactor the whole ci into stages

.github/actions/build_debug/action.yml

@@ -0,0 +1,32 @@
+name: 'Build Debug'
+description: 'Build with debug profile'
+inputs:
+  github_token:
+    description: "Github Token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: arduino/setup-protoc@v1
+      with:
+        version: '3.x'
+        repo-token: ${{ inputs.github_token }}
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        target: ${{ matrix.config.target }}
+        components: rustfmt, clippy
+
+    - uses: Swatinem/rust-cache@v1
+
+    - uses: actions-rs/cargo@v1
+      with:
+        command: build
+        use-cross: ${{ matrix.config.cross }}
+        args: --target ${{ matrix.config.target }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: debug-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/${{ matrix.config.target }}/debug/databend-*

.github/actions/build_release/action.yml

@@ -0,0 +1,44 @@
+name: 'Build Release'
+description: 'Build with release profile'
+inputs:
+  github_token:
+    description: "Github Token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: arduino/setup-protoc@v1
+      with:
+        version: '3.x'
+        repo-token: ${{ inputs.github_token }}
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        target: ${{ matrix.config.target }}
+        components: rustfmt, clippy
+
+    - uses: Swatinem/rust-cache@v1
+
+    - name: Build Databend
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        use-cross: ${{ matrix.config.cross }}
+        args: --release --target ${{ matrix.config.target }}
+
+    - name: Build perf-tools
+      uses: actions-rs/cargo@v1
+      # Only build perf tools on ubuntu
+      if: matrix.config.os == 'ubuntu-latest'
+      with:
+        command: build
+        use-cross: ${{ matrix.config.cross }}
+        args: --release --target ${{ matrix.config.target }} --bin databend-benchmark
+      env:
+        RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gabi"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/${{ matrix.config.target }}/release/databend-*

.github/actions/check/action.yml

@@ -0,0 +1,57 @@
+name: 'Check'
+description: 'Check will do all essential checks'
+inputs:
+  github_token:
+    description: "Github Token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: arduino/setup-protoc@v1
+      with:
+        version: '3.x'
+        repo-token: ${{ inputs.github_token }}
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        components: rustfmt, clippy
+
+    - uses: Swatinem/rust-cache@v1
+
+    - name: Format
+      uses: actions-rs/cargo@v1
+      with:
+        command: fmt
+        args: --all -- --check
+
+    - name: Clippy
+      uses: actions-rs/cargo@v1
+      with:
+        command: clippy
+        args: --tests -- -D warnings
+
+    - name: Check License Header
+      uses: apache/skywalking-eyes@main
+      env:
+        GITHUB_TOKEN: ${{ inputs.github_token }}
+      with:
+        log: info
+
+    - name: YAML Lint
+      uses: ibiqlik/action-yamllint@v3
+      with:
+        file_or_dir: ./
+        config_file: .yamllint.yml
+
+    - name: Install cargo-audit
+      uses: actions-rs/cargo@v1
+      if: "!contains(github.event.head_commit.message, 'skip audit')"
+      with:
+        command: install
+        args: cargo-audit
+
+    - name: Audit dependencies
+      uses: actions-rs/cargo@v1
+      if: "!contains(github.event.head_commit.message, 'skip audit')"
+      with:
+        command: audit

.github/actions/publish_image/action.yml

@@ -0,0 +1,53 @@
+name: 'Publish Image'
+description: 'Publish releases in container image'
+inputs:
+  dockerhub_username:
+    description: "The user name of dockerhub"
+    required: true
+  dockerhub_token:
+    description: "The token of dockerhub"
+    required: true
+  dockerhub_namespace:
+    description: "The namespace of dockerhub"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ inputs.dockerhub_username }}
+        password: ${{ inputs.dockerhub_token }}
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/release
+
+    - name: Build and publish databend image
+      id: docker_build
+      uses: docker/build-push-action@v2
+      with:
+        push: true
+        tags: ${{ inputs.dockerhub_namespace }}/databend:master # assume latest tag is the latest release tag
+        context: .
+        file: ./docker/Dockerfile
+    - name: Image digest
+      shell: bash
+      run: echo ${{ steps.docker_build.outputs.digest }}
+
+    - name: Build and publish perf_tools images
+      id: docker_build_perf
+      uses: docker/build-push-action@v2
+      with:
+        push: true
+        tags: ${{ inputs.dockerhub_namespace }}/perf-tool:latest # assume latest tag is the latest release tag
+        platforms: linux/amd64, linux/arm64
+        context: .
+        file: ./docker/perf-tool/Dockerfile
+    - name: Perf image digest
+      shell: bash
+      run: echo ${{ steps.docker_build_perf.outputs.digest }} && cat ./tests/perfs/perfs.yaml

.github/actions/test_stateful_standalone/action.yml

@@ -0,0 +1,46 @@
+name: 'Test Stateful Standalone'
+description: 'Running stateful tests in standalone mode'
+inputs:
+  profile:
+    description: 'The profile for this test'
+    required: true
+    default: 'debug'
+runs:
+  using: "composite"
+  steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        target: ${{ matrix.config.target }}
+
+    - uses: Swatinem/rust-cache@v1
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/debug
+
+    - name: Set up file as executable
+      shell: bash
+      run: |
+        chmod +x ./target/debug/databend-*
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d -p 9900:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/tests/data  --recursive
+
+    - name: Run Stateless Tests with Standalone mode (ubuntu-latest only)
+      shell: bash
+      run: |
+        bash ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh

.github/actions/test_stateless_cluster/action.yml

@@ -0,0 +1,41 @@
+name: 'Test Stateless Cluster'
+description: 'Running stateless tests in cluster mode'
+inputs:
+  profile:
+    description: 'The profile for this test'
+    required: true
+    default: 'debug'
+runs:
+  using: "composite"
+  steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        target: ${{ matrix.config.target }}
+
+    - uses: Swatinem/rust-cache@v1
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/debug
+
+    - name: Install mysql client for macos-11
+      shell: bash
+      if: ${{ matrix.config.os == 'macos-11' }}
+      run: |
+        brew install mysql
+
+    - name: Install python dependences
+      shell: bash
+      run: |
+        pip3 install --user boto3 "moto[all]" yapf shfmt-py
+
+    - name: Set up file as executable
+      shell: bash
+      run: |
+        chmod +x ./target/debug/databend-*
+
+    - name: Run Stateless Tests with Cluster mode
+      shell: bash
+      run: |
+        bash ./scripts/ci/ci-run-stateless-tests-cluster.sh

.github/actions/test_stateless_standalone/action.yml

@@ -0,0 +1,48 @@
+name: 'Test Stateless Standalone'
+description: 'Running stateless tests in standalone mode'
+inputs:
+  profile:
+    description: 'The profile for this test'
+    required: true
+    default: 'debug'
+runs:
+  using: "composite"
+  steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.config.toolchain }}
+        target: ${{ matrix.config.target }}
+
+    - uses: Swatinem/rust-cache@v1
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/debug
+
+    - name: Install mysql client for macos-11
+      shell: bash
+      if: ${{ matrix.config.os == 'macos-11' }}
+      run: |
+        brew install mysql
+
+    - name: Install python dependences
+      shell: bash
+      run: |
+        pip3 install --user boto3 "moto[all]" yapf shfmt-py
+
+    - name: Set up file as executable
+      shell: bash
+      run: |
+        chmod +x ./target/debug/databend-*
+
+    - name: Run Stateless Tests with Standalone mode
+      shell: bash
+      if: matrix.config.os != 'ubuntu-latest'
+      run: |
+        bash ./scripts/ci/ci-run-stateless-tests-standalone.sh
+
+    - name: Run Stateless Tests with Standalone mode, with embedded meta-store
+      shell: bash
+      run: |
+        bash ./scripts/ci/ci-run-tests-embedded-meta.sh