From e264ce4500cefa42b70b47395c6c7ae8a464d65e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 14 Dec 2021 10:15:48 +0800 Subject: [PATCH] Refactor CI Signed-off-by: Xuanwo --- .github/actions/build_debug/action.yml | 32 +++++ .github/actions/build_release/action.yml | 44 +++++++ .../actions/cache-cargo-registry/action.yaml | 23 ---- .github/actions/check/action.yml | 57 +++++++++ .github/actions/publish_image/action.yml | 53 ++++++++ .../test_stateful_standalone/action.yml | 46 +++++++ .../actions/test_stateless_cluster/action.yml | 41 ++++++ .../test_stateless_standalone/action.yml | 48 +++++++ .github/actions/test_unit/action.yml | 65 ++++++++++ .github/workflows/databend-base.yml | 55 -------- .github/workflows/databend-docker.yml | 79 ------------ .github/workflows/developing.yml | 96 ++++++++++++++ .github/workflows/license-checker.yml | 20 --- .github/workflows/production.yml | 120 ++++++++++++++++++ .github/workflows/rust-audit.yml | 34 ----- .../workflows/stateful-tests-standalone.yml | 60 --------- .github/workflows/stateless-tests-cluster.yml | 47 ------- .../workflows/stateless-tests-standalone.yml | 51 -------- ...ch.yaml => test-performance-dispatch.yaml} | 0 .github/workflows/unit-tests.yml | 75 ----------- ...lack-notify.yml => utils-slack-notify.yml} | 0 .yamllint.yml | 4 + docker/Dockerfile | 16 +-- docker/perf-tool/Dockerfile | 4 +- 24 files changed, 613 insertions(+), 457 deletions(-) create mode 100644 .github/actions/build_debug/action.yml create mode 100644 .github/actions/build_release/action.yml delete mode 100644 .github/actions/cache-cargo-registry/action.yaml create mode 100644 .github/actions/check/action.yml create mode 100644 .github/actions/publish_image/action.yml create mode 100644 .github/actions/test_stateful_standalone/action.yml create mode 100644 .github/actions/test_stateless_cluster/action.yml create mode 100644 .github/actions/test_stateless_standalone/action.yml create mode 100644 .github/actions/test_unit/action.yml delete mode 100644 .github/workflows/databend-base.yml delete mode 100644 .github/workflows/databend-docker.yml create mode 100644 .github/workflows/developing.yml delete mode 100644 .github/workflows/license-checker.yml create mode 100644 .github/workflows/production.yml delete mode 100644 .github/workflows/rust-audit.yml delete mode 100644 .github/workflows/stateful-tests-standalone.yml delete mode 100644 .github/workflows/stateless-tests-cluster.yml delete mode 100644 .github/workflows/stateless-tests-standalone.yml rename .github/workflows/{databend-performance-dispatch.yaml => test-performance-dispatch.yaml} (100%) delete mode 100644 .github/workflows/unit-tests.yml rename .github/workflows/{slack-notify.yml => utils-slack-notify.yml} (100%) diff --git a/.github/actions/build_debug/action.yml b/.github/actions/build_debug/action.yml new file mode 100644 index 0000000000000..511101dd9451b --- /dev/null +++ b/.github/actions/build_debug/action.yml @@ -0,0 +1,32 @@ +name: 'Build Debug' +description: 'Build with debug profile' +inputs: + github_token: + description: "Github Token" + required: true +runs: + using: "composite" + steps: + - uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ inputs.github_token }} + + - uses: actions-rs/toolchain@v1 + with: + target: ${{ matrix.config.target }} + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v1 + + - uses: actions-rs/cargo@v1 + with: + command: build + use-cross: ${{ matrix.config.cross }} + args: --target ${{ matrix.config.target }} + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: debug-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/${{ matrix.config.target }}/debug/databend-* diff --git a/.github/actions/build_release/action.yml b/.github/actions/build_release/action.yml new file mode 100644 index 0000000000000..fbe88f7d2b8e4 --- /dev/null +++ b/.github/actions/build_release/action.yml @@ -0,0 +1,44 @@ +name: 'Build Release' +description: 'Build with release profile' +inputs: + github_token: + description: "Github Token" + required: true +runs: + using: "composite" + steps: + - uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ inputs.github_token }} + + - uses: actions-rs/toolchain@v1 + with: + target: ${{ matrix.config.target }} + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v1 + + - name: Build Databend + uses: actions-rs/cargo@v1 + with: + command: build + use-cross: ${{ matrix.config.cross }} + args: --release --target ${{ matrix.config.target }} + + - name: Build perf-tools + uses: actions-rs/cargo@v1 + # Only build perf tools on ubuntu + if: matrix.config.os == 'ubuntu-latest' + with: + command: build + use-cross: ${{ matrix.config.cross }} + args: --release --target ${{ matrix.config.target }} --bin databend-benchmark + env: + RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gabi" + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/${{ matrix.config.target }}/release/databend-* diff --git a/.github/actions/cache-cargo-registry/action.yaml b/.github/actions/cache-cargo-registry/action.yaml deleted file mode 100644 index 5a0ef4c810a2f..0000000000000 --- a/.github/actions/cache-cargo-registry/action.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: 'Cache cargo registry' -description: 'Cache the cargo registry' -inputs: - cache_reset_key: - description: 'the key for cargo cache' - required: true -runs: - using: "composite" - steps: - - name: Cache cargo registry - uses: actions/cache@v2 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-buildcache-${{ inputs.cache_reset_key }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-buildcache-${{ inputs.cache_reset_key }}- - ${{ runner.os }}-cargo-buildcache- - ${{ runner.os }}-cargo- diff --git a/.github/actions/check/action.yml b/.github/actions/check/action.yml new file mode 100644 index 0000000000000..ae13694cdf3e0 --- /dev/null +++ b/.github/actions/check/action.yml @@ -0,0 +1,57 @@ +name: 'Check' +description: 'Check will do all essential checks' +inputs: + github_token: + description: "Github Token" + required: true +runs: + using: "composite" + steps: + - uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ inputs.github_token }} + + - uses: actions-rs/toolchain@v1 + with: + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v1 + + - name: Format + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --tests -- -D warnings + + - name: Check License Header + uses: apache/skywalking-eyes@main + env: + GITHUB_TOKEN: ${{ inputs.github_token }} + with: + log: info + + - name: YAML Lint + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: ./ + config_file: .yamllint.yml + + - name: Install cargo-audit + uses: actions-rs/cargo@v1 + if: "!contains(github.event.head_commit.message, 'skip audit')" + with: + command: install + args: cargo-audit + + - name: Audit dependencies + uses: actions-rs/cargo@v1 + if: "!contains(github.event.head_commit.message, 'skip audit')" + with: + command: audit diff --git a/.github/actions/publish_image/action.yml b/.github/actions/publish_image/action.yml new file mode 100644 index 0000000000000..c6cad6fdede03 --- /dev/null +++ b/.github/actions/publish_image/action.yml @@ -0,0 +1,53 @@ +name: 'Publish Image' +description: 'Publish releases in container image' +inputs: + dockerhub_username: + description: "The user name of dockerhub" + required: true + dockerhub_token: + description: "The token of dockerhub" + required: true + dockerhub_namespace: + description: "The namespace of dockerhub" + required: true +runs: + using: "composite" + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ inputs.dockerhub_username }} + password: ${{ inputs.dockerhub_token }} + + - uses: actions/download-artifact@v2 + with: + name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/release + + - name: Build and publish databend image + id: docker_build + uses: docker/build-push-action@v2 + with: + push: true + tags: ${{ inputs.dockerhub_namespace }}/databend:master # assume latest tag is the latest release tag + context: . + file: ./docker/Dockerfile + - name: Image digest + shell: bash + run: echo ${{ steps.docker_build.outputs.digest }} + + - name: Build and publish perf_tools images + id: docker_build_perf + uses: docker/build-push-action@v2 + with: + push: true + tags: ${{ inputs.dockerhub_namespace }}/perf-tool:latest # assume latest tag is the latest release tag + platforms: linux/amd64, linux/arm64 + context: . + file: ./docker/perf-tool/Dockerfile + - name: Perf image digest + shell: bash + run: echo ${{ steps.docker_build_perf.outputs.digest }} && cat ./tests/perfs/perfs.yaml diff --git a/.github/actions/test_stateful_standalone/action.yml b/.github/actions/test_stateful_standalone/action.yml new file mode 100644 index 0000000000000..600e2914dceed --- /dev/null +++ b/.github/actions/test_stateful_standalone/action.yml @@ -0,0 +1,46 @@ +name: 'Test Stateful Standalone' +description: 'Running stateful tests in standalone mode' +inputs: + profile: + description: 'The profile for this test' + required: true + default: 'debug' +runs: + using: "composite" + steps: + - uses: actions-rs/toolchain@v1 + with: + target: ${{ matrix.config.target }} + + - uses: Swatinem/rust-cache@v1 + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/debug + + - name: Set up file as executable + shell: bash + run: | + chmod +x ./target/debug/databend-* + + - name: Minio Setup for (ubuntu-latest only) + shell: bash + run: | + docker run -d -p 9900:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket + aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/tests/data --recursive + + - name: Run Stateless Tests with Standalone mode (ubuntu-latest only) + shell: bash + run: | + bash ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh diff --git a/.github/actions/test_stateless_cluster/action.yml b/.github/actions/test_stateless_cluster/action.yml new file mode 100644 index 0000000000000..c986ec4f24b43 --- /dev/null +++ b/.github/actions/test_stateless_cluster/action.yml @@ -0,0 +1,41 @@ +name: 'Test Stateless Cluster' +description: 'Running stateless tests in cluster mode' +inputs: + profile: + description: 'The profile for this test' + required: true + default: 'debug' +runs: + using: "composite" + steps: + - uses: actions-rs/toolchain@v1 + with: + target: ${{ matrix.config.target }} + + - uses: Swatinem/rust-cache@v1 + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/debug + + - name: Install mysql client for macos-11 + shell: bash + if: ${{ matrix.config.os == 'macos-11' }} + run: | + brew install mysql + + - name: Install python dependences + shell: bash + run: | + pip3 install --user boto3 "moto[all]" yapf shfmt-py + + - name: Set up file as executable + shell: bash + run: | + chmod +x ./target/debug/databend-* + + - name: Run Stateless Tests with Cluster mode + shell: bash + run: | + bash ./scripts/ci/ci-run-stateless-tests-cluster.sh diff --git a/.github/actions/test_stateless_standalone/action.yml b/.github/actions/test_stateless_standalone/action.yml new file mode 100644 index 0000000000000..55e66535a00fb --- /dev/null +++ b/.github/actions/test_stateless_standalone/action.yml @@ -0,0 +1,48 @@ +name: 'Test Stateless Standalone' +description: 'Running stateless tests in standalone mode' +inputs: + profile: + description: 'The profile for this test' + required: true + default: 'debug' +runs: + using: "composite" + steps: + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.config.toolchain }} + target: ${{ matrix.config.target }} + + - uses: Swatinem/rust-cache@v1 + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }} + path: ./target/debug + + - name: Install mysql client for macos-11 + shell: bash + if: ${{ matrix.config.os == 'macos-11' }} + run: | + brew install mysql + + - name: Install python dependences + shell: bash + run: | + pip3 install --user boto3 "moto[all]" yapf shfmt-py + + - name: Set up file as executable + shell: bash + run: | + chmod +x ./target/debug/databend-* + + - name: Run Stateless Tests with Standalone mode + shell: bash + if: matrix.config.os != 'ubuntu-latest' + run: | + bash ./scripts/ci/ci-run-stateless-tests-standalone.sh + + - name: Run Stateless Tests with Standalone mode, with embedded meta-store + shell: bash + run: | + bash ./scripts/ci/ci-run-tests-embedded-meta.sh diff --git a/.github/actions/test_unit/action.yml b/.github/actions/test_unit/action.yml new file mode 100644 index 0000000000000..4f02c0deb93d1 --- /dev/null +++ b/.github/actions/test_unit/action.yml @@ -0,0 +1,65 @@ +name: 'Test Unite' +description: 'Running unit tests' +inputs: + github_token: + description: "Github Token" + required: true + codecov_token: + description: "CodeCov Token" + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v2 + + - uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ inputs.github_token }} + + - uses: actions-rs/toolchain@v1 + with: + target: ${{ matrix.config.target }} + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v1 + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features --no-fail-fast + env: + RUST_TEST_THREADS: 2 + RUST_LOG: debug + RUST_BACKTRACE: full + RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' + RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' + + - name: Install grcov + uses: actions-rs/cargo@v1 + with: + command: install + args: grcov + + - name: Run code coverage + uses: actions-rs/grcov@v0.1.5 + id: coverage + + - name: Upload to codecov.io + uses: codecov/codecov-action@v1 + with: + token: ${{ inputs.codecov_token }} + file: ${{ steps.coverage.outputs.report }} + + - name: Upload artifact + uses: actions/upload-artifact@v2 + if: failure() + with: + path: | + _local_fs/ + _logs/ + _meta/ + metasrv/_logs/ + query/_logs/ + store/_logs/ diff --git a/.github/workflows/databend-base.yml b/.github/workflows/databend-base.yml deleted file mode 100644 index 5a539ed18b70a..0000000000000 --- a/.github/workflows/databend-base.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Databend Base -on: - push: - paths-ignore: - - 'website/' - - '*.md' - pull_request: - paths-ignore: - - 'website/' - - '*.md' - -env: - CARGO_TERM_COLOR: always - PROTOC: protoc - -jobs: - lint_and_build: - name: "Lint & Build" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: YAML Lint - uses: ibiqlik/action-yamllint@v3 - with: - file_or_dir: ./ - config_file: .yamllint.yml - - - name: Rust setup - run: bash ./scripts/setup/dev_setup.sh - - - uses: ./.github/actions/cache-cargo-registry - with: - cache_reset_key: ${{ secrets.CACHE_RESET_KEY }} - - - name: Run clippy - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --tests -- -D warnings - env: - CARGO_INCREMENTAL: '0' - - - name: Check formating - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - env: - CARGO_INCREMENTAL: '0' - - - name: Build - run: cargo build --verbose - env: - CARGO_INCREMENTAL: '0' diff --git a/.github/workflows/databend-docker.yml b/.github/workflows/databend-docker.yml deleted file mode 100644 index 6e9b4fb6a915f..0000000000000 --- a/.github/workflows/databend-docker.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Push to Dockerhub -on: - push: - branches: main - paths-ignore: - - 'website/' - - '*.md' - -jobs: - build: - runs-on: ${{ matrix.config.os }} - strategy: - fail-fast: false - matrix: - config: - # Linux - - { os: ubuntu-latest, toolchain: stable, target: x86_64-unknown-linux-gnu, cross: false } - - { os: ubuntu-latest, toolchain: stable, target: aarch64-unknown-linux-gnu, cross: true } - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push databend - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ${{ secrets.DOCKERHUB_NAMESPACE }}/databend:master # assume latest tag is the latest release tag - context: . - file: ./docker/Dockerfile - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - - name: Rust setup - run: | - bash ./scripts/setup/dev_setup.sh - cargo install --version 0.2.1 cross - - - uses: ./.github/actions/cache-cargo-registry - with: - cache_reset_key: ${{ secrets.CACHE_RESET_KEY }} - - - name: Build Perf Tool - run: | - if [ ${{ matrix.config.cross }} = true ]; then - RUSTFLAGS="-C link-arg=-Wl,--compress-debug-sections=zlib-gabi" cross build --release --target=${{ matrix.config.target }} --bin databend-benchmark - else - cargo build --release --target=${{ matrix.config.target }} --bin databend-benchmark - objcopy --compress-debug-sections=zlib-gnu ./target/${{ matrix.config.target }}/release/databend-benchmark - fi - mkdir -p ./distro - mv ./target/${{ matrix.config.target }}/release/databend-benchmark ./distro - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Build and push perf_tools - id: docker_build_perf - uses: docker/build-push-action@v2 - with: - push: true - tags: ${{ secrets.DOCKERHUB_NAMESPACE }}/perf-tool:latest # assume latest tag is the latest release tag - platforms: linux/amd64, linux/arm64 - context: . - file: ./docker/perf-tool/Dockerfile - - - name: Perf image digest - run: echo ${{ steps.docker_build_perf.outputs.digest }} && cat ./tests/perfs/perfs.yaml diff --git a/.github/workflows/developing.yml b/.github/workflows/developing.yml new file mode 100644 index 0000000000000..634844600a2ca --- /dev/null +++ b/.github/workflows/developing.yml @@ -0,0 +1,96 @@ +name: Developing + +on: + push: + branches-ignore: + - main + paths-ignore: + - 'website/' + - '*.md' + pull_request: + paths-ignore: + - 'website/' + - '*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + PROTOC: protoc + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/check + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + build: + runs-on: ${{ matrix.config.os }} + needs: check + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, cross: false } + - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, cross: true } + - { os: macos-11, target: x86_64-apple-darwin, cross: false } + - { os: macos-11, target: aarch64-apple-darwin, cross: false } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/build_debug + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + test_unit: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_unit + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} + + test_stateless_standalone: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateless_standalone + + test_stateless_cluster: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateless_cluster + + test_stateful_standalone: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateful_standalone diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml deleted file mode 100644 index 858ea5219dc09..0000000000000 --- a/.github/workflows/license-checker.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: License checker - -on: - push: - pull_request: - branches: - - main - -jobs: - check-license: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Check License Header - uses: apache/skywalking-eyes@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - log: info diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000000000..c40f25fea269b --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,120 @@ +name: Production + +on: + push: + branches: + - main + paths-ignore: + - 'website/' + - '*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + PROTOC: protoc + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/check + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + build: + runs-on: ${{ matrix.config.os }} + needs: check + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, cross: false } + - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, cross: true } + - { os: macos-11, target: x86_64-apple-darwin, cross: false } + - { os: macos-11, target: aarch64-apple-darwin, cross: false } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/build_release + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + test_unit: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_unit + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} + + test_stateless_standalone: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateless_standalone + with: + profile: release + + test_stateless_cluster: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + - { os: macos-11, target: x86_64-apple-darwin } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateless_cluster + with: + profile: release + + test_stateful_standalone: + runs-on: ${{ matrix.config.os }} + needs: build + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/test_stateful_standalone + with: + profile: release + + publish_image: + runs-on: ${{ matrix.config.os }} + # Release after all tests have been passed. + needs: [ + test_unit, + test_stateless_standalone, + test_stateless_cluster, + test_stateful_standalone + ] + strategy: + matrix: + config: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, cross: false } + - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, cross: true } + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/publish_image + with: + dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} + dockerhub_namespace: ${{ secrets.DOCKERHUB_NAMESPACE }} diff --git a/.github/workflows/rust-audit.yml b/.github/workflows/rust-audit.yml deleted file mode 100644 index 762d7b593d08c..0000000000000 --- a/.github/workflows/rust-audit.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Audit Security - -on: - push: - paths: - - '**/Cargo.lock' - pull_request: - paths: - - '**/Cargo.lock' - schedule: - - cron: '25 4 * * *' - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - audit-security: - name: Audit Security - runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'skip audit')" - steps: - - uses: actions/checkout@v2 - - - name: Install cargo-audit - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-audit - - - name: Audit dependencies - uses: actions-rs/cargo@v1 - with: - command: audit diff --git a/.github/workflows/stateful-tests-standalone.yml b/.github/workflows/stateful-tests-standalone.yml deleted file mode 100644 index 12b9c4df42a99..0000000000000 --- a/.github/workflows/stateful-tests-standalone.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Stateful(Standalone) -on: - push: - paths-ignore: - - 'website/' - - '*.md' - pull_request: - paths-ignore: - - 'website/' - - '*.md' - -env: - CARGO_TERM_COLOR: always - PROTOC: protoc - -jobs: - build: - name: "Tests" - runs-on: ${{ matrix.config.os }} - strategy: - fail-fast: true - matrix: - config: - # Linux (ubuntu-latest only) - - { os: ubuntu-latest, toolchain: stable, target: x86_64-unknown-linux-gnu, cross: false } - steps: - - uses: actions/checkout@v2 - - - name: Rust setup - run: | - bash ./scripts/setup/dev_setup.sh -o -t - - - uses: ./.github/actions/cache-cargo-registry - with: - cache_reset_key: ${{ secrets.CACHE_RESET_KEY }} - - - name: Build - run: cargo build --verbose - env: - CARGO_INCREMENTAL: '0' - - - name: Minio Setup for (ubuntu-latest only) - run: | - docker run -d -p 9900:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket - aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/tests/data --recursive - - - - name: Run Stateless Tests with Standalone mode (ubuntu-latest only) - run: | - bash ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh diff --git a/.github/workflows/stateless-tests-cluster.yml b/.github/workflows/stateless-tests-cluster.yml deleted file mode 100644 index 77b42a1ead3ad..0000000000000 --- a/.github/workflows/stateless-tests-cluster.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Stateless(Cluster) -on: - push: - paths-ignore: - - 'website/' - - '*.md' - pull_request: - paths-ignore: - - 'website/' - - '*.md' - -env: - CARGO_TERM_COLOR: always - PROTOC: protoc - -jobs: - build: - name: "Tests" - runs-on: ${{ matrix.config.os }} - strategy: - fail-fast: true - matrix: - config: - # Linux - - { os: ubuntu-latest, toolchain: stable, target: x86_64-unknown-linux-gnu, cross: false } - # Macos - - { os: macos-latest, toolchain: stable, target: x86_64-apple-darwin, cross: false } - steps: - - name: Checkout source code - uses: actions/checkout@v2 - - - name: Rust setup - run: | - bash ./scripts/setup/dev_setup.sh -o -t - - - uses: ./.github/actions/cache-cargo-registry - with: - cache_reset_key: ${{ secrets.CACHE_RESET_KEY }} - - - name: Build - run: cargo build --verbose - env: - CARGO_INCREMENTAL: '0' - - - name: Run Stateless Tests with Cluster mode - run: | - bash ./scripts/ci/ci-run-stateless-tests-cluster.sh diff --git a/.github/workflows/stateless-tests-standalone.yml b/.github/workflows/stateless-tests-standalone.yml deleted file mode 100644 index d42aa2c87cdaa..0000000000000 --- a/.github/workflows/stateless-tests-standalone.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Stateless(Standalone) -on: - push: - paths-ignore: - - 'website/' - - '*.md' - pull_request: - paths-ignore: - - 'website/' - - '*.md' - -env: - CARGO_TERM_COLOR: always - PROTOC: protoc - -jobs: - build: - name: "Tests" - runs-on: ${{ matrix.config.os }} - strategy: - fail-fast: true - matrix: - config: - # Linux - - { os: ubuntu-latest, toolchain: stable, target: x86_64-unknown-linux-gnu, cross: false } - # Macos - - { os: macos-latest, toolchain: stable, target: x86_64-apple-darwin, cross: false } - steps: - - uses: actions/checkout@v2 - - - name: Rust setup - run: | - bash ./scripts/setup/dev_setup.sh -o -t - - - uses: ./.github/actions/cache-cargo-registry - with: - cache_reset_key: ${{ secrets.CACHE_RESET_KEY }} - - - name: Build - run: cargo build --verbose - env: - CARGO_INCREMENTAL: '0' - - - name: Run Stateless Tests with Standalone mode - if: matrix.config.os != 'ubuntu-latest' - run: | - bash ./scripts/ci/ci-run-stateless-tests-standalone.sh - - - name: Run Stateless Tests with Standalone mode, with embedded meta-store - run: | - bash ./scripts/ci/ci-run-tests-embedded-meta.sh diff --git a/.github/workflows/databend-performance-dispatch.yaml b/.github/workflows/test-performance-dispatch.yaml similarity index 100% rename from .github/workflows/databend-performance-dispatch.yaml rename to .github/workflows/test-performance-dispatch.yaml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 279d3e43d0e6c..0000000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Unit Tests -on: - push: - paths-ignore: - - 'website/' - - '*.md' - pull_request: - paths-ignore: - - 'website/' - - '*.md' - -env: - CARGO_TERM_COLOR: always - PROTOC: protoc - -jobs: - test: - name: "Tests" - runs-on: ${{ matrix.config.os }} - strategy: - fail-fast: true - matrix: - config: - # Linux - - { os: ubuntu-latest, toolchain: stable, target: x86_64-unknown-linux-gnu, cross: false } - # Macos - - { os: macos-11, toolchain: stable, target: x86_64-apple-darwin, cross: false } - steps: - - name: Checkout source code - uses: actions/checkout@v2 - - - name: Rust setup - run: bash ./scripts/setup/dev_setup.sh - - - name: Cache Rust - uses: Swatinem/rust-cache@v1 - - - name: Run test - uses: actions-rs/cargo@v1 - with: - command: test - components: rustfmt - args: --all-features --no-fail-fast - env: - RUST_TEST_THREADS: 2 - RUST_LOG: debug - RUST_BACKTRACE: full - CARGO_INCREMENTAL: '0' - RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' - RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' - - - name: Install grcov - run: cargo install --locked grcov - - - name: Run code coverage - uses: actions-rs/grcov@v0.1.5 - id: coverage - - - name: Upload to codecov.io - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ${{ steps.coverage.outputs.report }} - - - name: Upload artifact - uses: actions/upload-artifact@v2 - if: failure() - with: - path: | - _local_fs/ - _logs/ - _meta/ - metasrv/_logs/ - query/_logs/ - store/_logs/ diff --git a/.github/workflows/slack-notify.yml b/.github/workflows/utils-slack-notify.yml similarity index 100% rename from .github/workflows/slack-notify.yml rename to .github/workflows/utils-slack-notify.yml diff --git a/.yamllint.yml b/.yamllint.yml index c00a329d98d7e..26de7bed11aee 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -1,5 +1,9 @@ extends: default +ignore: | + build/ + target/ + rules: # 80 chars should be enough, but don't fail if a line is longer line-length: diff --git a/docker/Dockerfile b/docker/Dockerfile index bcdc5ec372725..4099d24fcbbda 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,7 @@ -FROM rust:1.48.0-buster AS builder - -COPY ./ /app -WORKDIR /app -RUN make setup -RUN make build - FROM debian:buster -COPY --from=builder /app/target/release/databend-query /databend-query -COPY --from=builder /app/target/release/databend-meta /databend-meta -COPY --from=builder /app/docker/databend-query-docker.toml /databend-query.toml -COPY --from=builder /app/docker/bootstrap.sh /bootstrap.sh + +COPY ./target/release/databend-query /databend-query +COPY ./target/release/databend-meta /databend-meta +COPY ./docker/databend-query-docker.toml /databend-query.toml +COPY ./docker/bootstrap.sh /bootstrap.sh ENTRYPOINT ["/bootstrap.sh"] diff --git a/docker/perf-tool/Dockerfile b/docker/perf-tool/Dockerfile index 54e42e8caa0f3..d0d1b139eb81c 100644 --- a/docker/perf-tool/Dockerfile +++ b/docker/perf-tool/Dockerfile @@ -1,7 +1,7 @@ FROM python:3 COPY ./tests/perfs/perfs.py /perfs.py COPY ./tests/perfs/compare.py /compare.py -COPY ./distro/databend-benchmark /databend-benchmark +COPY ./target/release/databend-benchmark /databend-benchmark COPY ./tests/perfs/perfs.yaml /perfs.yaml RUN rm /usr/bin/lsb_release || true @@ -15,4 +15,4 @@ ENV SERVER_HOST "127.0.0.1" ENV SERVER_PORT "9090" ENV ITERATION "3" ENV CONCURRENCY "1" -CMD ["python", "./perfs.py", "-b", "${BIN_LOCATION}", "--host", "${SERVER_HOST}", "-p", "${SERVER_PORT}", "-c", "${CONCURRENCY}}", "-i", "${ITERATION}"] \ No newline at end of file +CMD ["python", "./perfs.py", "-b", "${BIN_LOCATION}", "--host", "${SERVER_HOST}", "-p", "${SERVER_PORT}", "-c", "${CONCURRENCY}}", "-i", "${ITERATION}"]