Skip to content

Commit e190130

Browse files
BohuTANGBohuTANG
authored
Merge pull request #3344 from Xuanwo/ci
ci: Refactor the whole ci into stages
2 parents 09b1aa2 + e264ce4 commit e190130

File tree

24 files changed

+613
-457
lines changed

24 files changed

+613
-457
lines changed

‎.github/actions/build_debug/action.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: 'Build Debug'
2+
description: 'Build with debug profile'
3+
inputs:
4+
github_token:
5+
description: "Github Token"
6+
required: true
7+
runs:
8+
using: "composite"
9+
steps:
10+
- uses: arduino/setup-protoc@v1
11+
with:
12+
version: '3.x'
13+
repo-token: ${{ inputs.github_token }}
14+
15+
- uses: actions-rs/toolchain@v1
16+
with:
17+
target: ${{ matrix.config.target }}
18+
components: rustfmt, clippy
19+
20+
- uses: Swatinem/rust-cache@v1
21+
22+
- uses: actions-rs/cargo@v1
23+
with:
24+
command: build
25+
use-cross: ${{ matrix.config.cross }}
26+
args: --target ${{ matrix.config.target }}
27+
28+
- name: Upload artifact
29+
uses: actions/upload-artifact@v2
30+
with:
31+
name: debug-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
32+
path: ./target/${{ matrix.config.target }}/debug/databend-*

‎.github/actions/build_release/action.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: 'Build Release'
2+
description: 'Build with release profile'
3+
inputs:
4+
github_token:
5+
description: "Github Token"
6+
required: true
7+
runs:
8+
using: "composite"
9+
steps:
10+
- uses: arduino/setup-protoc@v1
11+
with:
12+
version: '3.x'
13+
repo-token: ${{ inputs.github_token }}
14+
15+
- uses: actions-rs/toolchain@v1
16+
with:
17+
target: ${{ matrix.config.target }}
18+
components: rustfmt, clippy
19+
20+
- uses: Swatinem/rust-cache@v1
21+
22+
- name: Build Databend
23+
uses: actions-rs/cargo@v1
24+
with:
25+
command: build
26+
use-cross: ${{ matrix.config.cross }}
27+
args: --release --target ${{ matrix.config.target }}
28+
29+
- name: Build perf-tools
30+
uses: actions-rs/cargo@v1
31+
# Only build perf tools on ubuntu
32+
if: matrix.config.os == 'ubuntu-latest'
33+
with:
34+
command: build
35+
use-cross: ${{ matrix.config.cross }}
36+
args: --release --target ${{ matrix.config.target }} --bin databend-benchmark
37+
env:
38+
RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gabi"
39+
40+
- name: Upload artifact
41+
uses: actions/upload-artifact@v2
42+
with:
43+
name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
44+
path: ./target/${{ matrix.config.target }}/release/databend-*

‎.github/actions/cache-cargo-registry/action.yaml‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.github/actions/check/action.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: 'Check'
2+
description: 'Check will do all essential checks'
3+
inputs:
4+
github_token:
5+
description: "Github Token"
6+
required: true
7+
runs:
8+
using: "composite"
9+
steps:
10+
- uses: arduino/setup-protoc@v1
11+
with:
12+
version: '3.x'
13+
repo-token: ${{ inputs.github_token }}
14+
15+
- uses: actions-rs/toolchain@v1
16+
with:
17+
components: rustfmt, clippy
18+
19+
- uses: Swatinem/rust-cache@v1
20+
21+
- name: Format
22+
uses: actions-rs/cargo@v1
23+
with:
24+
command: fmt
25+
args: --all -- --check
26+
27+
- name: Clippy
28+
uses: actions-rs/cargo@v1
29+
with:
30+
command: clippy
31+
args: --tests -- -D warnings
32+
33+
- name: Check License Header
34+
uses: apache/skywalking-eyes@main
35+
env:
36+
GITHUB_TOKEN: ${{ inputs.github_token }}
37+
with:
38+
log: info
39+
40+
- name: YAML Lint
41+
uses: ibiqlik/action-yamllint@v3
42+
with:
43+
file_or_dir: ./
44+
config_file: .yamllint.yml
45+
46+
- name: Install cargo-audit
47+
uses: actions-rs/cargo@v1
48+
if: "!contains(github.event.head_commit.message, 'skip audit')"
49+
with:
50+
command: install
51+
args: cargo-audit
52+
53+
- name: Audit dependencies
54+
uses: actions-rs/cargo@v1
55+
if: "!contains(github.event.head_commit.message, 'skip audit')"
56+
with:
57+
command: audit

‎.github/actions/publish_image/action.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: 'Publish Image'
2+
description: 'Publish releases in container image'
3+
inputs:
4+
dockerhub_username:
5+
description: "The user name of dockerhub"
6+
required: true
7+
dockerhub_token:
8+
description: "The token of dockerhub"
9+
required: true
10+
dockerhub_namespace:
11+
description: "The namespace of dockerhub"
12+
required: true
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Set up Docker Buildx
17+
uses: docker/setup-buildx-action@v1
18+
19+
- name: Login to DockerHub
20+
uses: docker/login-action@v1
21+
with:
22+
username: ${{ inputs.dockerhub_username }}
23+
password: ${{ inputs.dockerhub_token }}
24+
25+
- uses: actions/download-artifact@v2
26+
with:
27+
name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
28+
path: ./target/release
29+
30+
- name: Build and publish databend image
31+
id: docker_build
32+
uses: docker/build-push-action@v2
33+
with:
34+
push: true
35+
tags: ${{ inputs.dockerhub_namespace }}/databend:master # assume latest tag is the latest release tag
36+
context: .
37+
file: ./docker/Dockerfile
38+
- name: Image digest
39+
shell: bash
40+
run: echo ${{ steps.docker_build.outputs.digest }}
41+
42+
- name: Build and publish perf_tools images
43+
id: docker_build_perf
44+
uses: docker/build-push-action@v2
45+
with:
46+
push: true
47+
tags: ${{ inputs.dockerhub_namespace }}/perf-tool:latest # assume latest tag is the latest release tag
48+
platforms: linux/amd64, linux/arm64
49+
context: .
50+
file: ./docker/perf-tool/Dockerfile
51+
- name: Perf image digest
52+
shell: bash
53+
run: echo ${{ steps.docker_build_perf.outputs.digest }} && cat ./tests/perfs/perfs.yaml

‎.github/actions/test_stateful_standalone/action.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: 'Test Stateful Standalone'
2+
description: 'Running stateful tests in standalone mode'
3+
inputs:
4+
profile:
5+
description: 'The profile for this test'
6+
required: true
7+
default: 'debug'
8+
runs:
9+
using: "composite"
10+
steps:
11+
- uses: actions-rs/toolchain@v1
12+
with:
13+
target: ${{ matrix.config.target }}
14+
15+
- uses: Swatinem/rust-cache@v1
16+
17+
- uses: actions/download-artifact@v2
18+
with:
19+
name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
20+
path: ./target/debug
21+
22+
- name: Set up file as executable
23+
shell: bash
24+
run: |
25+
chmod +x ./target/debug/databend-*
26+
27+
- name: Minio Setup for (ubuntu-latest only)
28+
shell: bash
29+
run: |
30+
docker run -d -p 9900:9000 --name minio \
31+
-e "MINIO_ACCESS_KEY=minioadmin" \
32+
-e "MINIO_SECRET_KEY=minioadmin" \
33+
-v /tmp/data:/data \
34+
-v /tmp/config:/root/.minio \
35+
minio/minio server /data
36+
37+
export AWS_ACCESS_KEY_ID=minioadmin
38+
export AWS_SECRET_ACCESS_KEY=minioadmin
39+
export AWS_EC2_METADATA_DISABLED=true
40+
aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
41+
aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/tests/data --recursive
42+
43+
- name: Run Stateless Tests with Standalone mode (ubuntu-latest only)
44+
shell: bash
45+
run: |
46+
bash ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh

‎.github/actions/test_stateless_cluster/action.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: 'Test Stateless Cluster'
2+
description: 'Running stateless tests in cluster mode'
3+
inputs:
4+
profile:
5+
description: 'The profile for this test'
6+
required: true
7+
default: 'debug'
8+
runs:
9+
using: "composite"
10+
steps:
11+
- uses: actions-rs/toolchain@v1
12+
with:
13+
target: ${{ matrix.config.target }}
14+
15+
- uses: Swatinem/rust-cache@v1
16+
17+
- uses: actions/download-artifact@v2
18+
with:
19+
name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
20+
path: ./target/debug
21+
22+
- name: Install mysql client for macos-11
23+
shell: bash
24+
if: ${{ matrix.config.os == 'macos-11' }}
25+
run: |
26+
brew install mysql
27+
28+
- name: Install python dependences
29+
shell: bash
30+
run: |
31+
pip3 install --user boto3 "moto[all]" yapf shfmt-py
32+
33+
- name: Set up file as executable
34+
shell: bash
35+
run: |
36+
chmod +x ./target/debug/databend-*
37+
38+
- name: Run Stateless Tests with Cluster mode
39+
shell: bash
40+
run: |
41+
bash ./scripts/ci/ci-run-stateless-tests-cluster.sh

‎.github/actions/test_stateless_standalone/action.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: 'Test Stateless Standalone'
2+
description: 'Running stateless tests in standalone mode'
3+
inputs:
4+
profile:
5+
description: 'The profile for this test'
6+
required: true
7+
default: 'debug'
8+
runs:
9+
using: "composite"
10+
steps:
11+
- uses: actions-rs/toolchain@v1
12+
with:
13+
toolchain: ${{ matrix.config.toolchain }}
14+
target: ${{ matrix.config.target }}
15+
16+
- uses: Swatinem/rust-cache@v1
17+
18+
- uses: actions/download-artifact@v2
19+
with:
20+
name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
21+
path: ./target/debug
22+
23+
- name: Install mysql client for macos-11
24+
shell: bash
25+
if: ${{ matrix.config.os == 'macos-11' }}
26+
run: |
27+
brew install mysql
28+
29+
- name: Install python dependences
30+
shell: bash
31+
run: |
32+
pip3 install --user boto3 "moto[all]" yapf shfmt-py
33+
34+
- name: Set up file as executable
35+
shell: bash
36+
run: |
37+
chmod +x ./target/debug/databend-*
38+
39+
- name: Run Stateless Tests with Standalone mode
40+
shell: bash
41+
if: matrix.config.os != 'ubuntu-latest'
42+
run: |
43+
bash ./scripts/ci/ci-run-stateless-tests-standalone.sh
44+
45+
- name: Run Stateless Tests with Standalone mode, with embedded meta-store
46+
shell: bash
47+
run: |
48+
bash ./scripts/ci/ci-run-tests-embedded-meta.sh

0 commit comments

Comments
 (0)