From 350f72f12f8eef21a57e0d4721658780113d5197 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 20:09:17 +0530 Subject: [PATCH 01/14] Add docker compose based integration tests on each PR --- .github/workflows/build.yaml | 20 ++++++++++++--- Dockerfile | 1 + Dockerfile.release | 47 ------------------------------------ Dockerfile.static | 34 -------------------------- docker-compose.yaml | 30 ++++++++++++----------- 5 files changed, 33 insertions(+), 99 deletions(-) delete mode 100644 Dockerfile.release delete mode 100644 Dockerfile.static diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 71776590b..8e4f1f1b3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -37,11 +37,23 @@ jobs: # push: true # tags: ${{ steps.meta.outputs.tags }} - test: + docker-compose-test: + name: Docker Compose Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Tests + uses: isbang/compose-action@v1.4.1 + with: + compose-file: "./docker-compose.yaml" + + unit-tests: name: Test Suite runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -55,7 +67,7 @@ jobs: name: Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -71,7 +83,7 @@ jobs: name: Clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: profile: minimal diff --git a/Dockerfile b/Dockerfile index 928302206..21b9c8a62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ FROM gcr.io/distroless/cc:latest WORKDIR /parseable +COPY --from=builder /lib/x86_64-linux-gnu/liblzma.so* /lib/x86_64-linux-gnu/ COPY --from=builder /parseable/target/release/parseable /usr/local/bin/parseable CMD ["/usr/local/bin/parseable"] diff --git a/Dockerfile.release b/Dockerfile.release deleted file mode 100644 index 20d92a2a7..000000000 --- a/Dockerfile.release +++ /dev/null @@ -1,47 +0,0 @@ -# Parseable Server (C) 2022 - 2023 Parseable, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -FROM alpine:3.16 - -RUN apk update --quiet && \ - apk add -q --no-cache tini - -# Create appuser -ENV USER=parseable -ENV UID=10001 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" - -# This directory should hold all the data related to parseable so we're going -# to move our PWD in there. -WORKDIR /parseable - -RUN \ - wget https://github.com/parseablehq/parseable/releases/latest/download/parseable_linux_x86_64 -O /bin/parseable && \ - chmod +x /bin/parseable - -USER parseable:parseable - -EXPOSE 8000/tcp - -ENTRYPOINT ["tini", "--"] -CMD /bin/parseable diff --git a/Dockerfile.static b/Dockerfile.static deleted file mode 100644 index e9ebc78eb..000000000 --- a/Dockerfile.static +++ /dev/null @@ -1,34 +0,0 @@ -# Parseable Server (C) 2022 - 2023 Parseable, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# Compile -FROM rust:1.67-alpine AS compiler - -WORKDIR /parseable - -RUN apk add --no-cache musl-dev - -COPY . . - -RUN set -eux; \ - apkArch="$(apk --print-arch)"; \ - if [ "$apkArch" = "aarch64" ]; then \ - export JEMALLOC_SYS_WITH_LG_PAGE=16; \ - fi && \ - cargo build --release --target x86_64-unknown-linux-musl - -FROM scratch AS export-stage - -COPY --from=compiler parseable/target/x86_64-unknown-linux-musl/release/parseable parseable diff --git a/docker-compose.yaml b/docker-compose.yaml index ada0e4803..a7de3c28c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,7 +30,9 @@ services: - parseable parseable: - image: parseable/parseable:v0.2.1 + build: + context: . + dockerfile: Dockerfile command: ["parseable", "s3-store"] ports: - 8000 @@ -54,7 +56,7 @@ services: - minio quest-smoke: - image: parseable/quest:v0.1 + image: ghcr.io/parseablehq/quest:main command: ["smoke", "http://parseable:8000", "admin", "admin"] networks: - parseable @@ -66,15 +68,15 @@ services: delay: 10s max_attempts: 10 - quest-load: - image: parseable/quest:v0.1 - command: ["load", "http://parseable:8000", "admin", "admin"] - networks: - - parseable - depends_on: - - quest-smoke - deploy: - restart_policy: - condition: on-failure - delay: 10s - max_attempts: 10 + # quest-load: + # image: ghcr.io/parseablehq/quest:main + # command: ["load", "http://parseable:8000", "admin", "admin"] + # networks: + # - parseable + # depends_on: + # - quest-smoke + # deploy: + # restart_policy: + # condition: on-failure + # delay: 10s + # max_attempts: 10 From 3c40d2d9ee852eb74ebadbb9e7b7ffc3d303141e Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 20:36:33 +0530 Subject: [PATCH 02/14] temp --- .github/workflows/build.yaml | 41 ++++++------------------------------ 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e4f1f1b3..8e456ac36 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,3 @@ -## on: [push, pull_request] on: pull_request: paths-ignore: @@ -7,50 +6,24 @@ on: - 'assets/**' - '**.md' -name: Continuous Integration +name: Parseable CI Workflow jobs: - # docker: - # name: Docker Build & Push - # runs-on: ubuntu-latest - # steps: - # - name: Docker meta - # id: meta - # uses: docker/metadata-action@v4 - # with: - # images: | - # parseable/parseable - # tags: | - # type=sha - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # - name: Login to DockerHub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Build and push - # uses: docker/build-push-action@v3 - # with: - # push: true - # tags: ${{ steps.meta.outputs.tags }} docker-compose-test: - name: Docker Compose Test + name: Docker Compose integration tests runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - - name: Tests uses: isbang/compose-action@v1.4.1 with: - compose-file: "./docker-compose.yaml" + compose-file: ./docker-compose.yaml + up-flag: --build unit-tests: - name: Test Suite + name: Unit tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -64,7 +37,7 @@ jobs: command: test fmt: - name: Rustfmt + name: Rust fmt check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -80,7 +53,7 @@ jobs: args: --all -- --check clippy: - name: Clippy + name: Cargo Clippy check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 7f3caf4ea4abaf85d8643282586119349cfc2842 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 20:57:31 +0530 Subject: [PATCH 03/14] temp --- .github/workflows/build.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e456ac36..1116b24e9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,10 +17,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Tests - uses: isbang/compose-action@v1.4.1 + uses: cloudposse/github-action-docker-compose-test-run@main with: - compose-file: ./docker-compose.yaml - up-flag: --build + file: ./docker-compose.yaml + service: minio parseable quest-smoke + command: docker compose up --build unit-tests: name: Unit tests From cb7abc51d59c286798a430162094e5adb9d40ca9 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 21:17:11 +0530 Subject: [PATCH 04/14] temp --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1116b24e9..c84a05cbd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,8 +20,8 @@ jobs: uses: cloudposse/github-action-docker-compose-test-run@main with: file: ./docker-compose.yaml - service: minio parseable quest-smoke - command: docker compose up --build + service: quest-smoke + command: smoke http://parseable:8000 admin admin unit-tests: name: Unit tests From a8022b88336f85edfe3577d6aa4acf69ef1cd497 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 21:38:51 +0530 Subject: [PATCH 05/14] temp --- .github/workflows/build.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c84a05cbd..0cab63ebd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,12 +16,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Tests - uses: cloudposse/github-action-docker-compose-test-run@main - with: - file: ./docker-compose.yaml - service: quest-smoke - command: smoke http://parseable:8000 admin admin + - name: Start compose + run: docker-compose -f "docker-compose.yml" up --build + - name: Stop compose + if: always() + run: docker-compose -f "docker-compose.yml" down unit-tests: name: Unit tests From 99a0001f1d0c33479d616c891d143fe591bc1276 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 21:40:47 +0530 Subject: [PATCH 06/14] temp --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0cab63ebd..c571d7310 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,10 +17,10 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Start compose - run: docker-compose -f "docker-compose.yml" up --build + run: docker-compose -f "docker-compose.yaml" up --build - name: Stop compose if: always() - run: docker-compose -f "docker-compose.yml" down + run: docker-compose -f "docker-compose.yaml" down unit-tests: name: Unit tests From af65c96e680f53aba1a04dda293e95c1a5f5476a Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 14 Feb 2023 22:06:54 +0530 Subject: [PATCH 07/14] temp --- docker-compose.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index a7de3c28c..6d755d786 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -45,6 +45,8 @@ services: - P_STAGING_DIR=/tmp/data - P_USERNAME=admin - P_PASSWORD=admin + - RUST_BACKTRACE=1 + - RUST_LOG=info networks: - parseable healthcheck: @@ -55,7 +57,7 @@ services: depends_on: - minio - quest-smoke: + quest: image: ghcr.io/parseablehq/quest:main command: ["smoke", "http://parseable:8000", "admin", "admin"] networks: @@ -66,17 +68,4 @@ services: restart_policy: condition: on-failure delay: 10s - max_attempts: 10 - - # quest-load: - # image: ghcr.io/parseablehq/quest:main - # command: ["load", "http://parseable:8000", "admin", "admin"] - # networks: - # - parseable - # depends_on: - # - quest-smoke - # deploy: - # restart_policy: - # condition: on-failure - # delay: 10s - # max_attempts: 10 + max_attempts: 5 From eae69fdae1e0003c72c0472290148355a91d4aa4 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 12:52:52 +0530 Subject: [PATCH 08/14] temp --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c571d7310..15640a435 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,6 +16,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Clean compose volumes before running + if: always() + run: docker volume rm $(docker volume ls -q ) - name: Start compose run: docker-compose -f "docker-compose.yaml" up --build - name: Stop compose From 29cf78563e318d6340ae9603d445609fd499712c Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 12:58:19 +0530 Subject: [PATCH 09/14] temp --- .github/workflows/build.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 15640a435..c571d7310 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,9 +16,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Clean compose volumes before running - if: always() - run: docker volume rm $(docker volume ls -q ) - name: Start compose run: docker-compose -f "docker-compose.yaml" up --build - name: Stop compose From f87198488ee9ef1cf90a3aabee9b114a55814cbf Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 14:00:10 +0530 Subject: [PATCH 10/14] check with release --- docker-compose.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6d755d786..136b82f8b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,9 +30,10 @@ services: - parseable parseable: - build: - context: . - dockerfile: Dockerfile + image: parseable/parseable:v0.2.1 + # build: + # context: . + # dockerfile: Dockerfile command: ["parseable", "s3-store"] ports: - 8000 From 80b8db2fe5d00efe12fd7cbee59c7a7482260ff6 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 14:26:20 +0530 Subject: [PATCH 11/14] temp --- docker-compose.yaml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 136b82f8b..d518e51e9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,24 +30,21 @@ services: - parseable parseable: - image: parseable/parseable:v0.2.1 - # build: - # context: . - # dockerfile: Dockerfile + build: + context: . + dockerfile: Dockerfile command: ["parseable", "s3-store"] ports: - 8000 environment: - - P_S3_URL=http://minio:9000 - - P_S3_ACCESS_KEY=parseable - - P_S3_SECRET_KEY=supersecret - - P_S3_REGION=us-east-1 - - P_S3_BUCKET=parseable - - P_STAGING_DIR=/tmp/data - - P_USERNAME=admin - - P_PASSWORD=admin - - RUST_BACKTRACE=1 - - RUST_LOG=info + - P_S3_URL="http://minio:9000" + - P_S3_ACCESS_KEY="parseable" + - P_S3_SECRET_KEY="supersecret" + - P_S3_REGION="us-east-1" + - P_S3_BUCKET="parseable" + - P_STAGING_DIR"=/tmp/data" + - P_USERNAME="admin" + - P_PASSWORD="admin" networks: - parseable healthcheck: From a4ab437f92a760250f1604d071b2e7a6bdb819d7 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 15:14:03 +0530 Subject: [PATCH 12/14] temp --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index d518e51e9..c62dfc1b0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -42,7 +42,7 @@ services: - P_S3_SECRET_KEY="supersecret" - P_S3_REGION="us-east-1" - P_S3_BUCKET="parseable" - - P_STAGING_DIR"=/tmp/data" + - P_STAGING_DIR="/tmp/data" - P_USERNAME="admin" - P_PASSWORD="admin" networks: @@ -66,4 +66,4 @@ services: restart_policy: condition: on-failure delay: 10s - max_attempts: 5 + max_attempts: 3 From 6b9226bd80739df59b1655ab53487de687b1ce4c Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 15 Feb 2023 16:32:21 +0530 Subject: [PATCH 13/14] update --- Dockerfile | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21b9c8a62..1b3d14a13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,19 +13,50 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -FROM rust:1.67.0 as builder + +# Compile +FROM rust:1.67-alpine AS compiler WORKDIR /parseable +RUN apk add --no-cache musl-dev + COPY . . -RUN cargo build --release - -FROM gcr.io/distroless/cc:latest +RUN set -eux; \ + apkArch="$(apk --print-arch)"; \ + if [ "$apkArch" = "aarch64" ]; then \ + export JEMALLOC_SYS_WITH_LG_PAGE=16; \ + fi && \ + cargo build --release + +# Run +FROM alpine:3.16 + +RUN apk update --quiet \ + && apk add -q --no-cache tini + +# Create appuser +ENV USER=parseable +ENV UID=10001 + +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + "${USER}" WORKDIR /parseable -COPY --from=builder /lib/x86_64-linux-gnu/liblzma.so* /lib/x86_64-linux-gnu/ -COPY --from=builder /parseable/target/release/parseable /usr/local/bin/parseable +COPY --from=compiler /parseable/target/release/parseable /bin/parseable +RUN chown -R parseable /parseable/ + +USER parseable:parseable + +EXPOSE 8000/tcp -CMD ["/usr/local/bin/parseable"] +ENTRYPOINT ["tini", "--"] +CMD /bin/parseable From 4a8a8a7ec2eceebcd5ced80ef37040f9a35956b7 Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Wed, 15 Feb 2023 20:26:49 +0530 Subject: [PATCH 14/14] Re-export deps with static feature --- Cargo.lock | 2 ++ server/Cargo.toml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 71df22a7c..8690c0d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2785,6 +2785,7 @@ dependencies = [ "aws-smithy-async", "base64 0.21.0", "bytes", + "bzip2", "cargo_toml", "chrono", "chrono-humanize", @@ -2828,6 +2829,7 @@ dependencies = [ "ureq", "vergen", "xxhash-rust", + "xz2", "zip", ] diff --git a/server/Cargo.toml b/server/Cargo.toml index 7d5daae37..ac6082ddc 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -70,6 +70,8 @@ ureq = { version = "2.6", features = ["json"] } hex = "0.4" itertools = "0.10" xxhash-rust = { version = "0.8", features = ["xxh3"] } +xz2 = { version = "*", features=["static"] } +bzip2 = { version = "*", features=["static"] } [build-dependencies] static-files = "0.2"