From 58fc186cc3aa10ef9e108919671a353c656c81af Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Sat, 18 Feb 2023 19:51:25 +0530 Subject: [PATCH 01/11] Force path style --- server/src/storage/s3.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/storage/s3.rs b/server/src/storage/s3.rs index 10e77a454..c903c1bab 100644 --- a/server/src/storage/s3.rs +++ b/server/src/storage/s3.rs @@ -137,6 +137,7 @@ impl ObjectStorageProvider for S3Config { let config = aws_sdk_s3::Config::builder() .region(region) .endpoint_url(uri) + .force_path_style(true) .credentials_provider(creds) .retry_config(RetryConfig::standard().with_max_attempts(5)) .sleep_impl(default_async_sleep().expect("sleep impl is provided for tokio rt")) From 31e30ce45c9a20fb13ade9db08abe4df33150d2f Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 12:59:29 +0530 Subject: [PATCH 02/11] Manually add .so files needed for runtime in container --- Dockerfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4576ca06a..fe6d3befa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # along with this program. If not, see . -FROM rust:slim-bullseye as builder +FROM rust:slim-bullseye as build LABEL org.opencontainers.image.title="Parseable" LABEL maintainer="Parseable Team " @@ -26,9 +26,17 @@ WORKDIR /parseable COPY . . RUN cargo build --release +RUN mkdir -p /app/lib && \ + cp -LR $(ldd /parseable/target/release/router | grep "=>" | cut -d ' ' -f 3) /app/lib + FROM gcr.io/distroless/cc-debian11:nonroot WORKDIR /parseable -COPY --from=builder /parseable/target/release/parseable /usr/bin/parseable +COPY --from=build /app/lib /app/lib +COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 +COPY --from=build /app/target/release/router /app/router +COPY --from=build /parseable/target/release/parseable /usr/bin/parseable + +ENV LD_LIBRARY_PATH=/app/lib CMD ["parseable"] From d01984b6b26baf717973bbfe24ffbf490570e930 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 13:03:17 +0530 Subject: [PATCH 03/11] temp --- docker-compose.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1ee47c176..208bdb1c7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,7 +5,6 @@ networks: driver: bridge services: - minio: image: minio/minio:RELEASE.2023-02-10T18-48-39Z entrypoint: @@ -31,11 +30,11 @@ services: - parseable-internal parseable: - image: parseablehq/parseable:v0.2.1 - # build: - # context: . - # dockerfile: Dockerfile - # command: ["parseable", "s3-store"] + # image: parseable/parseable:v0.2.2-stage + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] ports: - 8000 environment: From e37984590772759b6542a71eb3be538a6321d727 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 13:09:36 +0530 Subject: [PATCH 04/11] temp --- .github/workflows/build-push.yaml | 79 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 8aed27c58..90de5748e 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -1,8 +1,7 @@ name: Parseable Docker build and push on: - push: - branches: ['main'] + pull_request: paths-ignore: - 'docs/**' - 'helm/**' @@ -15,17 +14,17 @@ env: jobs: - # docker-compose-test: - # name: Docker Compose integration tests - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v3 - # - name: Start compose - # run: docker-compose -f "docker-compose.yaml" up --build - # - name: Stop compose - # if: always() - # run: docker-compose -f "docker-compose.yaml" down + docker-compose-test: + name: Docker Compose integration tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Start compose + run: docker-compose -f "docker-compose.yaml" up --build + - name: Stop compose + if: always() + run: docker-compose -f "docker-compose.yaml" down # docker-build: # name: Docker build @@ -36,30 +35,30 @@ jobs: # - name: Build Docker image # run: docker build . - build-and-push-image: - name: Build and push Docker image to GitHub Container Registry - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # build-and-push-image: + # name: Build and push Docker image to GitHub Container Registry + # runs-on: ubuntu-latest + # permissions: + # contents: read + # packages: write + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + # - name: Log in to the Container registry + # uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + # with: + # registry: ${{ env.REGISTRY }} + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + # - name: Extract metadata (tags, labels) for Docker + # id: meta + # uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + # with: + # images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # - name: Build and push Docker image + # uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + # with: + # context: . + # push: true + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} From 63dbbaa0d1bdc3420cdb12eab44327f185064a20 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 13:27:02 +0530 Subject: [PATCH 05/11] temp --- .github/workflows/build-push.yaml | 2 +- .github/workflows/ci.yaml | 2 +- Dockerfile | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 90de5748e..ca3c98333 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -1,4 +1,4 @@ -name: Parseable Docker build and push +name: Integration tests on: pull_request: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 73c2fa254..4f2578cf5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Parseable CI Workflow +name: Linter on: pull_request: diff --git a/Dockerfile b/Dockerfile index fe6d3befa..7eac5cbea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ COPY . . RUN cargo build --release RUN mkdir -p /app/lib && \ - cp -LR $(ldd /parseable/target/release/router | grep "=>" | cut -d ' ' -f 3) /app/lib + cp -LR $(ldd /parseable/target/release/parseable | grep "=>" | cut -d ' ' -f 3) /app/lib FROM gcr.io/distroless/cc-debian11:nonroot @@ -35,7 +35,6 @@ WORKDIR /parseable COPY --from=build /app/lib /app/lib COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 -COPY --from=build /app/target/release/router /app/router COPY --from=build /parseable/target/release/parseable /usr/bin/parseable ENV LD_LIBRARY_PATH=/app/lib From c8e6bda9290015ad5b41705b36323c766b7886a5 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 14:05:01 +0530 Subject: [PATCH 06/11] temp --- Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7eac5cbea..9ff101dfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # along with this program. If not, see . -FROM rust:slim-bullseye as build +FROM rust:1.67.0-bullseye as build LABEL org.opencontainers.image.title="Parseable" LABEL maintainer="Parseable Team " @@ -26,16 +26,19 @@ WORKDIR /parseable COPY . . RUN cargo build --release -RUN mkdir -p /app/lib && \ - cp -LR $(ldd /parseable/target/release/parseable | grep "=>" | cut -d ' ' -f 3) /app/lib +RUN mkdir -p /app/lib +RUN cp -LR $(ldd /parseable/target/release/parseable | grep "=>" | cut -d ' ' -f 3) /app/lib +RUN ls -la /app/lib FROM gcr.io/distroless/cc-debian11:nonroot +ENV LD_LIBRARY_PATH=/app/lib WORKDIR /parseable COPY --from=build /app/lib /app/lib -COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 +COPY --from=build /lib64/ld-linux-x86-64.so.2 /app/lib/ld-linux-x86-64.so.2 COPY --from=build /parseable/target/release/parseable /usr/bin/parseable -ENV LD_LIBRARY_PATH=/app/lib +RUN ls -la /app/lib + CMD ["parseable"] From c6b072022c6e5d3db93956488368686f7fe08f66 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 14:30:34 +0530 Subject: [PATCH 07/11] tmep --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ff101dfd..efb71eff2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,4 @@ COPY --from=build /app/lib /app/lib COPY --from=build /lib64/ld-linux-x86-64.so.2 /app/lib/ld-linux-x86-64.so.2 COPY --from=build /parseable/target/release/parseable /usr/bin/parseable -RUN ls -la /app/lib - CMD ["parseable"] From 500f4268cc7cf67069934ad74aeaeb916d7cc769 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 14:31:34 +0530 Subject: [PATCH 08/11] temp --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index efb71eff2..6f8a441e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - +# build stage FROM rust:1.67.0-bullseye as build LABEL org.opencontainers.image.title="Parseable" @@ -28,9 +28,10 @@ RUN cargo build --release RUN mkdir -p /app/lib RUN cp -LR $(ldd /parseable/target/release/parseable | grep "=>" | cut -d ' ' -f 3) /app/lib -RUN ls -la /app/lib +# run stage FROM gcr.io/distroless/cc-debian11:nonroot + ENV LD_LIBRARY_PATH=/app/lib WORKDIR /parseable From 41f541a6c23db73f68aeb1895f922110947781bc Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 19:57:29 +0530 Subject: [PATCH 09/11] temp --- Dockerfile | 15 +++------------ docker-compose.yaml | 3 +-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6f8a441e9..0df5b133f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # along with this program. If not, see . # build stage -FROM rust:1.67.0-bullseye as build +FROM rust:1.67.0-bullseye as builder LABEL org.opencontainers.image.title="Parseable" LABEL maintainer="Parseable Team " @@ -22,22 +22,13 @@ LABEL org.opencontainers.image.vendor="Cloudnatively Pvt Ltd" LABEL org.opencontainers.image.licenses="AGPL-3.0" WORKDIR /parseable - COPY . . RUN cargo build --release -RUN mkdir -p /app/lib -RUN cp -LR $(ldd /parseable/target/release/parseable | grep "=>" | cut -d ' ' -f 3) /app/lib - -# run stage +# final stage FROM gcr.io/distroless/cc-debian11:nonroot -ENV LD_LIBRARY_PATH=/app/lib - WORKDIR /parseable - -COPY --from=build /app/lib /app/lib -COPY --from=build /lib64/ld-linux-x86-64.so.2 /app/lib/ld-linux-x86-64.so.2 -COPY --from=build /parseable/target/release/parseable /usr/bin/parseable +COPY --from=builder /parseable/target/release/parseable /usr/bin/parseable CMD ["parseable"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 208bdb1c7..b81cdd2f1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,7 +30,6 @@ services: - parseable-internal parseable: - # image: parseable/parseable:v0.2.2-stage build: context: . dockerfile: Dockerfile @@ -67,4 +66,4 @@ services: restart_policy: condition: on-failure delay: 10s - max_attempts: 3 + max_attempts: 1 From 678936a34ed80be0aee6cc60988ca0fbda410c12 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 20:15:17 +0530 Subject: [PATCH 10/11] temp --- docker-compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b81cdd2f1..09c487ce3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -43,8 +43,8 @@ services: - P_S3_REGION=us-east-1 - P_S3_BUCKET=parseable - P_STAGING_DIR=/tmp/data - - P_USERNAME=admin - - P_PASSWORD=admin + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin networks: - parseable-internal healthcheck: @@ -57,7 +57,7 @@ services: quest: image: ghcr.io/parseablehq/quest:main - command: ["smoke", "http://parseable:8000", "admin", "admin"] + command: ["smoke", "http://parseable:8000", "parseableadmin", "parseableadmin"] networks: - parseable-internal depends_on: @@ -66,4 +66,4 @@ services: restart_policy: condition: on-failure delay: 10s - max_attempts: 1 + max_attempts: 3 From cdb30bda518d192f8c7bb6e9e55637c03a2488ac Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 18 Feb 2023 20:20:08 +0530 Subject: [PATCH 11/11] temp --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 09c487ce3..9ca46c106 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -45,6 +45,7 @@ services: - P_STAGING_DIR=/tmp/data - P_USERNAME=parseableadmin - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false networks: - parseable-internal healthcheck: