From 3c4261d36f7fd75c50671e6bcc5d9cbebed5ff87 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Tue, 23 Apr 2024 11:46:10 +0530 Subject: [PATCH 1/5] docker-compose added for distributed workflow updated to test standalone and distributed both fixes : #644 --- .github/workflows/integration-test.yaml | 14 +- docker-compose-distributed-test.yaml | 267 ++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 docker-compose-distributed-test.yaml diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 738128b06..1eef5fe6c 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -11,7 +11,7 @@ on: jobs: docker-compose-test: - name: Quest Smoke and Load Tests + name: Quest Smoke and Load Tests for Standalone deployments runs-on: ubuntu-latest steps: - name: Checkout @@ -21,3 +21,15 @@ jobs: - name: Stop compose if: always() run: docker-compose -f docker-compose-test.yaml down + + docker-compose-distributed-test: + name: Quest Smoke and Load Tests for Distributed deployments + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Start compose + run: docker-compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest + - name: Stop compose + if: always() + run: docker-compose -f docker-compose-distributed-test.yaml down diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml new file mode 100644 index 000000000..d26762693 --- /dev/null +++ b/docker-compose-distributed-test.yaml @@ -0,0 +1,267 @@ +version: "3.7" +networks: + parseable-internal: +services: + # minio + minio: + image: minio/minio:RELEASE.2023-02-10T18-48-39Z + entrypoint: + - sh + - -euc + - | + mkdir -p /tmp/minio/parseable && \ + minio server /tmp/minio + environment: + - MINIO_ROOT_USER=parseable + - MINIO_ROOT_PASSWORD=supersecret + - MINIO_UPDATE=off + ports: + - 9000 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 15s + timeout: 20s + retries: 5 + networks: + - parseable-internal + # query server + parseable-query: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=query + - RUST_LOG=DEBUG + networks: + - parseable-internal + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server one + parseable-ingest-one: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000 + - RUST_LOG=DEBUG + networks: + - parseable-internal + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server two + parseable-ingest-two: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - RUST_LOG=DEBUG + - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server three + parseable-ingest-three: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - RUST_LOG=DEBUG + - P_INGESTOR_ENDPOINT=parseable-ingest-three:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server four + parseable-ingest-four: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - RUST_LOG=DEBUG + - P_INGESTOR_ENDPOINT=parseable-ingest-four:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # penguin lb + penguin: + image: ghcr.io/eshanatnight/penguin:multi + command: ["penguin"] + environment: + - P_ADDR_ONE=parseable-ingest-one:8000 + - P_ADDR_TWO=parseable-ingest-two:8000 + - P_ADDR_THREE=parseable-ingest-three:8000 + - P_ADDR_FOUR=parseable-ingest-four:8000 + - RUST_LOG=DEBUG + ports: + - 6188 + depends_on: + - parseable-query + - minio + - parseable-ingest-one + - parseable-ingest-two + - parseable-ingest-three + - parseable-ingest-four + # ignore this for now + # healthcheck: + # test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + # interval: 15s + # timeout: 20s + # retries: 5 + networks: + - parseable-internal + quest: + platform: linux/amd64 + image: ghcr.io/eshanatnight/quest:main + command: + [ + "load", + "http://parseable-query:8000", + "parseableadmin", + "parseableadmin", + "20", + "10", + "5m", + "minio:9000", + "parseable", + "supersecret", + "parseable", + "http://penguin:6188", + "parseableadmin", + "parseableadmin", + ] + networks: + - parseable-internal + depends_on: + - parseable-query + - penguin + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 From 23e3687c61a09b3bd2ce3711696459ffbe428a38 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Tue, 23 Apr 2024 12:00:09 +0530 Subject: [PATCH 2/5] updated ports, quest build in wqdocker-compose-distributed-test --- docker-compose-distributed-test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index d26762693..f051a6cb0 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -16,7 +16,7 @@ services: - MINIO_ROOT_PASSWORD=supersecret - MINIO_UPDATE=off ports: - - 9000 + - 9000:9000 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 15s @@ -31,7 +31,7 @@ services: dockerfile: Dockerfile command: ["parseable", "s3-store"] ports: - - 8000 + - 8000:8000 environment: - P_S3_URL=http://minio:9000 - P_S3_ACCESS_KEY=parseable @@ -218,7 +218,7 @@ services: - P_ADDR_FOUR=parseable-ingest-four:8000 - RUST_LOG=DEBUG ports: - - 6188 + - 6188:6188 depends_on: - parseable-query - minio @@ -236,7 +236,7 @@ services: - parseable-internal quest: platform: linux/amd64 - image: ghcr.io/eshanatnight/quest:main + image: ghcr.io/parseablehq/quest:main command: [ "load", From 234cfa32cd878142ff41c640826fd7d4c7f12c3f Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Tue, 23 Apr 2024 13:29:38 +0530 Subject: [PATCH 3/5] updated image and platform for penguin --- docker-compose-distributed-test.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index f051a6cb0..d92c58df7 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -209,7 +209,8 @@ services: max_attempts: 3 # penguin lb penguin: - image: ghcr.io/eshanatnight/penguin:multi + platform: linux/amd64 + image: parseable/parseable:edge command: ["penguin"] environment: - P_ADDR_ONE=parseable-ingest-one:8000 @@ -226,12 +227,6 @@ services: - parseable-ingest-two - parseable-ingest-three - parseable-ingest-four - # ignore this for now - # healthcheck: - # test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] - # interval: 15s - # timeout: 20s - # retries: 5 networks: - parseable-internal quest: From e044f7bdc9f5c21337257c34251406846bd0f7cb Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Tue, 23 Apr 2024 15:19:48 +0530 Subject: [PATCH 4/5] updated penguin image name and command --- docker-compose-distributed-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index d92c58df7..f3a366164 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -210,8 +210,8 @@ services: # penguin lb penguin: platform: linux/amd64 - image: parseable/parseable:edge - command: ["penguin"] + image: parseable/penguin:edge + command: ["./target/release/penguin"] environment: - P_ADDR_ONE=parseable-ingest-one:8000 - P_ADDR_TWO=parseable-ingest-two:8000 From 7ac6b2675542c0b90b1392d5c2e00830872cc041 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Tue, 23 Apr 2024 15:42:00 +0530 Subject: [PATCH 5/5] removed RUST_LOG=DEBUG --- docker-compose-distributed-test.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index f3a366164..e40912e0e 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -44,7 +44,6 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=query - - RUST_LOG=DEBUG networks: - parseable-internal healthcheck: @@ -80,7 +79,6 @@ services: - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=ingest - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000 - - RUST_LOG=DEBUG networks: - parseable-internal healthcheck: @@ -116,7 +114,6 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=ingest - - RUST_LOG=DEBUG - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000 networks: - parseable-internal @@ -153,7 +150,6 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=ingest - - RUST_LOG=DEBUG - P_INGESTOR_ENDPOINT=parseable-ingest-three:8000 networks: - parseable-internal @@ -190,7 +186,6 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=ingest - - RUST_LOG=DEBUG - P_INGESTOR_ENDPOINT=parseable-ingest-four:8000 networks: - parseable-internal @@ -217,7 +212,6 @@ services: - P_ADDR_TWO=parseable-ingest-two:8000 - P_ADDR_THREE=parseable-ingest-three:8000 - P_ADDR_FOUR=parseable-ingest-four:8000 - - RUST_LOG=DEBUG ports: - 6188:6188 depends_on: