docker-compose added for distributed

.github/workflows/integration-test.yaml

@@ -11,7 +11,7 @@ on:
 jobs:
 
   docker-compose-test:
-    name: Quest Smoke and Load Tests
+    name: Quest Smoke and Load Tests for Standalone deployments
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -21,3 +21,15 @@ jobs:
       - name: Stop compose
         if: always()
         run: docker-compose -f docker-compose-test.yaml down
+
+  docker-compose-distributed-test:
+    name: Quest Smoke and Load Tests for Distributed deployments
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Start compose
+        run: docker-compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker-compose -f docker-compose-distributed-test.yaml down

docker-compose-distributed-test.yaml

@@ -0,0 +1,256 @@
+version: "3.7"
+networks:
+  parseable-internal:
+services:
+  # minio
+  minio:
+    image: minio/minio:RELEASE.2023-02-10T18-48-39Z
+    entrypoint:
+      - sh
+      - -euc
+      - |
+        mkdir -p /tmp/minio/parseable && \
+        minio server /tmp/minio
+    environment:
+      - MINIO_ROOT_USER=parseable
+      - MINIO_ROOT_PASSWORD=supersecret
+      - MINIO_UPDATE=off
+    ports:
+      - 9000:9000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    networks:
+      - parseable-internal
+  # query server
+  parseable-query:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["parseable", "s3-store"]
+    ports:
+      - 8000:8000
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=query
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+  # ingest server one
+  parseable-ingest-one:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["parseable", "s3-store"]
+    ports:
+      - 8000
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+  # ingest server two
+  parseable-ingest-two:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["parseable", "s3-store"]
+    ports:
+      - 8000
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+  # ingest server three
+  parseable-ingest-three:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["parseable", "s3-store"]
+    ports:
+      - 8000
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-three:8000
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+  # ingest server four
+  parseable-ingest-four:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["parseable", "s3-store"]
+    ports:
+      - 8000
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-four:8000
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+  # penguin lb
+  penguin:
+    platform: linux/amd64
+    image: parseable/penguin:edge
+    command: ["./target/release/penguin"]
+    environment:
+      - P_ADDR_ONE=parseable-ingest-one:8000
+      - P_ADDR_TWO=parseable-ingest-two:8000
+      - P_ADDR_THREE=parseable-ingest-three:8000
+      - P_ADDR_FOUR=parseable-ingest-four:8000
+    ports:
+      - 6188:6188
+    depends_on:
+      - parseable-query
+      - minio
+      - parseable-ingest-one
+      - parseable-ingest-two
+      - parseable-ingest-three
+      - parseable-ingest-four
+    networks:
+      - parseable-internal
+  quest:
+    platform: linux/amd64
+    image: ghcr.io/parseablehq/quest:main
+    command:
+      [
+        "load",
+        "http://parseable-query:8000",
+        "parseableadmin",
+        "parseableadmin",
+        "20",
+        "10",
+        "5m",
+        "minio:9000",
+        "parseable",
+        "supersecret",
+        "parseable",
+        "http://penguin:6188",
+        "parseableadmin",
+        "parseableadmin",
+      ]
+    networks:
+      - parseable-internal
+    depends_on:
+      - parseable-query
+      - penguin
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3