diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 738128b06..1eef5fe6c 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -11,7 +11,7 @@ on: jobs: docker-compose-test: - name: Quest Smoke and Load Tests + name: Quest Smoke and Load Tests for Standalone deployments runs-on: ubuntu-latest steps: - name: Checkout @@ -21,3 +21,15 @@ jobs: - name: Stop compose if: always() run: docker-compose -f docker-compose-test.yaml down + + docker-compose-distributed-test: + name: Quest Smoke and Load Tests for Distributed deployments + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Start compose + run: docker-compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest + - name: Stop compose + if: always() + run: docker-compose -f docker-compose-distributed-test.yaml down diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml new file mode 100644 index 000000000..e40912e0e --- /dev/null +++ b/docker-compose-distributed-test.yaml @@ -0,0 +1,256 @@ +version: "3.7" +networks: + parseable-internal: +services: + # minio + minio: + image: minio/minio:RELEASE.2023-02-10T18-48-39Z + entrypoint: + - sh + - -euc + - | + mkdir -p /tmp/minio/parseable && \ + minio server /tmp/minio + environment: + - MINIO_ROOT_USER=parseable + - MINIO_ROOT_PASSWORD=supersecret + - MINIO_UPDATE=off + ports: + - 9000:9000 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 15s + timeout: 20s + retries: 5 + networks: + - parseable-internal + # query server + parseable-query: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000:8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=query + networks: + - parseable-internal + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server one + parseable-ingest-one: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000 + networks: + - parseable-internal + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server two + parseable-ingest-two: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server three + parseable-ingest-three: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - P_INGESTOR_ENDPOINT=parseable-ingest-three:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # ingest server four + parseable-ingest-four: + build: + context: . + dockerfile: Dockerfile + command: ["parseable", "s3-store"] + ports: + - 8000 + environment: + - P_S3_URL=http://minio:9000 + - P_S3_ACCESS_KEY=parseable + - P_S3_SECRET_KEY=supersecret + - P_S3_REGION=us-east-1 + - P_S3_BUCKET=parseable + - P_STAGING_DIR=/tmp/data + - P_USERNAME=parseableadmin + - P_PASSWORD=parseableadmin + - P_CHECK_UPDATE=false + - P_PARQUET_COMPRESSION_ALGO=snappy + - P_MODE=ingest + - P_INGESTOR_ENDPOINT=parseable-ingest-four:8000 + networks: + - parseable-internal + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] + interval: 15s + timeout: 20s + retries: 5 + depends_on: + - parseable-query + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + # penguin lb + penguin: + platform: linux/amd64 + image: parseable/penguin:edge + command: ["./target/release/penguin"] + environment: + - P_ADDR_ONE=parseable-ingest-one:8000 + - P_ADDR_TWO=parseable-ingest-two:8000 + - P_ADDR_THREE=parseable-ingest-three:8000 + - P_ADDR_FOUR=parseable-ingest-four:8000 + ports: + - 6188:6188 + depends_on: + - parseable-query + - minio + - parseable-ingest-one + - parseable-ingest-two + - parseable-ingest-three + - parseable-ingest-four + networks: + - parseable-internal + quest: + platform: linux/amd64 + image: ghcr.io/parseablehq/quest:main + command: + [ + "load", + "http://parseable-query:8000", + "parseableadmin", + "parseableadmin", + "20", + "10", + "5m", + "minio:9000", + "parseable", + "supersecret", + "parseable", + "http://penguin:6188", + "parseableadmin", + "parseableadmin", + ] + networks: + - parseable-internal + depends_on: + - parseable-query + - penguin + - minio + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3