From 487a2d52a24d492b8af92a41d54ef9071bac311e Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Wed, 5 Feb 2025 16:05:22 +0530 Subject: [PATCH 1/2] fix: kafka build on aarch64 --- .github/workflows/build-push-edge-kafka.yaml | 17 +++++++++++++++-- Dockerfile.kafka | 11 ++++++----- docker-compose-distributed-test-with-kafka.yaml | 2 ++ docker-compose-test-with-kafka.yaml | 2 ++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-push-edge-kafka.yaml b/.github/workflows/build-push-edge-kafka.yaml index afa866d0b..75b857d32 100644 --- a/.github/workflows/build-push-edge-kafka.yaml +++ b/.github/workflows/build-push-edge-kafka.yaml @@ -37,11 +37,24 @@ jobs: with: images: parseable/parseable - - name: Build and push + - name: Build and push x86_64 uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc with: context: . file: ./Dockerfile.kafka push: true tags: parseable/parseable:edge-kafka - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 + build-args: | + LIB_DIR=x86_64-linux-gnu + + - name: Build and push aarch64 + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + file: ./Dockerfile.kafka + push: true + tags: parseable/parseable:edge-kafka + platforms: =linux/arm64 + build-args: | + LIB_DIR=aarch64-linux-gnu diff --git a/Dockerfile.kafka b/Dockerfile.kafka index 0c45fe7d6..108f74c75 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -49,16 +49,17 @@ RUN cargo build --release --features kafka # final stage FROM gcr.io/distroless/cc-debian12:latest +# Copy only the libraries that binary needs since kafka is statically linked +ARG LIB_DIR +COPY --from=builder /usr/lib/${LIB_DIR}/libsasl2.so.2 /usr/lib/${LIB_DIR}/ +COPY --from=builder /usr/lib/${LIB_DIR}/libssl.so.3 /usr/lib/${LIB_DIR}/ +COPY --from=builder /usr/lib/${LIB_DIR}/libcrypto.so.3 /usr/lib/${LIB_DIR}/ + WORKDIR /parseable # Copy the Parseable binary from builder COPY --from=builder /parseable/target/release/parseable /usr/bin/parseable -# Copy only the libraries that binary needs since kafka is statically linked -COPY --from=builder /usr/lib/x86_64-linux-gnu/libsasl2.so.2 /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.so.3 /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.so.3 /usr/lib/x86_64-linux-gnu/ - # Copy CA certificates COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml index d65f7390b..ca72378ed 100644 --- a/docker-compose-distributed-test-with-kafka.yaml +++ b/docker-compose-distributed-test-with-kafka.yaml @@ -94,6 +94,8 @@ services: build: context: . dockerfile: Dockerfile.kafka + args: + - LIB_DIR=x86_64-linux-gnu platform: linux/amd64 command: [ "parseable", "s3-store", ] expose: diff --git a/docker-compose-test-with-kafka.yaml b/docker-compose-test-with-kafka.yaml index 0e747c097..5923f6565 100644 --- a/docker-compose-test-with-kafka.yaml +++ b/docker-compose-test-with-kafka.yaml @@ -28,6 +28,8 @@ services: build: context: . dockerfile: Dockerfile.kafka + args: + - LIB_DIR=x86_64-linux-gnu platform: linux/amd64 command: [ "parseable", "s3-store", ] ports: From b5f00c7ee165d12bc1ed307f2b022e43fe34386e Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Wed, 5 Feb 2025 20:25:35 +0530 Subject: [PATCH 2/2] second ingestor node --- docker-compose-distributed-test-with-kafka.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml index ca72378ed..d9f35581b 100644 --- a/docker-compose-distributed-test-with-kafka.yaml +++ b/docker-compose-distributed-test-with-kafka.yaml @@ -142,6 +142,8 @@ services: build: context: . dockerfile: Dockerfile.kafka + args: + - LIB_DIR=x86_64-linux-gnu platform: linux/amd64 command: [ "parseable", "s3-store", ] expose: