From 8a18d5e30fb4051b5a096002c2d7b314ca46332b Mon Sep 17 00:00:00 2001 From: hippalus Date: Tue, 8 Apr 2025 14:17:47 +0200 Subject: [PATCH 1/3] Add mandatory kafka env values to parseable helm template --- helm/templates/ingestor-statefulset.yaml | 8 ++++++++ helm/templates/standalone-deployment.yaml | 12 +++++++++++- helm/values.yaml | 6 +++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml index 9773a1242..d5979249f 100644 --- a/helm/templates/ingestor-statefulset.yaml +++ b/helm/templates/ingestor-statefulset.yaml @@ -139,6 +139,14 @@ spec: {{- end }} - name: P_MODE value: "ingest" + {{- if .Values.parseable.kafkaConnector.enabled }} + - name: P_KAFKA_BOOTSTRAP_SERVERS + value: {{ .Values.parseable.kafkaConnector.bootstrapServers | quote }} + - name: P_KAFKA_CONSUMER_TOPICS + value: {{ .Values.parseable.kafkaConnector.consumerTopics | quote }} + - name: P_KAFKA_PARTITION_LISTENER_CONCURRENCY + value: {{ .Values.parseable.kafkaConnector.partitionListenerConcurrency | quote }} + {{- end }} ports: - containerPort: {{ .Values.parseable.highAvailability.ingestor.port }} {{- with .Values.readinessProbe }} diff --git a/helm/templates/standalone-deployment.yaml b/helm/templates/standalone-deployment.yaml index 2c28052ef..1bf5a030b 100644 --- a/helm/templates/standalone-deployment.yaml +++ b/helm/templates/standalone-deployment.yaml @@ -36,7 +36,7 @@ spec: imagePullPolicy: {{ .Values.parseable.image.pullPolicy }} # Uncomment to debug # command: [ "/bin/sh", "-c", "sleep 1000000" ] - args: ["/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}] + args: [ "/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}] env: - name: HOSTNAME valueFrom: @@ -111,6 +111,16 @@ spec: {{- end }} {{- end }} {{- end }} + + {{- if .Values.parseable.kafkaConnector.enabled }} + - name: P_KAFKA_BOOTSTRAP_SERVERS + value: {{ .Values.parseable.kafkaConnector.bootstrapServers | quote }} + - name: P_KAFKA_CONSUMER_TOPICS + value: {{ .Values.parseable.kafkaConnector.consumerTopics | quote }} + - name: P_KAFKA_PARTITION_LISTENER_CONCURRENCY + value: {{ .Values.parseable.kafkaConnector.partitionListenerConcurrency | quote }} + {{- end }} + ports: - containerPort: 8000 {{- with .Values.readinessProbe }} diff --git a/helm/values.yaml b/helm/values.yaml index 1bdd710d1..3be2762ce 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -248,7 +248,11 @@ parseable: attachMetadata: {} scrapeClass: "" bodySizeLimit: {} - + kafkaConnector: + bootstrapServers: "my-kafka.kafka.svc.cluster.local:9092" + enabled: false + consumerTopics: test-log-stream-0,test-log-stream-1 + partitionListenerConcurrency: 2 # Default values for Vector # See Vector helm documentation to learn more: # https://vector.dev/docs/setup/installation/package-managers/helm/ From 7f99d7f6b0fc8e2219c25f445e9724ea4bac5b27 Mon Sep 17 00:00:00 2001 From: hippalus Date: Fri, 11 Apr 2025 01:17:58 +0200 Subject: [PATCH 2/3] #1290 update template --- helm/templates/ingestor-statefulset.yaml | 11 +++-- helm/templates/standalone-deployment.yaml | 15 ++++--- helm/values.yaml | 50 +++++++++++++++++++++-- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml index d5979249f..5143e78f3 100644 --- a/helm/templates/ingestor-statefulset.yaml +++ b/helm/templates/ingestor-statefulset.yaml @@ -140,13 +140,12 @@ spec: - name: P_MODE value: "ingest" {{- if .Values.parseable.kafkaConnector.enabled }} - - name: P_KAFKA_BOOTSTRAP_SERVERS - value: {{ .Values.parseable.kafkaConnector.bootstrapServers | quote }} - - name: P_KAFKA_CONSUMER_TOPICS - value: {{ .Values.parseable.kafkaConnector.consumerTopics | quote }} - - name: P_KAFKA_PARTITION_LISTENER_CONCURRENCY - value: {{ .Values.parseable.kafkaConnector.partitionListenerConcurrency | quote }} + {{- range $key, $value := .Values.parseable.kafkaConnector.env }} + - name: {{ $key }} + value: {{ tpl $value $ | quote }} + {{- end }} {{- end }} + ports: - containerPort: {{ .Values.parseable.highAvailability.ingestor.port }} {{- with .Values.readinessProbe }} diff --git a/helm/templates/standalone-deployment.yaml b/helm/templates/standalone-deployment.yaml index 1bf5a030b..968d237b8 100644 --- a/helm/templates/standalone-deployment.yaml +++ b/helm/templates/standalone-deployment.yaml @@ -70,6 +70,13 @@ spec: value: {{ .Values.parseable.auditLogging.p_password | quote }} {{- end }} + {{- if .Values.parseable.kafkaConnector.enabled }} + {{- range $key, $value := .Values.parseable.kafkaConnector.env }} + - name: {{ $key }} + value: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} + {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }} {{- range $secret := .Values.parseable.s3ModeSecret.secrets }} {{- range $key := $secret.keys }} @@ -112,14 +119,6 @@ spec: {{- end }} {{- end }} - {{- if .Values.parseable.kafkaConnector.enabled }} - - name: P_KAFKA_BOOTSTRAP_SERVERS - value: {{ .Values.parseable.kafkaConnector.bootstrapServers | quote }} - - name: P_KAFKA_CONSUMER_TOPICS - value: {{ .Values.parseable.kafkaConnector.consumerTopics | quote }} - - name: P_KAFKA_PARTITION_LISTENER_CONCURRENCY - value: {{ .Values.parseable.kafkaConnector.partitionListenerConcurrency | quote }} - {{- end }} ports: - containerPort: 8000 diff --git a/helm/values.yaml b/helm/values.yaml index 3be2762ce..b49c5c1f2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -217,7 +217,6 @@ parseable: fsGroupChangePolicy: "Always" nameOverride: "" fullnameOverride: "" - nodeSelector: {} affinity: {} podLabels: app: parseable @@ -249,10 +248,53 @@ parseable: scrapeClass: "" bodySizeLimit: {} kafkaConnector: - bootstrapServers: "my-kafka.kafka.svc.cluster.local:9092" enabled: false - consumerTopics: test-log-stream-0,test-log-stream-1 - partitionListenerConcurrency: 2 + env: + # General Kafka Configuration + P_KAFKA_BOOTSTRAP_SERVERS: "my-kafka.kafka.svc.cluster.local:9092" # Comma-separated list of Kafka bootstrap servers + P_KAFKA_PARTITION_LISTENER_CONCURRENCY: "2" # Number of parallel threads for Kafka partition listeners + P_KAFKA_CONSUMER_TOPICS: "test-log-stream-0,test-log-stream-1" + + # Default values. Change only if necessary. + + # P_KAFKA_CLIENT_ID: "parseable-connect" # Client ID for Kafka connection + # P_CONNECTOR_BAD_DATA_POLICY: "fail" # Default Policy for handling bad data + + # Consumer Configuration + # P_KAFKA_CONSUMER_GROUP_ID: "parseable-connect-cg" + # P_KAFKA_CONSUMER_BUFFER_SIZE: "10000" + # P_KAFKA_CONSUMER_BUFFER_TIMEOUT: "10000ms" + # P_KAFKA_CONSUMER_SESSION_TIMEOUT: "60000" + # P_KAFKA_CONSUMER_HEARTBEAT_INTERVAL: "3000" + # P_KAFKA_CONSUMER_PARTITION_STRATEGY: "roundrobin,range" + # P_KAFKA_CONSUMER_MAX_POLL_INTERVAL: "300000" + # P_KAFKA_CONSUMER_ENABLE_AUTO_OFFSET_STORE: "true" + # P_KAFKA_CONSUMER_AUTO_OFFSET_RESET: "earliest" + # P_KAFKA_CONSUMER_FETCH_MIN_BYTES: "1" + # P_KAFKA_CONSUMER_FETCH_MAX_BYTES: "52428800" + # P_KAFKA_CONSUMER_FETCH_MAX_WAIT: "500" + # P_KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES: "1048576" + # P_KAFKA_CONSUMER_QUEUED_MIN_MESSAGES: "100000" + # P_KAFKA_CONSUMER_QUEUED_MAX_MESSAGES_KBYTES: "65536" + # P_KAFKA_CONSUMER_ENABLE_PARTITION_EOF: "false" + # P_KAFKA_CONSUMER_CHECK_CRCS: "false" + # P_KAFKA_CONSUMER_ISOLATION_LEVEL: "read_committed" + # P_KAFKA_CONSUMER_FETCH_MESSAGE_MAX_BYTES: "1048576" + # P_KAFKA_CONSUMER_STATS_INTERVAL: "10000" + + # Security Configuration - commented out but available if needed + # P_KAFKA_SECURITY_PROTOCOL: "SSL" # Security protocol used for communication (e.g., SSL, SASL_SSL, SASL_PLAINTEXT) + # P_KAFKA_SSL_CA_LOCATION: "/etc/ssl/certs/ca-certificates.crt" # CA certificate file path + # P_KAFKA_SSL_CERTIFICATE_LOCATION: "/etc/ssl/certs/client-cert.pem" # Client certificate file path + # P_KAFKA_SSL_KEY_LOCATION: "/etc/ssl/private/client-key.pem" # Client key file path + # P_KAFKA_SSL_KEY_PASSWORD: "" # Password for the client key file + + # Security Configuration with SASL_SSL example + # P_KAFKA_SECURITY_PROTOCOL: "SASL_SSL" + # P_KAFKA_SSL_CA_LOCATION: "/etc/ssl/certs/ca-certificates.crt" + # P_KAFKA_SASL_MECHANISM: "SCRAM-SHA-512" # Can also be PLAIN, SCRAM-SHA-256, GSSAPI, etc. + # P_KAFKA_SASL_USERNAME: "kafka-user" + # Default values for Vector # See Vector helm documentation to learn more: # https://vector.dev/docs/setup/installation/package-managers/helm/ From 3c7ae4fcfe6236c42f65405d282e1c2a03dbdf18 Mon Sep 17 00:00:00 2001 From: hippalus Date: Fri, 11 Apr 2025 02:21:25 +0200 Subject: [PATCH 3/3] #1290 Add Security Configuration Options to KafkaConnector Env --- helm/values.yaml | 114 ++++++++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index b49c5c1f2..168debbd5 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -9,26 +9,26 @@ parseable: ## Please note that highAvailability is not supported in local mode highAvailability: enabled: false - ingestor: - affinity: {} + ingestor: + affinity: { } # podAntiAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # - labelSelector: # matchLabels: # app: parseable - # component: ingestor - # topologyKey: kubernetes.io/hostname + # component: ingestor + # topologyKey: kubernetes.io/hostname port: 8000 extraLabels: app: parseable - podAnnotations: {} - nodeSelector: {} - tolerations: [] + podAnnotations: { } + nodeSelector: { } + tolerations: [ ] labels: app: parseable component: ingestor count: 3 - env: + env: RUST_LOG: warn ## Use this endpoint to send events to ingestors ## Console (UI) is available on the other service (that points to the query pod) @@ -85,8 +85,8 @@ parseable: storageClass: "" accessMode: ReadWriteOnce size: 5Gi - ## enabling the disk on querier adds the hot-tier. - ## local-store is not supported on hot-tier. + ## enabling the disk on querier adds the hot-tier. + ## local-store is not supported on hot-tier. querier: enabled: false storageClass: "" @@ -155,8 +155,8 @@ parseable: serviceAccount: create: true name: "parseable" - annotations: {} - nodeSelector: {} + annotations: { } + nodeSelector: { } service: type: ClusterIP port: 80 @@ -164,7 +164,7 @@ parseable: httpGet: path: /api/v1/readiness port: 8000 - toleration: [] + toleration: [ ] resources: limits: cpu: 500m @@ -181,9 +181,9 @@ parseable: repository: busybox tag: latest pullPolicy: IfNotPresent - command: [] - args: [] - env: + command: [ ] + args: [ ] + env: RUST_LOG: warn ports: 8000 volumeMounts: @@ -217,36 +217,36 @@ parseable: fsGroupChangePolicy: "Always" nameOverride: "" fullnameOverride: "" - affinity: {} + affinity: { } podLabels: app: parseable component: query - tolerations: [] + tolerations: [ ] ## Use this section to create ServiceMonitor object for ## this Parseable deployment. Read more on ServiceMonitor ## here: https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.ServiceMonitor metrics: serviceMonitor: enabled: false - labels: {} + labels: { } namespace: "" spec: jobLabel: "" - targetLabels: [] - podTargetLabels: [] - endpoints: [] - selector: {} - namespaceSelector: {} + targetLabels: [ ] + podTargetLabels: [ ] + endpoints: [ ] + selector: { } + namespaceSelector: { } sampleLimit: 0 - scrapeProtocols: [] + scrapeProtocols: [ ] targetLimit: 0 labelLimit: 0 labelNameLengthLimit: 0 labelValueLengthLimit: 0 keepDroppedTargets: 0 - attachMetadata: {} + attachMetadata: { } scrapeClass: "" - bodySizeLimit: {} + bodySizeLimit: { } kafkaConnector: enabled: false env: @@ -254,13 +254,10 @@ parseable: P_KAFKA_BOOTSTRAP_SERVERS: "my-kafka.kafka.svc.cluster.local:9092" # Comma-separated list of Kafka bootstrap servers P_KAFKA_PARTITION_LISTENER_CONCURRENCY: "2" # Number of parallel threads for Kafka partition listeners P_KAFKA_CONSUMER_TOPICS: "test-log-stream-0,test-log-stream-1" - - # Default values. Change only if necessary. - # P_KAFKA_CLIENT_ID: "parseable-connect" # Client ID for Kafka connection # P_CONNECTOR_BAD_DATA_POLICY: "fail" # Default Policy for handling bad data - # Consumer Configuration + # Consumer Configurations. These config are Default Parseable KafkaConnector configs. Change only if necessary. # P_KAFKA_CONSUMER_GROUP_ID: "parseable-connect-cg" # P_KAFKA_CONSUMER_BUFFER_SIZE: "10000" # P_KAFKA_CONSUMER_BUFFER_TIMEOUT: "10000ms" @@ -282,20 +279,37 @@ parseable: # P_KAFKA_CONSUMER_FETCH_MESSAGE_MAX_BYTES: "1048576" # P_KAFKA_CONSUMER_STATS_INTERVAL: "10000" - # Security Configuration - commented out but available if needed - # P_KAFKA_SECURITY_PROTOCOL: "SSL" # Security protocol used for communication (e.g., SSL, SASL_SSL, SASL_PLAINTEXT) - # P_KAFKA_SSL_CA_LOCATION: "/etc/ssl/certs/ca-certificates.crt" # CA certificate file path - # P_KAFKA_SSL_CERTIFICATE_LOCATION: "/etc/ssl/certs/client-cert.pem" # Client certificate file path - # P_KAFKA_SSL_KEY_LOCATION: "/etc/ssl/private/client-key.pem" # Client key file path - # P_KAFKA_SSL_KEY_PASSWORD: "" # Password for the client key file + # Security Configuration Options - By Default PLAINTEXT + + # Option 1: SSL Encryption Only + # P_KAFKA_SECURITY_PROTOCOL: "SSL" + # P_KAFKA_SSL_CA_LOCATION: "/etc/ssl/certs/ca-certificates.crt" + # P_KAFKA_SSL_CERTIFICATE_LOCATION: "/etc/ssl/certs/client-cert.pem" + # P_KAFKA_SSL_KEY_LOCATION: "/etc/ssl/private/client-key.pem" + # P_KAFKA_SSL_KEY_PASSWORD: "my-key-password" # Optional: only if key is password protected - # Security Configuration with SASL_SSL example + # Option 2: SASL Authentication with SSL Encryption # P_KAFKA_SECURITY_PROTOCOL: "SASL_SSL" # P_KAFKA_SSL_CA_LOCATION: "/etc/ssl/certs/ca-certificates.crt" - # P_KAFKA_SASL_MECHANISM: "SCRAM-SHA-512" # Can also be PLAIN, SCRAM-SHA-256, GSSAPI, etc. + # P_KAFKA_SASL_MECHANISM: "SCRAM-SHA-512" # Can also be PLAIN, SCRAM-SHA-256, or GSSAPI + # P_KAFKA_SASL_USERNAME: "kafka-user" + # P_KAFKA_SASL_PASSWORD: "kafka-password" + + # Option 3: SASL Authentication without Encryption + # P_KAFKA_SECURITY_PROTOCOL: "SASL_PLAINTEXT" + # P_KAFKA_SASL_MECHANISM: "PLAIN" # Can also be SCRAM-SHA-256, SCRAM-SHA-512, or GSSAPI # P_KAFKA_SASL_USERNAME: "kafka-user" + # P_KAFKA_SASL_PASSWORD: "kafka-password" + + # Option 4: OAuth Bearer Token Authentication (Not supported yet) + # P_KAFKA_SECURITY_PROTOCOL: "SASL_SSL" + # P_KAFKA_SASL_MECHANISM: "OAUTHBEARER" + # P_KAFKA_OAUTH_TOKEN_ENDPOINT: "https://oauth.example.com/token" + # P_KAFKA_OAUTH_CLIENT_ID: "kafka-client" + # P_KAFKA_OAUTH_CLIENT_SECRET: "client-secret" + # P_KAFKA_OAUTH_SCOPE: "kafka-access" # Optional: only if required by OAuth provider -# Default values for Vector +## Default values for Vector # See Vector helm documentation to learn more: # https://vector.dev/docs/setup/installation/package-managers/helm/ vector: @@ -305,7 +319,7 @@ vector: image: repository: timberio/vector pullPolicy: IfNotPresent - pullSecrets: [] + pullSecrets: [ ] tag: "" sha: "" replicas: 1 @@ -318,7 +332,7 @@ vector: create: true serviceAccount: create: true - annotations: {} + annotations: { } name: automountToken: true podLabels: @@ -330,13 +344,13 @@ vector: service: enabled: true type: "ClusterIP" - annotations: {} - topologyKeys: [] - ports: [] + annotations: { } + topologyKeys: [ ] + ports: [ ] externalTrafficPolicy: "" loadBalancerIP: "" ipFamilyPolicy: "" - ipFamilies: [] + ipFamilies: [ ] serviceHeadless: enabled: true dnsPolicy: ClusterFirst @@ -399,7 +413,7 @@ fluent-bit: tag: latest serviceAccount: create: true - annotations: {} + annotations: { } name: rbac: create: true @@ -410,8 +424,8 @@ fluent-bit: type: ClusterIP port: 2020 loadBalancerClass: - loadBalancerSourceRanges: [] - labels: {} + loadBalancerSourceRanges: [ ] + labels: { } livenessProbe: httpGet: path: / @@ -503,7 +517,7 @@ fluent-bit: Password {{ .Values.serverPassword }} Stream k8s-events - upstream: {} + upstream: { } customParsers: | [PARSER]