From d3634c6bec15f033844624a7b3b74245aa0ec32f Mon Sep 17 00:00:00 2001 From: AdheipSingh Date: Mon, 23 Dec 2024 16:02:40 +0530 Subject: [PATCH 1/3] default size to 5Gi --- helm/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 1f8c2f5cf..56d8d26a7 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -55,22 +55,22 @@ parseable: enabled: false storageClass: "" accessMode: ReadWriteOnce - size: 1Gi + size: 5Gi ingestor: enabled: false storageClass: "" accessMode: ReadWriteOnce - size: 1Gi + size: 5Gi data: enabled: false storageClass: "" accessMode: ReadWriteOnce - size: 1Gi + size: 5Gi querier: enabled: false storageClass: "" accessMode: ReadWriteOnce - size: 1Gi + size: 5Gi ## comment out the secrets depending upon deployment option ## localModeSecret if store is set to local ## blobModeSecret if store is set to blob-store From 2e09dbe11a8721e6e87318da750978038949ac4e Mon Sep 17 00:00:00 2001 From: AdheipSingh Date: Wed, 1 Jan 2025 16:06:39 +0530 Subject: [PATCH 2/3] nginx error logs parser --- helm/values.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/helm/values.yaml b/helm/values.yaml index 56d8d26a7..131d77a9f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -377,6 +377,14 @@ fluent-bit: Mem_Buf_Limit 5MB Skip_Long_Lines On + # [INPUT] + # Name tail + # Path /var/log/containers/{NGINX_DEPLOYMENT_NAME}-*.log + # Parser nginx_error + # Tag kube.* + # Mem_Buf_Limit 5MB + # Skip_Long_Lines On + [INPUT] name kubernetes_events tag k8s_events @@ -427,6 +435,14 @@ fluent-bit: Time_Keep Off Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%L + + [PARSER] + Name nginx_error + Format regex + Regex ^(?\S+Z) stderr F (?\S+ \S+) \[(?\S+)\] (?\d+#\d+): \*(?\d+) (?.*?) client: (?\S+), server: (?\S+), request: "(?\S+) (?\S+) HTTP/\S+", upstream: "(?[^"]+)", host: "(?\S+)"$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%LZ + # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname" volumeMounts: - name: config From cc8207126592a6a93457a52fc3f0872176126a69 Mon Sep 17 00:00:00 2001 From: AdheipSingh Date: Fri, 3 Jan 2025 14:44:11 +0530 Subject: [PATCH 3/3] hot tier , nginx parser --- helm/templates/querier-statefulset.yaml | 4 +++ helm/values.yaml | 35 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/helm/templates/querier-statefulset.yaml b/helm/templates/querier-statefulset.yaml index 2694eb75f..4a9e7ad8c 100644 --- a/helm/templates/querier-statefulset.yaml +++ b/helm/templates/querier-statefulset.yaml @@ -52,6 +52,10 @@ spec: fieldPath: metadata.name - name: P_MODE value: "query" + {{- if .Values.parseable.persistence.querier.enabled }} + - name: P_HOT_TIER_DIR + value: "/parseable/hot-tier" + {{- end }} {{- range $key, $value := .Values.parseable.env }} - name: {{ $key }} value: {{ tpl $value $ | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 131d77a9f..0afc2d1a7 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -5,7 +5,7 @@ parseable: pullPolicy: Always ## object store can be local-store, s3-store, blob-store or gcs-store. store: local-store - ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors) + ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors + hot tier) ## Please note that highAvailability is not supported in local mode highAvailability: enabled: false @@ -66,11 +66,13 @@ parseable: storageClass: "" accessMode: ReadWriteOnce size: 5Gi + ## enabling the disk on querier adds the hot-tier. + ## local-store is not supported on hot-tier. querier: enabled: false storageClass: "" accessMode: ReadWriteOnce - size: 5Gi + size: 20Gi ## comment out the secrets depending upon deployment option ## localModeSecret if store is set to local ## blobModeSecret if store is set to blob-store @@ -379,7 +381,15 @@ fluent-bit: # [INPUT] # Name tail - # Path /var/log/containers/{NGINX_DEPLOYMENT_NAME}-*.log + # Path /var/log/containers/{{ NGINX_POD_NAME }}-*.log + # Parser nginx_access + # Tag kube.* + # Mem_Buf_Limit 5MB + # Skip_Long_Lines On + + # [INPUT] + # Name tail + # Path /var/log/containers/{{ NGINX_POD_NAME }}-*.log # Parser nginx_error # Tag kube.* # Mem_Buf_Limit 5MB @@ -436,12 +446,19 @@ fluent-bit: Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%L - [PARSER] - Name nginx_error - Format regex - Regex ^(?\S+Z) stderr F (?\S+ \S+) \[(?\S+)\] (?\d+#\d+): \*(?\d+) (?.*?) client: (?\S+), server: (?\S+), request: "(?\S+) (?\S+) HTTP/\S+", upstream: "(?[^"]+)", host: "(?\S+)"$ - Time_Key timestamp - Time_Format %Y-%m-%dT%H:%M:%S.%LZ + # [PARSER] + # Name nginx_error + # Format regex + # Regex ^(?\S+Z) stderr F (?\S+ \S+) \[(?\S+)\] (?\d+#\d+): \*(?\d+) (?.*?) client: (?\S+), server: (?\S+), request: "(?\S+) (?\S+) HTTP/\S+", upstream: "(?[^"]+)", host: "(?\S+)"$ + # Time_Key timestamp + # Time_Format %Y/%m/%d %H:%M:%S + + # [PARSER] + # Name nginx_access + # Format regex + # Regex (?[^ ]*) - (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*) "(?[^\"]*)" "(?[^\"]*)" (?[^ ]*) (?[^ ]*) \[(?[^ ]*)\] \[(?[^ ]*)\] (?[^,]*),?(?:[^,]*),?(?:[^ ]*) (?[^,]*),?(?:[^,]*),?(?:[^ ]*) (?[^,]*),?(?:[^,]*),?(?:[^ ]*) (?[^,]*),?(?:[^,]*),?(?:[^ ]*) (?[^ ]*) + # Time_Key timestamp + # Time_Format %d/%b/%Y:%H:%M:%S %z # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname" volumeMounts: