From a25878ddb7e1ee73ebd997e3f09f04d4f51f7340 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 6 May 2023 16:24:21 +0530 Subject: [PATCH 1/3] Add log stream creation and set retention in Helm Chart --- helm/templates/_helpers_config_logstream.txt | 22 +++++++ helm/templates/configmap.yaml | 10 ++++ helm/templates/deployment.yaml | 1 + helm/templates/logstream-job.yaml | 63 ++++++++++++++++++++ helm/values.yaml | 33 +++++++--- server/src/handlers/http/ingest.rs | 2 +- 6 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 helm/templates/_helpers_config_logstream.txt create mode 100644 helm/templates/configmap.yaml create mode 100644 helm/templates/logstream-job.yaml diff --git a/helm/templates/_helpers_config_logstream.txt b/helm/templates/_helpers_config_logstream.txt new file mode 100644 index 000000000..a6f4f9cd4 --- /dev/null +++ b/helm/templates/_helpers_config_logstream.txt @@ -0,0 +1,22 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. +IFS=$' \t\r\n' + +createStream() { + STREAM=$1 + echo; + echo \"Creating the log stream $STREAM\"; + curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM"; +} + +setRetention() { + +} + +{{ if .Values.parseable.logstream }} +# Create the logstream +{{- range .Values.parseable.logstream }} +createStream {{.name}} +setRetention {{.retention}} +{{- end }} +{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml new file mode 100644 index 000000000..ca59b31f4 --- /dev/null +++ b/helm/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "parseable.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "parseable.labels" . | nindent 4 }} +data: + config-logstream: |- + {{- include (print $.Template.BasePath "/_helpers_config_logstream.txt") . | nindent 4 }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 3aa592843..2d669724d 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "parseable.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "parseable.labels" . | nindent 4 }} spec: diff --git a/helm/templates/logstream-job.yaml b/helm/templates/logstream-job.yaml new file mode 100644 index 000000000..15b1dcd17 --- /dev/null +++ b/helm/templates/logstream-job.yaml @@ -0,0 +1,63 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "parseable.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "parseable.labels" . | nindent 4 }} +spec: + template: + metadata: + {{- with .Values.parseable.podAnnotations }} + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "parseable.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: OnFailure + securityContext: + {{- toYaml .Values.parseable.podSecurityContext | nindent 8 }} + volumes: + - name: parseable-init + projected: + sources: + - configMap: + name: {{ include "parseable.fullname" . }} + containers: + - name: config-logstream + volumeMounts: + - name: parseable-init + mountPath: /config + securityContext: + {{- toYaml .Values.parseable.securityContext | nindent 12 }} + env: + {{- if .Values.parseable.local }} + {{- range $secret := .Values.parseable.localModeSecret }} + {{- range $key := $secret.keys }} + {{- $envPrefix := $secret.prefix | default "" | upper }} + {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} + - name: {{ $envPrefix }}{{ $envKey }} + valueFrom: + secretKeyRef: + name: {{ $secret.name }} + key: {{ $key }} + {{- end }} + {{- end }} + {{- else}} + {{- range $secret := .Values.parseable.s3ModeSecret }} + {{- range $key := $secret.keys }} + {{- $envPrefix := $secret.prefix | default "" | upper }} + {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} + - name: {{ $envPrefix }}{{ $envKey }} + valueFrom: + secretKeyRef: + name: {{ $secret.name }} + key: {{ $key }} + {{- end }} + {{- end }} + {{- end }} + image: curlimages/curl:8.00.0 + command: [ "/bin/sh", "/config/config-logstream" ] + backoffLimit: 20 diff --git a/helm/values.yaml b/helm/values.yaml index 5154bfce2..6f3bddf20 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,11 +1,27 @@ parseable: image: repository: parseable/parseable - tag: v0.4.3 + tag: v0.4.0 pullPolicy: Always - local: false - # env: - # RUST_LOG: info + ## Set to true if you want to deploy Parseable in local mode (store logs + ## on local mount point instead of S3 bucket) + local: true + env: + RUST_LOG: error + ## Use this to create and configure Parseable logstreams + logstream: + - name: "vectordemo" + retention: + description: "Delete logs after 30 days" + action: "delete" + duration: "30d" + - name: "fluentbitdemo" + retention: + description: "Delete logs after 30 days" + action: "delete" + duration: "30d" + ## enable/disable persistence using PVC for the Data and Staging directories + ## Note that Data directory is needed only for local mode persistence: staging: enabled: false @@ -17,6 +33,7 @@ parseable: storageClass: "" accessMode: ReadWriteOnce size: 1Gi + ## Use localModeSecret: - type: env name: parseable-env-secret @@ -58,11 +75,11 @@ parseable: port: 80 resources: limits: - cpu: 500m - memory: 4Gi + cpu: 100m + memory: 250Mi requests: - cpu: 250m - memory: 1Gi + cpu: 100m + memory: 250Mi securityContext: allowPrivilegeEscalation: false podAnnotations: {} diff --git a/server/src/handlers/http/ingest.rs b/server/src/handlers/http/ingest.rs index d740b9d9d..8d3883f1c 100644 --- a/server/src/handlers/http/ingest.rs +++ b/server/src/handlers/http/ingest.rs @@ -32,7 +32,7 @@ use crate::metadata::STREAM_INFO; use crate::utils::header_parsing::{collect_labelled_headers, ParseHeaderError}; // Handler for POST /api/v1/ingest -// ingests events by extacting stream name from header +// ingests events by extracting stream name from header // creates if stream does not exist pub async fn ingest(req: HttpRequest, body: Bytes) -> Result { if let Some((_, stream_name)) = req From 26cac18611aa570ea21c2c3dc610234f94407ab9 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 6 May 2023 23:38:27 +0530 Subject: [PATCH 2/3] Add set retention call --- helm/templates/_helpers_config_logstream.txt | 24 +++++++++++++++++--- helm/values.yaml | 4 +--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/helm/templates/_helpers_config_logstream.txt b/helm/templates/_helpers_config_logstream.txt index a6f4f9cd4..784860125 100644 --- a/helm/templates/_helpers_config_logstream.txt +++ b/helm/templates/_helpers_config_logstream.txt @@ -6,17 +6,35 @@ createStream() { STREAM=$1 echo; echo \"Creating the log stream $STREAM\"; - curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM"; + + response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w '\n\n%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM"); + + http_code=$(tail -n1 <<< "$response") + content=$(sed '$ d' <<< "$response") + echo \"status code: $http_code\" + echo \"response message: $content\" } setRetention() { - + STREAM=$1 + ACTION=$2 + DURATION=$3 + + echo; + echo \"Setting the retention for $STREAM\"; + + response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w '\n\n%{http_code}' --location --request POST "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM/retention" --data "[{\"description\":\"$ACTION logs after $DURATION\",\"action\":\"$ACTION\",\"duration\":\"$DURATION\"}]"); + + http_code=$(tail -n1 <<< "$response") + content=$(sed '$ d' <<< "$response") + echo \"status code: $http_code\" + echo \"response message: $content\" } {{ if .Values.parseable.logstream }} # Create the logstream {{- range .Values.parseable.logstream }} createStream {{.name}} -setRetention {{.retention}} +setRetention {{.name}} {{.retention.action}} {{.retention.duration}} {{- end }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 6f3bddf20..1fcb66fa0 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,7 +1,7 @@ parseable: image: repository: parseable/parseable - tag: v0.4.0 + tag: v0.4.1 pullPolicy: Always ## Set to true if you want to deploy Parseable in local mode (store logs ## on local mount point instead of S3 bucket) @@ -12,12 +12,10 @@ parseable: logstream: - name: "vectordemo" retention: - description: "Delete logs after 30 days" action: "delete" duration: "30d" - name: "fluentbitdemo" retention: - description: "Delete logs after 30 days" action: "delete" duration: "30d" ## enable/disable persistence using PVC for the Data and Staging directories From 89696463d9a307e85ef2ae91568fde9f9e149b49 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sun, 7 May 2023 23:12:24 +0530 Subject: [PATCH 3/3] Add status code --- helm/templates/_helpers_config_logstream.txt | 24 +++++++------ helm/values.yaml | 37 ++++++++++---------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/helm/templates/_helpers_config_logstream.txt b/helm/templates/_helpers_config_logstream.txt index 784860125..6fa59b15b 100644 --- a/helm/templates/_helpers_config_logstream.txt +++ b/helm/templates/_helpers_config_logstream.txt @@ -7,12 +7,14 @@ createStream() { echo; echo \"Creating the log stream $STREAM\"; - response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w '\n\n%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM"); + response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w 'HTTPSTATUS:%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM"); - http_code=$(tail -n1 <<< "$response") - content=$(sed '$ d' <<< "$response") - echo \"status code: $http_code\" - echo \"response message: $content\" + HTTP_BODY=$(echo $response | sed -e 's/HTTPSTATUS\:.*//g') + HTTP_STATUS=$(echo $response | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + + echo \"API response:\" + echo \"HTTP status code: $HTTP_STATUS\" + echo \"HTTP response message: $HTTP_BODY\" } setRetention() { @@ -23,12 +25,14 @@ setRetention() { echo; echo \"Setting the retention for $STREAM\"; - response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w '\n\n%{http_code}' --location --request POST "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM/retention" --data "[{\"description\":\"$ACTION logs after $DURATION\",\"action\":\"$ACTION\",\"duration\":\"$DURATION\"}]"); + response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w 'HTTPSTATUS:%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM/retention" --data "[{\"description\":\"$ACTION logs after $DURATION\",\"action\":\"$ACTION\",\"duration\":\"$DURATION\"}]"); + + HTTP_BODY=$(echo $response | sed -e 's/HTTPSTATUS\:.*//g') + HTTP_STATUS=$(echo $response | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') - http_code=$(tail -n1 <<< "$response") - content=$(sed '$ d' <<< "$response") - echo \"status code: $http_code\" - echo \"response message: $content\" + echo \"API response:\" + echo \"HTTP status code: $HTTP_STATUS\" + echo \"HTTP response message: $HTTP_BODY\" } {{ if .Values.parseable.logstream }} diff --git a/helm/values.yaml b/helm/values.yaml index 1fcb66fa0..849dbcb1e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,23 +1,23 @@ parseable: image: repository: parseable/parseable - tag: v0.4.1 + tag: v0.4.3 pullPolicy: Always ## Set to true if you want to deploy Parseable in local mode (store logs ## on local mount point instead of S3 bucket) - local: true - env: - RUST_LOG: error - ## Use this to create and configure Parseable logstreams - logstream: - - name: "vectordemo" - retention: - action: "delete" - duration: "30d" - - name: "fluentbitdemo" - retention: - action: "delete" - duration: "30d" + local: false + # env: + # RUST_LOG: error + # ## Enable to create and configure logstreams + # logstream: + # - name: "vectordemo" + # retention: + # action: "delete" + # duration: "30d" + # - name: "fluentbitdemo" + # retention: + # action: "delete" + # duration: "30d" ## enable/disable persistence using PVC for the Data and Staging directories ## Note that Data directory is needed only for local mode persistence: @@ -31,7 +31,6 @@ parseable: storageClass: "" accessMode: ReadWriteOnce size: 1Gi - ## Use localModeSecret: - type: env name: parseable-env-secret @@ -73,11 +72,11 @@ parseable: port: 80 resources: limits: - cpu: 100m - memory: 250Mi + cpu: 500m + memory: 4Gi requests: - cpu: 100m - memory: 250Mi + cpu: 250m + memory: 1Gi securityContext: allowPrivilegeEscalation: false podAnnotations: {}