Skip to content

Commit 1be9ad8

Browse files
nitishtnitisht
authored
Add log stream creation and set retention in Helm Chart (#397)
1 parent d1e4394 commit 1be9ad8

File tree

6 files changed

+134
-2
lines changed

6 files changed

+134
-2
lines changed

‎helm/templates/_helpers_config_logstream.txt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/sh
2+
set -e ; # Have script exit in the event of a failed command.
3+
IFS=$' \t\r\n'
4+
5+
createStream() {
6+
STREAM=$1
7+
echo;
8+
echo \"Creating the log stream $STREAM\";
9+
10+
response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w 'HTTPSTATUS:%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM");
11+
12+
HTTP_BODY=$(echo $response | sed -e 's/HTTPSTATUS\:.*//g')
13+
HTTP_STATUS=$(echo $response | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
14+
15+
echo \"API response:\"
16+
echo \"HTTP status code: $HTTP_STATUS\"
17+
echo \"HTTP response message: $HTTP_BODY\"
18+
}
19+
20+
setRetention() {
21+
STREAM=$1
22+
ACTION=$2
23+
DURATION=$3
24+
25+
echo;
26+
echo \"Setting the retention for $STREAM\";
27+
28+
response=$(curl -sS --header 'Content-Type: application/json' -u "$P_USERNAME":"$P_PASSWORD" -w 'HTTPSTATUS:%{http_code}' --location --request PUT "http://{{ include "parseable.fullname" . }}.{{ .Release.Namespace }}:{{ $.Values.parseable.service.port }}/api/v1/logstream/$STREAM/retention" --data "[{\"description\":\"$ACTION logs after $DURATION\",\"action\":\"$ACTION\",\"duration\":\"$DURATION\"}]");
29+
30+
HTTP_BODY=$(echo $response | sed -e 's/HTTPSTATUS\:.*//g')
31+
HTTP_STATUS=$(echo $response | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
32+
33+
echo \"API response:\"
34+
echo \"HTTP status code: $HTTP_STATUS\"
35+
echo \"HTTP response message: $HTTP_BODY\"
36+
}
37+
38+
{{ if .Values.parseable.logstream }}
39+
# Create the logstream
40+
{{- range .Values.parseable.logstream }}
41+
createStream {{.name}}
42+
setRetention {{.name}} {{.retention.action}} {{.retention.duration}}
43+
{{- end }}
44+
{{- end }}

‎helm/templates/configmap.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: {{ include "parseable.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
{{- include "parseable.labels" . | nindent 4 }}
8+
data:
9+
config-logstream: |-
10+
{{- include (print $.Template.BasePath "/_helpers_config_logstream.txt") . | nindent 4 }}

‎helm/templates/deployment.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ apiVersion: apps/v1
22
kind: Deployment
33
metadata:
44
name: {{ include "parseable.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
56
labels:
67
{{- include "parseable.labels" . | nindent 4 }}
78
spec:

‎helm/templates/logstream-job.yaml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: {{ include "parseable.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
{{- include "parseable.labels" . | nindent 4 }}
8+
spec:
9+
template:
10+
metadata:
11+
{{- with .Values.parseable.podAnnotations }}
12+
annotations:
13+
"helm.sh/hook": post-install,post-upgrade
14+
{{- toYaml . | nindent 8 }}
15+
{{- end }}
16+
labels:
17+
{{- include "parseable.selectorLabels" . | nindent 8 }}
18+
spec:
19+
restartPolicy: OnFailure
20+
securityContext:
21+
{{- toYaml .Values.parseable.podSecurityContext | nindent 8 }}
22+
volumes:
23+
- name: parseable-init
24+
projected:
25+
sources:
26+
- configMap:
27+
name: {{ include "parseable.fullname" . }}
28+
containers:
29+
- name: config-logstream
30+
volumeMounts:
31+
- name: parseable-init
32+
mountPath: /config
33+
securityContext:
34+
{{- toYaml .Values.parseable.securityContext | nindent 12 }}
35+
env:
36+
{{- if .Values.parseable.local }}
37+
{{- range $secret := .Values.parseable.localModeSecret }}
38+
{{- range $key := $secret.keys }}
39+
{{- $envPrefix := $secret.prefix | default "" | upper }}
40+
{{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
41+
- name: {{ $envPrefix }}{{ $envKey }}
42+
valueFrom:
43+
secretKeyRef:
44+
name: {{ $secret.name }}
45+
key: {{ $key }}
46+
{{- end }}
47+
{{- end }}
48+
{{- else}}
49+
{{- range $secret := .Values.parseable.s3ModeSecret }}
50+
{{- range $key := $secret.keys }}
51+
{{- $envPrefix := $secret.prefix | default "" | upper }}
52+
{{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
53+
- name: {{ $envPrefix }}{{ $envKey }}
54+
valueFrom:
55+
secretKeyRef:
56+
name: {{ $secret.name }}
57+
key: {{ $key }}
58+
{{- end }}
59+
{{- end }}
60+
{{- end }}
61+
image: curlimages/curl:8.00.0
62+
command: [ "/bin/sh", "/config/config-logstream" ]
63+
backoffLimit: 20

‎helm/values.yaml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,23 @@ parseable:
33
repository: parseable/parseable
44
tag: v0.4.3
55
pullPolicy: Always
6+
## Set to true if you want to deploy Parseable in local mode (store logs
7+
## on local mount point instead of S3 bucket)
68
local: false
79
# env:
8-
# RUST_LOG: info
10+
# RUST_LOG: error
11+
# ## Enable to create and configure logstreams
12+
# logstream:
13+
# - name: "vectordemo"
14+
# retention:
15+
# action: "delete"
16+
# duration: "30d"
17+
# - name: "fluentbitdemo"
18+
# retention:
19+
# action: "delete"
20+
# duration: "30d"
21+
## enable/disable persistence using PVC for the Data and Staging directories
22+
## Note that Data directory is needed only for local mode
923
persistence:
1024
staging:
1125
enabled: false

‎server/src/handlers/http/ingest.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use crate::metadata::STREAM_INFO;
3232
use crate::utils::header_parsing::{collect_labelled_headers, ParseHeaderError};
3333

3434
// Handler for POST /api/v1/ingest
35-
// ingests events by extacting stream name from header
35+
// ingests events by extracting stream name from header
3636
// creates if stream does not exist
3737
pub async fn ingest(req: HttpRequest, body: Bytes) -> Result<HttpResponse, PostError> {
3838
if let Some((_, stream_name)) = req

0 commit comments

Comments
 (0)