Skip to content

Commit 1a55a11

Browse files
nikhilsinhaparseablenikhilsinhaparseable
authored
Merge branch 'main' into simplify-schema-merge
2 parents edd4d43 + 932daa0 commit 1a55a11

File tree

13 files changed

+1006
-89
lines changed

13 files changed

+1006
-89
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,19 @@ build = "build.rs"
1010
[dependencies]
1111
# Arrow and DataFusion ecosystem
1212
arrow = "54.0.0"
13-
arrow-array = "54.0.0"
13+
arrow-array = "54.0.0"
1414
arrow-flight = { version = "54.0.0", features = ["tls"] }
1515
arrow-ipc = { version = "54.0.0", features = ["zstd"] }
1616
arrow-json = "54.0.0"
1717
arrow-schema = { version = "54.0.0", features = ["serde"] }
1818
arrow-select = "54.0.0"
1919
datafusion = "45.0.0"
20-
object_store = { version = "0.11.2", features = ["cloud", "aws", "azure"] }
20+
object_store = { version = "0.11.2", features = [
21+
"cloud",
22+
"aws",
23+
"azure",
24+
"gcp",
25+
] }
2126
parquet = "54.0.0"
2227

2328
# Web server and HTTP-related
@@ -34,7 +39,11 @@ tower-http = { version = "0.6.1", features = ["cors"] }
3439
url = "2.4.0"
3540

3641
# Connectors dependencies
37-
rdkafka = { version = "0.37", optional = true, features = ["cmake-build", "tracing", "libz-static"] }
42+
rdkafka = { version = "0.37", optional = true, features = [
43+
"cmake-build",
44+
"tracing",
45+
"libz-static",
46+
] }
3847
sasl2-sys = { version = "0.1.22", optional = true, features = ["vendored"] }
3948

4049
# Authentication and Security
@@ -145,7 +154,14 @@ assets-sha1 = "3e703ef8bedf8ae55fd31713f6267ad14ad3d29d"
145154

146155
[features]
147156
debug = []
148-
kafka = ["rdkafka", "rdkafka/ssl-vendored", "rdkafka/ssl", "rdkafka/sasl", "sasl2-sys", "sasl2-sys/vendored"]
157+
kafka = [
158+
"rdkafka",
159+
"rdkafka/ssl-vendored",
160+
"rdkafka/ssl",
161+
"rdkafka/sasl",
162+
"sasl2-sys",
163+
"sasl2-sys/vendored",
164+
]
149165

150166
[profile.release-lto]
151167
inherits = "release"

‎Dockerfile.debug‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# along with this program. If not, see <http://www.gnu.org/licenses/>.
1515

1616
# build stage
17-
FROM rust:1.84.0-bookworm AS builder
17+
FROM docker.io/rust:1.84.0-bookworm AS builder
1818

1919
LABEL org.opencontainers.image.title="Parseable"
2020
LABEL maintainer="Parseable Team <[email protected]>"

‎docker-compose-gcs-distributed-test.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
networks:
2+
parseable-internal:
3+
4+
services:
5+
# query server
6+
parseable-query:
7+
container_name: parseable-query
8+
build:
9+
context: .
10+
dockerfile: Dockerfile.debug
11+
platform: linux/amd64
12+
command: ["parseable", "gcs-store"]
13+
ports:
14+
- "8000:8000"
15+
environment:
16+
- P_GCS_BUCKET=parseable-test-gcs-local
17+
- P_STAGING_DIR=/tmp/data
18+
- P_USERNAME=parseableadmin
19+
- P_PASSWORD=parseableadmin
20+
- P_CHECK_UPDATE=false
21+
- P_PARQUET_COMPRESSION_ALGO=snappy
22+
- P_MODE=query
23+
- RUST_LOG=warn
24+
- GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/${GCS_CREDENTIALS_FILE:-key.json}
25+
networks:
26+
- parseable-internal
27+
healthcheck:
28+
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
29+
interval: 15s
30+
timeout: 20s
31+
retries: 5
32+
deploy:
33+
restart_policy:
34+
condition: on-failure
35+
delay: 20s
36+
max_attempts:
37+
volumes:
38+
- "${GCS_CREDENTIALS_PATH:-./credentials}:/parseable/svc/:ro,z"
39+
# ingest server one
40+
parseable-ingest-one:
41+
container_name: parseable-ingest-one
42+
build:
43+
context: .
44+
dockerfile: Dockerfile.debug
45+
platform: linux/amd64
46+
command: ["parseable", "gcs-store"]
47+
ports:
48+
- "8000"
49+
environment:
50+
- P_GCS_BUCKET=parseable-test-gcs-local
51+
- P_STAGING_DIR=/tmp/data
52+
- P_USERNAME=parseableadmin
53+
- P_PASSWORD=parseableadmin
54+
- P_CHECK_UPDATE=false
55+
- P_PARQUET_COMPRESSION_ALGO=snappy
56+
- P_MODE=ingest
57+
- P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
58+
- RUST_LOG=warn
59+
- GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/${GCS_CREDENTIALS_FILE:-key.json}
60+
networks:
61+
- parseable-internal
62+
healthcheck:
63+
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
64+
interval: 15s
65+
timeout: 20s
66+
retries: 5
67+
depends_on:
68+
parseable-query:
69+
condition: service_healthy
70+
deploy:
71+
restart_policy:
72+
condition: on-failure
73+
delay: 20s
74+
max_attempts: 3
75+
volumes:
76+
- "${GCS_CREDENTIALS_PATH:-./credentials}:/parseable/svc/:ro,z"
77+
78+
quest:
79+
platform: linux/amd64
80+
image: ghcr.io/parseablehq/quest:main
81+
pull_policy: always
82+
command:
83+
[
84+
"load",
85+
"http://parseable-query:8000",
86+
"parseableadmin",
87+
"parseableadmin",
88+
"20",
89+
"10",
90+
"5m",
91+
"storage.googleapis.com",
92+
"",
93+
"",
94+
"parseable-test-gcs-local",
95+
"http://parseable-ingest-one:8000",
96+
"parseableadmin",
97+
"parseableadmin",
98+
]
99+
networks:
100+
- parseable-internal
101+
depends_on:
102+
parseable-query:
103+
condition: service_healthy
104+
parseable-ingest-one:
105+
condition: service_healthy
106+
deploy:
107+
restart_policy:
108+
condition: on-failure
109+
delay: 20s
110+
max_attempts: 3

‎helm/templates/ingestor-statefulset.yaml‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ spec:
6666
tolerations:
6767
{{- toYaml . | nindent 8 }}
6868
{{- end }}
69+
{{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
70+
volumes:
71+
- name: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
72+
secret:
73+
secretName: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
74+
{{- end }}
6975
containers:
7076
- name: {{ .Chart.Name }}
7177
securityContext:
@@ -74,19 +80,19 @@ spec:
7480
imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
7581
args:
7682
- /usr/bin/parseable
77-
- {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}
83+
- {{ .Values.parseable.store | quote }}
7884
- --ingestor-endpoint=$(HOSTNAME).{{ include "parseable.fullname" . }}-ingestor-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.parseable.highAvailability.ingestor.port }}
7985
env:
80-
{{- range $key, $value := .Values.parseable.highAvailability.ingestor.env }}
81-
- name: {{ $key }}
82-
value: {{ tpl $value $ | quote }}
8386
- name: HOSTNAME
8487
valueFrom:
8588
fieldRef:
8689
apiVersion: v1
8790
fieldPath: metadata.name
91+
{{- range $key, $value := .Values.parseable.highAvailability.ingestor.env }}
92+
- name: {{ $key }}
93+
value: {{ tpl $value $ | quote }}
8894
{{- end }}
89-
95+
9096
{{- if .Values.parseable.auditLogging.enabled }}
9197
- name: P_AUDIT_LOGGER
9298
value: {{ .Values.parseable.auditLogging.p_server | quote }}
@@ -111,11 +117,13 @@ spec:
111117
{{- end }}
112118

113119
{{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
120+
- name: GOOGLE_APPLICATION_CREDENTIALS
121+
value: {{ .Values.parseable.gcsModeSecret.auth.mount_path }}/{{ .Values.parseable.gcsModeSecret.auth.secret_key }}
114122
{{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
115123
{{- range $key := $secret.keys }}
116124
{{- $envPrefix := $secret.prefix | default "" | upper }}
117125
{{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
118-
- name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
126+
- name: {{ $envPrefix }}{{ $envKey }}
119127
valueFrom:
120128
secretKeyRef:
121129
name: {{ $secret.name }}
@@ -137,30 +145,39 @@ spec:
137145
{{- end }}
138146
{{- end }}
139147
{{- end }}
148+
140149
- name: P_MODE
141150
value: "ingest"
151+
142152
{{- if .Values.parseable.kafkaConnector.enabled }}
143-
{{- range $key, $value := .Values.parseable.kafkaConnector.env }}
153+
{{- range $key, $value := .Values.parseable.kafkaConnector.env }}
144154
- name: {{ $key }}
145155
value: {{ tpl $value $ | quote }}
146-
{{- end }}
156+
{{- end }}
147157
{{- end }}
148158

149159
ports:
150160
- containerPort: {{ .Values.parseable.highAvailability.ingestor.port }}
151-
{{- with .Values.readinessProbe }}
161+
{{- with .Values.readinessProbe }}
152162
readinessProbe:
153-
{{ toYaml . | nindent 12 }}
154-
{{- end }}
163+
{{- toYaml . | nindent 10 }}
164+
{{- end }}
155165
resources:
156-
{{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 12 }}
157-
{{- if .Values.parseable.persistence.ingestor.enabled }}
166+
{{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 10 }}
167+
{{- if or .Values.parseable.persistence.ingestor.enabled (and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled) }}
158168
volumeMounts:
169+
{{- if .Values.parseable.persistence.ingestor.enabled }}
159170
- mountPath: "/parseable/staging"
160171
name: stage-volume
161172
{{- end }}
162-
volumeClaimTemplates:
173+
{{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
174+
- mountPath: {{ .Values.parseable.gcsModeSecret.auth.mount_path }}
175+
name: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
176+
readOnly: true
177+
{{- end }}
178+
{{- end }}
163179
{{- if .Values.parseable.persistence.ingestor.enabled }}
180+
volumeClaimTemplates:
164181
- metadata:
165182
name: stage-volume
166183
spec:
@@ -171,4 +188,4 @@ spec:
171188
requests:
172189
storage: {{ .Values.parseable.persistence.ingestor.size | quote }}
173190
{{- end }}
174-
{{- end }}
191+
{{- end }}

0 commit comments

Comments
 (0)