parseablehq · nitisht · Jan 12, 2024 · Jan 12, 2024
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-# Parseable Server (C) 2022 - 2023 Parseable, Inc.
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as

diff --git a/Dockerfile.debug b/Dockerfile.debug
@@ -1,4 +1,4 @@
-# Parseable Server (C) 2022 - 2023 Parseable, Inc.
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as

diff --git a/server/build.rs b/server/build.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/about.rs b/server/src/about.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/alerts/mod.rs b/server/src/alerts/mod.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/alerts/parser.rs b/server/src/alerts/parser.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/alerts/rule.rs b/server/src/alerts/rule.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/alerts/target.rs b/server/src/alerts/target.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/analytics.rs b/server/src/analytics.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/banner.rs b/server/src/banner.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/catalog.rs b/server/src/catalog.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/catalog/column.rs b/server/src/catalog/column.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/catalog/manifest.rs b/server/src/catalog/manifest.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/catalog/snapshot.rs b/server/src/catalog/snapshot.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event.rs b/server/src/event.rs
@@ -1,5 +1,5 @@
 /*
-* Parseable Server (C) 2022 - 2023 Parseable, Inc.
+* Parseable Server (C) 2022 - 2024 Parseable, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event/format.rs b/server/src/event/format.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event/format/json.rs b/server/src/event/format/json.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event/writer.rs b/server/src/event/writer.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event/writer/file_writer.rs b/server/src/event/writer/file_writer.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/event/writer/mem_writer.rs b/server/src/event/writer/mem_writer.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers.rs b/server/src/handlers.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -22,16 +22,22 @@ pub mod livetail;
 const PREFIX_TAGS: &str = "x-p-tag-";
 const PREFIX_META: &str = "x-p-meta-";
 const STREAM_NAME_HEADER_KEY: &str = "x-p-stream";
-const AUTHORIZATION_KEY: &str = "authorization";
-const KINESIS_COMMON_ATTRIBUTES_KEY: &str = "x-amz-firehose-common-attributes";
 const LOG_SOURCE_KEY: &str = "x-p-log-source";
+
+const AUTHORIZATION_KEY: &str = "authorization";
 const SEPARATOR: char = '^';
 
 const OIDC_SCOPE: &str = "openid profile email";
 const COOKIE_AGE_DAYS: usize = 7;
 const SESSION_COOKIE_NAME: &str = "session";
 const USER_COOKIE_NAME: &str = "username";
 
-// constants for Log Source values for known sources
+// constants for log Source values for known sources and formats
 const LOG_SOURCE_KINESIS: &str = "kinesis";
+
+// OpenTelemetry sends data in JSON format with
+// specification as explained here https://opentelemetry.io/docs/specs/otel/logs/data-model/
 const LOG_SOURCE_OTEL: &str = "otel";
+
+// AWS Kinesis constants
+const KINESIS_COMMON_ATTRIBUTES_KEY: &str = "x-amz-firehose-common-attributes";
diff --git a/server/src/handlers/http.rs b/server/src/handlers/http.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/about.rs b/server/src/handlers/http/about.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/health_check.rs b/server/src/handlers/http/health_check.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/ingest.rs b/server/src/handlers/http/ingest.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/kinesis.rs b/server/src/handlers/http/kinesis.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -37,6 +37,30 @@ struct Data {
     data: String,
 }
 
+// Flatten Kinesis logs is used to flatten the Kinesis logs into a queryable JSON format.
+// Kinesis logs are in the format
+// {
+//     "requestId": "9b848d8a-2d89-474b-b073-04b8e5232210",
+//     "timestamp": 1705026780451,
+//     "records": [
+//         {
+//             "data": "eyJDSEFOR0UiOi0wLjQ1LCJQUklDRSI6NS4zNiwiVElDS0VSX1NZTUJPTCI6IkRFRyIsIlNFQ1RPUiI6IkVORVJHWSJ9"
+//         }
+//     ]
+// }
+// The data field is base64 encoded JSON (there can be multiple data fields), and there is a requestId and timestamp field.
+// Kinesis logs are flattened to the following format:
+// {
+//     "CHANGE": 3.16,
+//     "PRICE": 73.76,
+//     "SECTOR": "RETAIL",
+//     "TICKER_SYMBOL": "WMT",
+//     "p_metadata": "",
+//     "p_tags": "",
+//     "p_timestamp": "2024-01-11T09:08:34.290",
+//     "requestId": "b858288a-f5d8-4181-a746-3f3dd716be8a",
+//     "timestamp": "1704964113659"
+// }
 pub fn flatten_kinesis_logs(body: &Bytes) -> Vec<BTreeMap<String, Value>> {
     let body_str = std::str::from_utf8(body).unwrap();
     let message: Message = serde_json::from_str(body_str).unwrap();

diff --git a/server/src/handlers/http/llm.rs b/server/src/handlers/http/llm.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/logstream.rs b/server/src/handlers/http/logstream.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/middleware.rs b/server/src/handlers/http/middleware.rs
@@ -1,5 +1,5 @@
 /*
-* Parseable Server (C) 2022 - 2023 Parseable, Inc.
+* Parseable Server (C) 2022 - 2024 Parseable, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/oidc.rs b/server/src/handlers/http/oidc.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/query.rs b/server/src/handlers/http/query.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/rbac.rs b/server/src/handlers/http/rbac.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/http/role.rs b/server/src/handlers/http/role.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/handlers/livetail.rs b/server/src/handlers/livetail.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/livetail.rs b/server/src/livetail.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/localcache.rs b/server/src/localcache.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/main.rs b/server/src/main.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/metadata.rs b/server/src/metadata.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/metrics/mod.rs b/server/src/metrics/mod.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/metrics/storage.rs b/server/src/metrics/storage.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/migration.rs b/server/src/migration.rs
@@ -1,5 +1,5 @@
 /*
-* Parseable Server (C) 2022 - 2023 Parseable, Inc.
+* Parseable Server (C) 2022 - 2024 Parseable, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/migration/metadata_migration.rs b/server/src/migration/metadata_migration.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/migration/schema_migration.rs b/server/src/migration/schema_migration.rs
@@ -1,5 +1,5 @@
 /*
-* Parseable Server (C) 2022 - 2023 Parseable, Inc.
+* Parseable Server (C) 2022 - 2024 Parseable, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/migration/stream_metadata_migration.rs b/server/src/migration/stream_metadata_migration.rs
@@ -1,5 +1,5 @@
 /*
-* Parseable Server (C) 2022 - 2023 Parseable, Inc.
+* Parseable Server (C) 2022 - 2024 Parseable, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/oidc.rs b/server/src/oidc.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/option.rs b/server/src/option.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/query.rs b/server/src/query.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/query/filter_optimizer.rs b/server/src/query/filter_optimizer.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

diff --git a/server/src/query/listing_table_builder.rs b/server/src/query/listing_table_builder.rs
@@ -1,5 +1,5 @@
 /*
- * Parseable Server (C) 2022 - 2023 Parseable, Inc.
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as