Skip to content

Add new buffering options to all OP destinations #830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add new buffering options to all OP destinations #830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .generated-info
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"spec_repo_commit": "69305be",
"generated": "2025-08-05 16:25:22.016"
"spec_repo_commit": "884871f",
"generated": "2025-08-05 21:36:17.643"
}
91 changes: 91 additions & 0 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4946,6 +4946,8 @@ components:
description: Optional prefix for blobs written to the container.
example: logs/
type: string
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
container_name:
description: The name of the Azure Blob Storage container to store logs
in.
Expand Down Expand Up @@ -24998,6 +25000,8 @@ components:
description: The `microsoft_sentinel` destination forwards logs to Microsoft
Sentinel.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
client_id:
description: Azure AD client ID used for authentication.
example: a1b2c3d4-5678-90ab-cdef-1234567890ab
Expand Down Expand Up @@ -26630,6 +26634,8 @@ components:
properties:
auth:
$ref: '#/components/schemas/ObservabilityPipelineAmazonOpenSearchDestinationAuth'
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
bulk_index:
description: The index to write logs to.
example: logs-index
Expand Down Expand Up @@ -26708,6 +26714,8 @@ components:
description: S3 bucket name.
example: error-logs
type: string
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: Unique identifier for the destination component.
example: amazon-s3-destination
Expand Down Expand Up @@ -26827,6 +26835,28 @@ components:
role session.
type: string
type: object
ObservabilityPipelineBufferOptions:
description: Configuration for buffer settings on destination components.
oneOf:
- $ref: '#/components/schemas/ObservabilityPipelineDiskBufferOptions'
- $ref: '#/components/schemas/ObservabilityPipelineMemoryBufferOptions'
- $ref: '#/components/schemas/ObservabilityPipelineMemoryBufferSizeOptions'
ObservabilityPipelineBufferOptionsDiskType:
default: disk
description: The type of the buffer that will be configured, a disk buffer.
enum:
- disk
type: string
x-enum-varnames:
- DISK
ObservabilityPipelineBufferOptionsMemoryType:
default: memory
description: The type of the buffer that will be configured, a memory buffer.
enum:
- memory
type: string
x-enum-varnames:
- MEMORY
ObservabilityPipelineConfig:
description: Specifies the pipeline's configuration, including its sources,
processors, and destinations.
Expand Down Expand Up @@ -26982,6 +27012,8 @@ components:
ObservabilityPipelineDatadogLogsDestination:
description: The `datadog_logs` destination forwards logs to Datadog Log Management.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: The unique identifier for this component.
example: datadog-logs-destination
Expand Down Expand Up @@ -27083,12 +27115,27 @@ components:
type: string
x-enum-varnames:
- DEDUPE
ObservabilityPipelineDiskBufferOptions:
description: Options for configuring a disk buffer.
properties:
max_size:
description: Maximum size of the disk buffer.
example: 4096
format: int64
maximum: 536870912000
minimum: 268435488
type: integer
type:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsDiskType'
type: object
ObservabilityPipelineElasticsearchDestination:
description: The `elasticsearch` destination writes logs to an Elasticsearch
cluster.
properties:
api_version:
$ref: '#/components/schemas/ObservabilityPipelineElasticsearchDestinationApiVersion'
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
bulk_index:
description: The index to write logs to in Elasticsearch.
example: logs-index
Expand Down Expand Up @@ -27573,6 +27620,8 @@ components:
properties:
auth:
$ref: '#/components/schemas/ObservabilityPipelineGcpAuth'
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
customer_id:
description: The Google Chronicle customer ID.
example: abcdefg123456789
Expand Down Expand Up @@ -27639,6 +27688,8 @@ components:
description: Name of the GCS bucket.
example: error-logs
type: string
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: Unique identifier for the destination component.
example: gcs-destination
Expand Down Expand Up @@ -27949,6 +28000,32 @@ components:
type: string
x-enum-varnames:
- LOGSTASH
ObservabilityPipelineMemoryBufferOptions:
description: Options for configuring a memory buffer by byte size.
properties:
max_size:
description: Maximum size of the disk buffer.
example: 4096
format: int64
maximum: 13743895347
minimum: 1
type: integer
type:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
type: object
ObservabilityPipelineMemoryBufferSizeOptions:
description: Options for configuring a memory buffer by queue length.
properties:
max_events:
description: The `ObservabilityPipelineMemoryBufferSizeOptions` `max_events`.
example: 500
format: int64
maximum: 268435456
minimum: 1
type: integer
type:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
type: object
ObservabilityPipelineMetadataEntry:
description: A custom metadata entry.
properties:
Expand All @@ -27972,6 +28049,8 @@ components:
ObservabilityPipelineNewRelicDestination:
description: The `new_relic` destination sends logs to the New Relic platform.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: The unique identifier for this component.
example: new-relic-destination
Expand Down Expand Up @@ -28109,6 +28188,8 @@ components:
ObservabilityPipelineOpenSearchDestination:
description: The `opensearch` destination writes logs to an OpenSearch cluster.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
bulk_index:
description: The index to write logs to.
example: logs-index
Expand Down Expand Up @@ -28683,6 +28764,8 @@ components:
description: The `rsyslog` destination forwards logs to an external `rsyslog`
server over TCP or UDP using the syslog protocol.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: The unique identifier for this component.
example: rsyslog-destination
Expand Down Expand Up @@ -29157,6 +29240,8 @@ components:
ObservabilityPipelineSentinelOneDestination:
description: The `sentinel_one` destination sends logs to SentinelOne.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: The unique identifier for this component.
example: sentinelone-destination
Expand Down Expand Up @@ -29241,6 +29326,8 @@ components:
'
example: true
type: boolean
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
encoding:
$ref: '#/components/schemas/ObservabilityPipelineSplunkHecDestinationEncoding'
id:
Expand Down Expand Up @@ -29354,6 +29441,8 @@ components:
ObservabilityPipelineSumoLogicDestination:
description: The `sumo_logic` destination forwards logs to Sumo Logic.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
encoding:
$ref: '#/components/schemas/ObservabilityPipelineSumoLogicDestinationEncoding'
header_custom_fields:
Expand Down Expand Up @@ -29457,6 +29546,8 @@ components:
description: The `syslog_ng` destination forwards logs to an external `syslog-ng`
server over TCP or UDP using the syslog protocol.
properties:
buffer:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
id:
description: The unique identifier for this component.
example: syslog-ng-destination
Expand Down
12 changes: 12 additions & 0 deletions src/datadogV2/model/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3808,6 +3808,18 @@ pub mod model_observability_pipeline_config;
pub use self::model_observability_pipeline_config::ObservabilityPipelineConfig;
pub mod model_observability_pipeline_datadog_logs_destination;
pub use self::model_observability_pipeline_datadog_logs_destination::ObservabilityPipelineDatadogLogsDestination;
pub mod model_observability_pipeline_disk_buffer_options;
pub use self::model_observability_pipeline_disk_buffer_options::ObservabilityPipelineDiskBufferOptions;
pub mod model_observability_pipeline_buffer_options_disk_type;
pub use self::model_observability_pipeline_buffer_options_disk_type::ObservabilityPipelineBufferOptionsDiskType;
pub mod model_observability_pipeline_memory_buffer_options;
pub use self::model_observability_pipeline_memory_buffer_options::ObservabilityPipelineMemoryBufferOptions;
pub mod model_observability_pipeline_buffer_options_memory_type;
pub use self::model_observability_pipeline_buffer_options_memory_type::ObservabilityPipelineBufferOptionsMemoryType;
pub mod model_observability_pipeline_memory_buffer_size_options;
pub use self::model_observability_pipeline_memory_buffer_size_options::ObservabilityPipelineMemoryBufferSizeOptions;
pub mod model_observability_pipeline_buffer_options;
pub use self::model_observability_pipeline_buffer_options::ObservabilityPipelineBufferOptions;
pub mod model_observability_pipeline_datadog_logs_destination_type;
pub use self::model_observability_pipeline_datadog_logs_destination_type::ObservabilityPipelineDatadogLogsDestinationType;
pub mod model_observability_pipeline_amazon_s3_destination;
Expand Down
30 changes: 30 additions & 0 deletions src/datadogV2/model/model_azure_storage_destination.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ pub struct AzureStorageDestination {
/// Optional prefix for blobs written to the container.
#[serde(rename = "blob_prefix")]
pub blob_prefix: Option<String>,
/// Configuration for buffer settings on destination components.
#[serde(rename = "buffer")]
pub buffer: Option<crate::datadogV2::model::ObservabilityPipelineBufferOptions>,
/// The name of the Azure Blob Storage container to store logs in.
#[serde(rename = "container_name")]
pub container_name: String,
Expand Down Expand Up @@ -42,6 +45,7 @@ impl AzureStorageDestination {
) -> AzureStorageDestination {
AzureStorageDestination {
blob_prefix: None,
buffer: None,
container_name,
id,
inputs,
Expand All @@ -56,6 +60,14 @@ impl AzureStorageDestination {
self
}

pub fn buffer(
mut self,
value: crate::datadogV2::model::ObservabilityPipelineBufferOptions,
) -> Self {
self.buffer = Some(value);
self
}

pub fn additional_properties(
mut self,
value: std::collections::BTreeMap<String, serde_json::Value>,
Expand Down Expand Up @@ -83,6 +95,9 @@ impl<'de> Deserialize<'de> for AzureStorageDestination {
M: MapAccess<'a>,
{
let mut blob_prefix: Option<String> = None;
let mut buffer: Option<
crate::datadogV2::model::ObservabilityPipelineBufferOptions,
> = None;
let mut container_name: Option<String> = None;
let mut id: Option<String> = None;
let mut inputs: Option<Vec<String>> = None;
Expand All @@ -102,6 +117,20 @@ impl<'de> Deserialize<'de> for AzureStorageDestination {
blob_prefix =
Some(serde_json::from_value(v).map_err(M::Error::custom)?);
}
"buffer" => {
if v.is_null() {
continue;
}
buffer = Some(serde_json::from_value(v).map_err(M::Error::custom)?);
if let Some(ref _buffer) = buffer {
match _buffer {
crate::datadogV2::model::ObservabilityPipelineBufferOptions::UnparsedObject(_buffer) => {
_unparsed = true;
},
_ => {}
}
}
}
"container_name" => {
container_name =
Some(serde_json::from_value(v).map_err(M::Error::custom)?);
Expand Down Expand Up @@ -138,6 +167,7 @@ impl<'de> Deserialize<'de> for AzureStorageDestination {

let content = AzureStorageDestination {
blob_prefix,
buffer,
container_name,
id,
inputs,
Expand Down
30 changes: 30 additions & 0 deletions src/datadogV2/model/model_microsoft_sentinel_destination.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ use std::fmt::{self, Formatter};
#[skip_serializing_none]
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct MicrosoftSentinelDestination {
/// Configuration for buffer settings on destination components.
#[serde(rename = "buffer")]
pub buffer: Option<crate::datadogV2::model::ObservabilityPipelineBufferOptions>,
/// Azure AD client ID used for authentication.
#[serde(rename = "client_id")]
pub client_id: String,
Expand Down Expand Up @@ -50,6 +53,7 @@ impl MicrosoftSentinelDestination {
type_: crate::datadogV2::model::MicrosoftSentinelDestinationType,
) -> MicrosoftSentinelDestination {
MicrosoftSentinelDestination {
buffer: None,
client_id,
dcr_immutable_id,
id,
Expand All @@ -62,6 +66,14 @@ impl MicrosoftSentinelDestination {
}
}

pub fn buffer(
mut self,
value: crate::datadogV2::model::ObservabilityPipelineBufferOptions,
) -> Self {
self.buffer = Some(value);
self
}

pub fn additional_properties(
mut self,
value: std::collections::BTreeMap<String, serde_json::Value>,
Expand All @@ -88,6 +100,9 @@ impl<'de> Deserialize<'de> for MicrosoftSentinelDestination {
where
M: MapAccess<'a>,
{
let mut buffer: Option<
crate::datadogV2::model::ObservabilityPipelineBufferOptions,
> = None;
let mut client_id: Option<String> = None;
let mut dcr_immutable_id: Option<String> = None;
let mut id: Option<String> = None;
Expand All @@ -104,6 +119,20 @@ impl<'de> Deserialize<'de> for MicrosoftSentinelDestination {

while let Some((k, v)) = map.next_entry::<String, serde_json::Value>()? {
match k.as_str() {
"buffer" => {
if v.is_null() {
continue;
}
buffer = Some(serde_json::from_value(v).map_err(M::Error::custom)?);
if let Some(ref _buffer) = buffer {
match _buffer {
crate::datadogV2::model::ObservabilityPipelineBufferOptions::UnparsedObject(_buffer) => {
_unparsed = true;
},
_ => {}
}
}
}
"client_id" => {
client_id = Some(serde_json::from_value(v).map_err(M::Error::custom)?);
}
Expand Down Expand Up @@ -151,6 +180,7 @@ impl<'de> Deserialize<'de> for MicrosoftSentinelDestination {
let type_ = type_.ok_or_else(|| M::Error::missing_field("type_"))?;

let content = MicrosoftSentinelDestination {
buffer,
client_id,
dcr_immutable_id,
id,
Expand Down
Loading
Loading