diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index c1ace36c38..54628dbe57 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -43045,12 +43045,14 @@ components: ObservabilityPipelineAmazonS3Source: description: |- The `amazon_s3` source ingests logs from an Amazon S3 bucket. - It supports AWS authentication and TLS encryption. + It supports AWS authentication, TLS encryption, and configurable compression. **Supported pipeline types:** logs properties: auth: $ref: "#/components/schemas/ObservabilityPipelineAwsAuth" + compression: + $ref: "#/components/schemas/ObservabilityPipelineAmazonS3SourceCompression" id: description: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components). example: aws-s3-source @@ -43073,6 +43075,20 @@ components: - region type: object x-pipeline-types: [logs] + ObservabilityPipelineAmazonS3SourceCompression: + description: Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension. + enum: + - auto + - none + - gzip + - zstd + example: gzip + type: string + x-enum-varnames: + - AUTO + - NONE + - GZIP + - ZSTD ObservabilityPipelineAmazonS3SourceType: default: amazon_s3 description: The source type. Always `amazon_s3`. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index 19375e6c96..2a2b3b682c 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -19198,6 +19198,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_amazon\_s3\_source module :members: :show-inheritance: +datadog\_api\_client.v2.model.observability\_pipeline\_amazon\_s3\_source\_compression module +--------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_compression + :members: + :show-inheritance: + datadog\_api\_client.v2.model.observability\_pipeline\_amazon\_s3\_source\_type module -------------------------------------------------------------------------------------- diff --git a/examples/v2/observability-pipelines/ValidatePipeline_99164570.py b/examples/v2/observability-pipelines/ValidatePipeline_99164570.py new file mode 100644 index 0000000000..810cc75672 --- /dev/null +++ b/examples/v2/observability-pipelines/ValidatePipeline_99164570.py @@ -0,0 +1,83 @@ +""" +Validate an observability pipeline with amazon S3 source compression returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source import ObservabilityPipelineAmazonS3Source +from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_compression import ( + ObservabilityPipelineAmazonS3SourceCompression, +) +from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_type import ( + ObservabilityPipelineAmazonS3SourceType, +) +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import ( + ObservabilityPipelineDatadogLogsDestination, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import ( + ObservabilityPipelineDatadogLogsDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor +from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import ( + ObservabilityPipelineFilterProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineDatadogLogsDestination( + id="datadog-logs-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "amazon-s3-source", + ], + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="service:my-service", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], + ), + ], + sources=[ + ObservabilityPipelineAmazonS3Source( + id="amazon-s3-source", + type=ObservabilityPipelineAmazonS3SourceType.AMAZON_S3, + region="us-east-1", + compression=ObservabilityPipelineAmazonS3SourceCompression.GZIP, + ), + ], + ), + name="Pipeline with S3 Source Compression", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.validate_pipeline(body=body) + + print(response) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source.py b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source.py index 49388ad221..c67e590837 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source.py @@ -15,6 +15,9 @@ if TYPE_CHECKING: from datadog_api_client.v2.model.observability_pipeline_aws_auth import ObservabilityPipelineAwsAuth + from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_compression import ( + ObservabilityPipelineAmazonS3SourceCompression, + ) from datadog_api_client.v2.model.observability_pipeline_tls import ObservabilityPipelineTls from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_type import ( ObservabilityPipelineAmazonS3SourceType, @@ -25,6 +28,9 @@ class ObservabilityPipelineAmazonS3Source(ModelNormal): @cached_property def openapi_types(_): from datadog_api_client.v2.model.observability_pipeline_aws_auth import ObservabilityPipelineAwsAuth + from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_compression import ( + ObservabilityPipelineAmazonS3SourceCompression, + ) from datadog_api_client.v2.model.observability_pipeline_tls import ObservabilityPipelineTls from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_type import ( ObservabilityPipelineAmazonS3SourceType, @@ -32,6 +38,7 @@ def openapi_types(_): return { "auth": (ObservabilityPipelineAwsAuth,), + "compression": (ObservabilityPipelineAmazonS3SourceCompression,), "id": (str,), "region": (str,), "tls": (ObservabilityPipelineTls,), @@ -41,6 +48,7 @@ def openapi_types(_): attribute_map = { "auth": "auth", + "compression": "compression", "id": "id", "region": "region", "tls": "tls", @@ -54,13 +62,14 @@ def __init__( region: str, type: ObservabilityPipelineAmazonS3SourceType, auth: Union[ObservabilityPipelineAwsAuth, UnsetType] = unset, + compression: Union[ObservabilityPipelineAmazonS3SourceCompression, UnsetType] = unset, tls: Union[ObservabilityPipelineTls, UnsetType] = unset, url_key: Union[str, UnsetType] = unset, **kwargs, ): """ The ``amazon_s3`` source ingests logs from an Amazon S3 bucket. - It supports AWS authentication and TLS encryption. + It supports AWS authentication, TLS encryption, and configurable compression. **Supported pipeline types:** logs @@ -68,6 +77,9 @@ def __init__( If omitted, the system’s default credentials are used (for example, the IAM role and environment variables). :type auth: ObservabilityPipelineAwsAuth, optional + :param compression: Compression format for objects retrieved from the S3 bucket. Use ``auto`` to detect compression from the object's Content-Encoding header or file extension. + :type compression: ObservabilityPipelineAmazonS3SourceCompression, optional + :param id: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the ``input`` to downstream components). :type id: str @@ -85,6 +97,8 @@ def __init__( """ if auth is not unset: kwargs["auth"] = auth + if compression is not unset: + kwargs["compression"] = compression if tls is not unset: kwargs["tls"] = tls if url_key is not unset: diff --git a/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source_compression.py b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source_compression.py new file mode 100644 index 0000000000..ad39c69c53 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_source_compression.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineAmazonS3SourceCompression(ModelSimple): + """ + Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension. + + :param value: Must be one of ["auto", "none", "gzip", "zstd"]. + :type value: str + """ + + allowed_values = { + "auto", + "none", + "gzip", + "zstd", + } + AUTO: ClassVar["ObservabilityPipelineAmazonS3SourceCompression"] + NONE: ClassVar["ObservabilityPipelineAmazonS3SourceCompression"] + GZIP: ClassVar["ObservabilityPipelineAmazonS3SourceCompression"] + ZSTD: ClassVar["ObservabilityPipelineAmazonS3SourceCompression"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineAmazonS3SourceCompression.AUTO = ObservabilityPipelineAmazonS3SourceCompression("auto") +ObservabilityPipelineAmazonS3SourceCompression.NONE = ObservabilityPipelineAmazonS3SourceCompression("none") +ObservabilityPipelineAmazonS3SourceCompression.GZIP = ObservabilityPipelineAmazonS3SourceCompression("gzip") +ObservabilityPipelineAmazonS3SourceCompression.ZSTD = ObservabilityPipelineAmazonS3SourceCompression("zstd") diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config_source_item.py b/src/datadog_api_client/v2/model/observability_pipeline_config_source_item.py index 46a3d03c8d..58cab2a635 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config_source_item.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config_source_item.py @@ -31,6 +31,9 @@ def __init__(self, **kwargs): If omitted, the system’s default credentials are used (for example, the IAM role and environment variables). :type auth: ObservabilityPipelineAwsAuth, optional + :param compression: Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension. + :type compression: ObservabilityPipelineAmazonS3SourceCompression, optional + :param region: AWS region where the S3 bucket resides. :type region: str diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index bf808b8c20..5509fa1a98 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3675,6 +3675,9 @@ ObservabilityPipelineAmazonS3GenericEncodingParquetType, ) from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source import ObservabilityPipelineAmazonS3Source +from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_compression import ( + ObservabilityPipelineAmazonS3SourceCompression, +) from datadog_api_client.v2.model.observability_pipeline_amazon_s3_source_type import ( ObservabilityPipelineAmazonS3SourceType, ) @@ -9568,6 +9571,7 @@ "ObservabilityPipelineAmazonS3GenericEncodingParquet", "ObservabilityPipelineAmazonS3GenericEncodingParquetType", "ObservabilityPipelineAmazonS3Source", + "ObservabilityPipelineAmazonS3SourceCompression", "ObservabilityPipelineAmazonS3SourceType", "ObservabilityPipelineAmazonSecurityLakeDestination", "ObservabilityPipelineAmazonSecurityLakeDestinationType", diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.frozen new file mode 100644 index 0000000000..642f43ba78 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-04-08T12:44:25.060Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.yaml new file mode 100644 index 0000000000..f5d14d502d --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_amazon_s3_source_compression_returns_ok_response.yaml @@ -0,0 +1,23 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["amazon-s3-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"service:my-service","type":"filter"}]}],"sources":[{"compression":"gzip","id":"amazon-s3-source","region":"us-east-1","type":"amazon_s3"}]},"name":"Pipeline + with S3 Source Compression"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate + response: + body: + string: '{"errors":[]} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v2/features/observability_pipelines.feature b/tests/v2/features/observability_pipelines.feature index 7e8ceefc82..96a0bbc6cb 100644 --- a/tests/v2/features/observability_pipelines.feature +++ b/tests/v2/features/observability_pipelines.feature @@ -207,6 +207,14 @@ Feature: Observability Pipelines Then the response status is 200 OK And the response "errors" has length 0 + @team:DataDog/observability-pipelines + Scenario: Validate an observability pipeline with amazon S3 source compression returns "OK" response + Given new "ValidatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["amazon-s3-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "service:my-service", "type": "filter"}]}], "sources": [{"id": "amazon-s3-source", "type": "amazon_s3", "region": "us-east-1", "compression": "gzip"}]}, "name": "Pipeline with S3 Source Compression"}, "type": "pipelines"}} + When the request is sent + Then the response status is 200 OK + And the response "errors" has length 0 + @team:DataDog/observability-pipelines Scenario: Validate an observability pipeline with destination secret key returns "OK" response Given new "ValidatePipeline" request