From 95cb20e9d3487dcc4d4db961b1414df3dbd66214 Mon Sep 17 00:00:00 2001 From: "ci.datadog-api-spec" Date: Thu, 9 Apr 2026 07:26:55 +0000 Subject: [PATCH] Regenerate client from commit 5921132 of spec repo --- .generator/schemas/v2/openapi.yaml | 787 +++++++++++++++++- docs/datadog_api_client.v2.model.rst | 210 +++++ .../CreateLLMObsAnnotationQueue.py | 31 + ...CreateLLMObsAnnotationQueueInteractions.py | 40 + .../DeleteLLMObsAnnotationQueue.py | 14 + ...DeleteLLMObsAnnotationQueueInteractions.py | 34 + .../GetLLMObsAnnotatedInteractions.py | 16 + .../ListLLMObsAnnotationQueues.py | 14 + .../UpdateLLMObsAnnotationQueue.py | 32 + src/datadog_api_client/configuration.py | 7 + .../v2/api/llm_observability_api.py | 335 +++++++- .../llm_obs_annotated_interaction_item.py | 62 ++ ...d_interactions_data_attributes_response.py | 40 + ...bs_annotated_interactions_data_response.py | 64 ++ ...llm_obs_annotated_interactions_response.py | 44 + .../llm_obs_annotated_interactions_type.py | 35 + .../v2/model/llm_obs_annotation_item.py | 99 +++ ...nnotation_queue_data_attributes_request.py | 49 ++ ...notation_queue_data_attributes_response.py | 87 ++ .../llm_obs_annotation_queue_data_request.py | 54 ++ .../llm_obs_annotation_queue_data_response.py | 64 ++ ...m_obs_annotation_queue_interaction_item.py | 46 + ...otation_queue_interaction_response_item.py | 58 ++ ...ue_interactions_data_attributes_request.py | 50 ++ ...e_interactions_data_attributes_response.py | 44 + ...otation_queue_interactions_data_request.py | 61 ++ ...tation_queue_interactions_data_response.py | 68 ++ ...s_annotation_queue_interactions_request.py | 44 + ..._annotation_queue_interactions_response.py | 44 + ..._obs_annotation_queue_interactions_type.py | 35 + .../model/llm_obs_annotation_queue_request.py | 40 + .../llm_obs_annotation_queue_response.py | 40 + .../v2/model/llm_obs_annotation_queue_type.py | 35 + ...on_queue_update_data_attributes_request.py | 43 + ...bs_annotation_queue_update_data_request.py | 54 ++ ...llm_obs_annotation_queue_update_request.py | 44 + .../llm_obs_annotation_queues_response.py | 40 + ...ue_interactions_data_attributes_request.py | 40 + ...otation_queue_interactions_data_request.py | 61 ++ ...e_annotation_queue_interactions_request.py | 44 + .../v2/model/llm_obs_interaction_type.py | 38 + src/datadog_api_client/v2/models/__init__.py | 92 ++ tests/v2/features/llm_observability.feature | 155 +++- tests/v2/features/undo.json | 60 ++ 44 files changed, 3349 insertions(+), 5 deletions(-) create mode 100644 examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py create mode 100644 examples/v2/llm-observability/CreateLLMObsAnnotationQueueInteractions.py create mode 100644 examples/v2/llm-observability/DeleteLLMObsAnnotationQueue.py create mode 100644 examples/v2/llm-observability/DeleteLLMObsAnnotationQueueInteractions.py create mode 100644 examples/v2/llm-observability/GetLLMObsAnnotatedInteractions.py create mode 100644 examples/v2/llm-observability/ListLLMObsAnnotationQueues.py create mode 100644 examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotated_interaction_item.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_attributes_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotated_interactions_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotated_interactions_type.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_item.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_item.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_response_item.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_type.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_type.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_annotation_queues_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_attributes_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_interaction_type.py diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index d235bdcd3d..e632b11bed 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -758,6 +758,14 @@ components: schema: example: "my-job" type: string + LLMObsAnnotationQueueIDPathParameter: + description: The ID of the LLM Observability annotation queue. + example: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + in: path + name: queue_id + required: true + schema: + type: string LLMObsDatasetIDPathParameter: description: The ID of the LLM Observability dataset. example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" @@ -36063,6 +36071,386 @@ components: format: int64 type: integer type: object + LLMObsAnnotatedInteractionItem: + description: An interaction with its associated annotations. + properties: + annotations: + description: List of annotations for this interaction. + items: + $ref: "#/components/schemas/LLMObsAnnotationItem" + type: array + content_id: + description: Identifier of the content for this interaction. + example: "trace-abc-123" + type: string + id: + description: Unique identifier of the interaction. + example: "interaction-456" + type: string + type: + $ref: "#/components/schemas/LLMObsInteractionType" + required: + - id + - type + - content_id + - annotations + type: object + LLMObsAnnotatedInteractionsDataAttributesResponse: + description: Attributes containing the list of annotated interactions. + properties: + annotated_interactions: + description: List of interactions with their annotations. + items: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionItem" + type: array + required: + - annotated_interactions + type: object + LLMObsAnnotatedInteractionsDataResponse: + description: Data object for annotated interactions. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsDataAttributesResponse" + id: + description: The queue ID. + example: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + type: string + type: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsType" + required: + - id + - type + - attributes + type: object + LLMObsAnnotatedInteractionsResponse: + description: Response containing the annotated interactions for an annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsDataResponse" + required: + - data + type: object + LLMObsAnnotatedInteractionsType: + description: Resource type for annotated interactions. + enum: + - annotated_interactions + example: annotated_interactions + type: string + x-enum-varnames: + - ANNOTATED_INTERACTIONS + LLMObsAnnotationItem: + description: A single annotation on an interaction. + properties: + created_at: + description: Timestamp when the annotation was created. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + created_by: + description: Identifier of the user who created the annotation. + example: "00000000-0000-0000-0000-000000000002" + type: string + id: + description: Unique identifier of the annotation. + example: "annotation-789" + type: string + interaction_id: + description: Identifier of the interaction this annotation belongs to. + example: "interaction-456" + type: string + label_values: + additionalProperties: {} + description: The label values for this annotation. + example: + quality: "good" + type: object + modified_at: + description: Timestamp when the annotation was last modified. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + modified_by: + description: Identifier of the user who last modified the annotation. + example: "00000000-0000-0000-0000-000000000002" + type: string + required: + - id + - interaction_id + - label_values + - created_by + - created_at + - modified_by + - modified_at + type: object + LLMObsAnnotationQueueDataAttributesRequest: + description: Attributes for creating an LLM Observability annotation queue. + properties: + description: + description: Description of the annotation queue. + example: "Queue for annotating customer support traces" + type: string + name: + description: Name of the annotation queue. + example: "My annotation queue" + type: string + project_id: + description: Identifier of the project this queue belongs to. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + type: string + required: + - name + - project_id + type: object + LLMObsAnnotationQueueDataAttributesResponse: + description: Attributes of an LLM Observability annotation queue. + properties: + created_at: + description: Timestamp when the queue was created. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + created_by: + description: Identifier of the user who created the queue. + example: "00000000-0000-0000-0000-000000000002" + type: string + description: + description: Description of the annotation queue. + example: "Queue for annotating customer support traces" + type: string + modified_at: + description: Timestamp when the queue was last modified. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + modified_by: + description: Identifier of the user who last modified the queue. + example: "00000000-0000-0000-0000-000000000002" + type: string + name: + description: Name of the annotation queue. + example: "My annotation queue" + type: string + owned_by: + description: Identifier of the user who owns the queue. + example: "00000000-0000-0000-0000-000000000002" + type: string + project_id: + description: Identifier of the project this queue belongs to. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + type: string + required: + - name + - project_id + - description + - created_by + - created_at + - modified_by + - modified_at + - owned_by + type: object + LLMObsAnnotationQueueDataRequest: + description: Data object for creating an LLM Observability annotation queue. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueDataAttributesRequest" + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueType" + required: + - type + - attributes + type: object + LLMObsAnnotationQueueDataResponse: + description: Data object for an LLM Observability annotation queue. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueDataAttributesResponse" + id: + description: Unique identifier of the annotation queue. + example: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + type: string + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueType" + required: + - id + - type + - attributes + type: object + LLMObsAnnotationQueueInteractionItem: + description: A single interaction to add to an annotation queue. + properties: + content_id: + description: Identifier of the content (such as trace ID) for this interaction. + example: "trace-abc-123" + type: string + type: + $ref: "#/components/schemas/LLMObsInteractionType" + required: + - type + - content_id + type: object + LLMObsAnnotationQueueInteractionResponseItem: + description: A single interaction result. + properties: + already_existed: + description: Whether this interaction already existed in the queue. + example: false + type: boolean + content_id: + description: Identifier of the content for this interaction. + example: "trace-abc-123" + type: string + id: + description: Unique identifier of the interaction. + example: "00000000-0000-0000-0000-000000000000" + type: string + type: + $ref: "#/components/schemas/LLMObsInteractionType" + required: + - id + - type + - content_id + - already_existed + type: object + LLMObsAnnotationQueueInteractionsDataAttributesRequest: + description: Attributes for adding interactions to an annotation queue. + properties: + interactions: + description: List of interactions to add to the queue. Must contain at least one item. + items: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionItem" + minItems: 1 + type: array + required: + - interactions + type: object + LLMObsAnnotationQueueInteractionsDataAttributesResponse: + description: Attributes of the interaction addition response. + properties: + interactions: + description: List of interactions that were processed. + items: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionResponseItem" + type: array + required: + - interactions + type: object + LLMObsAnnotationQueueInteractionsDataRequest: + description: Data object for adding interactions to an annotation queue. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsDataAttributesRequest" + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsType" + required: + - type + - attributes + type: object + LLMObsAnnotationQueueInteractionsDataResponse: + description: Data object for the interaction addition response. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsDataAttributesResponse" + id: + description: The queue ID the interactions were added to. + example: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + type: string + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsType" + required: + - id + - type + - attributes + type: object + LLMObsAnnotationQueueInteractionsRequest: + description: Request to add interactions to an LLM Observability annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsDataRequest" + required: + - data + type: object + LLMObsAnnotationQueueInteractionsResponse: + description: Response containing the result of adding interactions to an annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsDataResponse" + required: + - data + type: object + LLMObsAnnotationQueueInteractionsType: + description: Resource type for annotation queue interactions. + enum: + - interactions + example: interactions + type: string + x-enum-varnames: + - INTERACTIONS + LLMObsAnnotationQueueRequest: + description: Request to create an LLM Observability annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueDataRequest" + required: + - data + type: object + LLMObsAnnotationQueueResponse: + description: Response containing a single LLM Observability annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueDataResponse" + required: + - data + type: object + LLMObsAnnotationQueueType: + description: Resource type of an LLM Observability annotation queue. + enum: + - queues + example: queues + type: string + x-enum-varnames: + - QUEUES + LLMObsAnnotationQueueUpdateDataAttributesRequest: + description: Attributes for updating an LLM Observability annotation queue. All fields are optional. + properties: + description: + description: Updated description of the annotation queue. + example: "Updated description" + type: string + name: + description: Updated name of the annotation queue. + example: "Updated queue name" + type: string + type: object + LLMObsAnnotationQueueUpdateDataRequest: + description: Data object for updating an LLM Observability annotation queue. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueUpdateDataAttributesRequest" + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueType" + required: + - type + - attributes + type: object + LLMObsAnnotationQueueUpdateRequest: + description: Request to update an LLM Observability annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueUpdateDataRequest" + required: + - data + type: object + LLMObsAnnotationQueuesResponse: + description: Response containing a list of LLM Observability annotation queues. + properties: + data: + description: List of annotation queues. + items: + $ref: "#/components/schemas/LLMObsAnnotationQueueDataResponse" + type: array + required: + - data + type: object LLMObsCursorMeta: description: Pagination cursor metadata. properties: @@ -36394,6 +36782,41 @@ components: required: - data type: object + LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest: + description: Attributes for deleting interactions from an annotation queue. + properties: + interaction_ids: + description: List of interaction IDs to delete. Must contain at least one item. + example: + - "00000000-0000-0000-0000-000000000000" + - "00000000-0000-0000-0000-000000000001" + items: + description: An interaction ID to delete. + type: string + minItems: 1 + type: array + required: + - interaction_ids + type: object + LLMObsDeleteAnnotationQueueInteractionsDataRequest: + description: Data object for deleting interactions from an annotation queue. + properties: + attributes: + $ref: "#/components/schemas/LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest" + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsType" + required: + - type + - attributes + type: object + LLMObsDeleteAnnotationQueueInteractionsRequest: + description: Request to delete interactions from an LLM Observability annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsDeleteAnnotationQueueInteractionsDataRequest" + required: + - data + type: object LLMObsDeleteDatasetRecordsDataAttributesRequest: description: Attributes for deleting records from an LLM Observability dataset. properties: @@ -36903,6 +37326,16 @@ components: required: - data type: object + LLMObsInteractionType: + description: Type of interaction in an annotation queue. + enum: + - trace + - experiment_trace + example: trace + type: string + x-enum-varnames: + - TRACE + - EXPERIMENT_TRACE LLMObsMetricAssessment: description: Assessment result for an LLM Observability experiment metric. enum: @@ -92366,6 +92799,358 @@ paths: operator: OR permissions: - org_management + /api/v2/llm-obs/v1/annotation-queues: + get: + description: |- + List annotation queues. Optionally filter by project ID or queue IDs. These parameters are mutually exclusive. + If neither is provided, all queues in the organization are returned. + operationId: ListLLMObsAnnotationQueues + parameters: + - description: Filter annotation queues by project ID. Cannot be used together with `queueIds`. + in: query + name: projectId + schema: + type: string + - description: Filter annotation queues by queue IDs (comma-separated). Cannot be used together with `projectId`. + in: query + name: queueIds + schema: + items: + type: string + type: array + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueuesResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability annotation queues + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + post: + description: |- + Create a new annotation queue. Only `name`, `project_id`, and `description` are accepted. + Fields such as `created_by`, `owned_by`, `created_at`, `modified_by`, and `modified_at` are inferred by the backend. + operationId: CreateLLMObsAnnotationQueue + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueRequest" + description: Create annotation queue payload. + required: true + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueResponse" + description: Created + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Create an LLM Observability annotation queue + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotation-queues/{queue_id}: + delete: + description: Delete an annotation queue by its ID. + operationId: DeleteLLMObsAnnotationQueue + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + responses: + "204": + description: No Content + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete an LLM Observability annotation queue + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + patch: + description: |- + Partially update an annotation queue. Only `name` and `description` can be updated. + operationId: UpdateLLMObsAnnotationQueue + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueUpdateRequest" + description: Update annotation queue payload. + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Update an LLM Observability annotation queue + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotation-queues/{queue_id}/annotated-interactions: + get: + description: Retrieve all interactions and their annotations for a given annotation queue. + operationId: GetLLMObsAnnotatedInteractions + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get annotated queue interactions + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotation-queues/{queue_id}/interactions: + post: + description: |- + Add one or more interactions (traces) to an annotation queue. + At least one interaction must be provided. + operationId: CreateLLMObsAnnotationQueueInteractions + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsRequest" + description: Add interactions payload. + required: true + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueInteractionsResponse" + description: Created + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Add annotation queue interactions + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotation-queues/{queue_id}/interactions/delete: + post: + description: Delete one or more interactions from an annotation queue. + operationId: DeleteLLMObsAnnotationQueueInteractions + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsDeleteAnnotationQueueInteractionsRequest" + description: Delete interactions payload. + required: true + responses: + "204": + description: No Content + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete annotation queue interactions + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/experiments: get: description: List all LLM Observability experiments sorted by creation date, newest first. @@ -115735,7 +116520,7 @@ tags: description: Find out more at url: "https://docs.datadoghq.com/account_management/api-app-keys/" name: Key Management - - description: Manage LLM Observability projects, datasets, dataset records, and experiments via the Experiments API. + - description: Manage LLM Observability projects, datasets, dataset records, experiments, and annotations. name: LLM Observability - description: |- Search your logs and send them to your Datadog platform over HTTP. See the [Log Management page](https://docs.datadoghq.com/logs/) for more information. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index e714684121..62f7073257 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -16230,6 +16230,188 @@ datadog\_api\_client.v2.model.list\_vulnerable\_assets\_response module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_annotated\_interaction\_item module +--------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotated_interaction_item + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotated\_interactions\_data\_attributes\_response module +-------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotated_interactions_data_attributes_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotated\_interactions\_data\_response module +-------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotated_interactions_data_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotated\_interactions\_response module +-------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotated_interactions_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotated\_interactions\_type module +---------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotated_interactions_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_item module +--------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_item + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_data\_attributes\_request module +------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_data\_attributes\_response module +-------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_data\_request module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_data_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_data\_response module +-------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_data_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interaction\_item module +----------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_item + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interaction\_response\_item module +--------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_response_item + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_data\_attributes\_request module +--------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_data\_attributes\_response module +---------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_data\_request module +--------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_data\_response module +---------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_request module +--------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_response module +---------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_type module +------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_request module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_response module +-------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_type module +---------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_update\_data\_attributes\_request module +--------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_attributes_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_update\_data\_request module +--------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_update\_request module +--------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_update_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queues\_response module +--------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queues_response + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_cursor\_meta module ----------------------------------------------------------- @@ -16398,6 +16580,27 @@ datadog\_api\_client.v2.model.llm\_obs\_datasets\_response module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_delete\_annotation\_queue\_interactions\_data\_attributes\_request module +----------------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_attributes_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_delete\_annotation\_queue\_interactions\_data\_request module +----------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_delete\_annotation\_queue\_interactions\_request module +----------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_request + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_delete\_dataset\_records\_data\_attributes\_request module -------------------------------------------------------------------------------------------------- @@ -16629,6 +16832,13 @@ datadog\_api\_client.v2.model.llm\_obs\_experiments\_response module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_interaction\_type module +---------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_interaction_type + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_metric\_assessment module ----------------------------------------------------------------- diff --git a/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py b/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py new file mode 100644 index 0000000000..33d165615d --- /dev/null +++ b/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py @@ -0,0 +1,31 @@ +""" +Create an LLM Observability annotation queue returns "Created" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_request import ( + LLMObsAnnotationQueueDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_request import LLMObsAnnotationQueueDataRequest +from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest +from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + +body = LLMObsAnnotationQueueRequest( + data=LLMObsAnnotationQueueDataRequest( + attributes=LLMObsAnnotationQueueDataAttributesRequest( + description="Queue for annotating customer support traces", + name="My annotation queue", + project_id="a33671aa-24fd-4dcd-9b33-a8ec7dde7751", + ), + type=LLMObsAnnotationQueueType.QUEUES, + ), +) + +configuration = Configuration() +configuration.unstable_operations["create_llm_obs_annotation_queue"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.create_llm_obs_annotation_queue(body=body) + + print(response) diff --git a/examples/v2/llm-observability/CreateLLMObsAnnotationQueueInteractions.py b/examples/v2/llm-observability/CreateLLMObsAnnotationQueueInteractions.py new file mode 100644 index 0000000000..3c7b8a2359 --- /dev/null +++ b/examples/v2/llm-observability/CreateLLMObsAnnotationQueueInteractions.py @@ -0,0 +1,40 @@ +""" +Add annotation queue interactions returns "Created" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_item import LLMObsAnnotationQueueInteractionItem +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_request import ( + LLMObsAnnotationQueueInteractionsDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_request import ( + LLMObsAnnotationQueueInteractionsDataRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_request import ( + LLMObsAnnotationQueueInteractionsRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import LLMObsAnnotationQueueInteractionsType +from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + +body = LLMObsAnnotationQueueInteractionsRequest( + data=LLMObsAnnotationQueueInteractionsDataRequest( + attributes=LLMObsAnnotationQueueInteractionsDataAttributesRequest( + interactions=[ + LLMObsAnnotationQueueInteractionItem( + content_id="trace-abc-123", + type=LLMObsInteractionType.TRACE, + ), + ], + ), + type=LLMObsAnnotationQueueInteractionsType.INTERACTIONS, + ), +) + +configuration = Configuration() +configuration.unstable_operations["create_llm_obs_annotation_queue_interactions"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.create_llm_obs_annotation_queue_interactions(queue_id="queue_id", body=body) + + print(response) diff --git a/examples/v2/llm-observability/DeleteLLMObsAnnotationQueue.py b/examples/v2/llm-observability/DeleteLLMObsAnnotationQueue.py new file mode 100644 index 0000000000..b75ab5773b --- /dev/null +++ b/examples/v2/llm-observability/DeleteLLMObsAnnotationQueue.py @@ -0,0 +1,14 @@ +""" +Delete an LLM Observability annotation queue returns "No Content" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["delete_llm_obs_annotation_queue"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + api_instance.delete_llm_obs_annotation_queue( + queue_id="queue_id", + ) diff --git a/examples/v2/llm-observability/DeleteLLMObsAnnotationQueueInteractions.py b/examples/v2/llm-observability/DeleteLLMObsAnnotationQueueInteractions.py new file mode 100644 index 0000000000..7cda3e399f --- /dev/null +++ b/examples/v2/llm-observability/DeleteLLMObsAnnotationQueueInteractions.py @@ -0,0 +1,34 @@ +""" +Delete annotation queue interactions returns "No Content" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import LLMObsAnnotationQueueInteractionsType +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_attributes_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataRequest, +) +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_request import ( + LLMObsDeleteAnnotationQueueInteractionsRequest, +) + +body = LLMObsDeleteAnnotationQueueInteractionsRequest( + data=LLMObsDeleteAnnotationQueueInteractionsDataRequest( + attributes=LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest( + interaction_ids=[ + "00000000-0000-0000-0000-000000000000", + "00000000-0000-0000-0000-000000000001", + ], + ), + type=LLMObsAnnotationQueueInteractionsType.INTERACTIONS, + ), +) + +configuration = Configuration() +configuration.unstable_operations["delete_llm_obs_annotation_queue_interactions"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + api_instance.delete_llm_obs_annotation_queue_interactions(queue_id="queue_id", body=body) diff --git a/examples/v2/llm-observability/GetLLMObsAnnotatedInteractions.py b/examples/v2/llm-observability/GetLLMObsAnnotatedInteractions.py new file mode 100644 index 0000000000..c74b7155a1 --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsAnnotatedInteractions.py @@ -0,0 +1,16 @@ +""" +Get annotated queue interactions returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["get_llm_obs_annotated_interactions"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.get_llm_obs_annotated_interactions( + queue_id="queue_id", + ) + + print(response) diff --git a/examples/v2/llm-observability/ListLLMObsAnnotationQueues.py b/examples/v2/llm-observability/ListLLMObsAnnotationQueues.py new file mode 100644 index 0000000000..50ec5808c7 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsAnnotationQueues.py @@ -0,0 +1,14 @@ +""" +List LLM Observability annotation queues returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["list_llm_obs_annotation_queues"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.list_llm_obs_annotation_queues() + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py new file mode 100644 index 0000000000..95e5b7a195 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py @@ -0,0 +1,32 @@ +""" +Update an LLM Observability annotation queue returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_attributes_request import ( + LLMObsAnnotationQueueUpdateDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_request import ( + LLMObsAnnotationQueueUpdateDataRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest + +body = LLMObsAnnotationQueueUpdateRequest( + data=LLMObsAnnotationQueueUpdateDataRequest( + attributes=LLMObsAnnotationQueueUpdateDataAttributesRequest( + description="Updated description", + name="Updated queue name", + ), + type=LLMObsAnnotationQueueType.QUEUES, + ), +) + +configuration = Configuration() +configuration.unstable_operations["update_llm_obs_annotation_queue"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.update_llm_obs_annotation_queue(queue_id="queue_id", body=body) + + print(response) diff --git a/src/datadog_api_client/configuration.py b/src/datadog_api_client/configuration.py index 84c9ad4641..05dd564455 100644 --- a/src/datadog_api_client/configuration.py +++ b/src/datadog_api_client/configuration.py @@ -400,19 +400,26 @@ def __init__( "v2.update_jira_issue_template": False, "v2.create_tenancy_config": False, "v2.get_tenancy_configs": False, + "v2.create_llm_obs_annotation_queue": False, + "v2.create_llm_obs_annotation_queue_interactions": False, "v2.create_llm_obs_dataset": False, "v2.create_llm_obs_dataset_records": False, "v2.create_llm_obs_experiment": False, "v2.create_llm_obs_experiment_events": False, "v2.create_llm_obs_project": False, + "v2.delete_llm_obs_annotation_queue": False, + "v2.delete_llm_obs_annotation_queue_interactions": False, "v2.delete_llm_obs_dataset_records": False, "v2.delete_llm_obs_datasets": False, "v2.delete_llm_obs_experiments": False, "v2.delete_llm_obs_projects": False, + "v2.get_llm_obs_annotated_interactions": False, + "v2.list_llm_obs_annotation_queues": False, "v2.list_llm_obs_dataset_records": False, "v2.list_llm_obs_datasets": False, "v2.list_llm_obs_experiments": False, "v2.list_llm_obs_projects": False, + "v2.update_llm_obs_annotation_queue": False, "v2.update_llm_obs_dataset": False, "v2.update_llm_obs_dataset_records": False, "v2.update_llm_obs_experiment": False, diff --git a/src/datadog_api_client/v2/api/llm_observability_api.py b/src/datadog_api_client/v2/api/llm_observability_api.py index c08b95c109..15492209ce 100644 --- a/src/datadog_api_client/v2/api/llm_observability_api.py +++ b/src/datadog_api_client/v2/api/llm_observability_api.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import Any, Dict, Union +from typing import Any, Dict, List, Union from datadog_api_client.api_client import ApiClient, Endpoint as _Endpoint from datadog_api_client.configuration import Configuration @@ -11,6 +11,20 @@ UnsetType, unset, ) +from datadog_api_client.v2.model.llm_obs_annotation_queues_response import LLMObsAnnotationQueuesResponse +from datadog_api_client.v2.model.llm_obs_annotation_queue_response import LLMObsAnnotationQueueResponse +from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest +from datadog_api_client.v2.model.llm_obs_annotated_interactions_response import LLMObsAnnotatedInteractionsResponse +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_response import ( + LLMObsAnnotationQueueInteractionsResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_request import ( + LLMObsAnnotationQueueInteractionsRequest, +) +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_request import ( + LLMObsDeleteAnnotationQueueInteractionsRequest, +) from datadog_api_client.v2.model.llm_obs_experiments_response import LLMObsExperimentsResponse from datadog_api_client.v2.model.llm_obs_experiment_response import LLMObsExperimentResponse from datadog_api_client.v2.model.llm_obs_experiment_request import LLMObsExperimentRequest @@ -36,7 +50,7 @@ class LLMObservabilityApi: """ - Manage LLM Observability projects, datasets, dataset records, and experiments via the Experiments API. + Manage LLM Observability projects, datasets, dataset records, experiments, and annotations. """ def __init__(self, api_client=None): @@ -44,6 +58,52 @@ def __init__(self, api_client=None): api_client = ApiClient(Configuration()) self.api_client = api_client + self._create_llm_obs_annotation_queue_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueueResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues", + "operation_id": "create_llm_obs_annotation_queue", + "http_method": "POST", + "version": "v2", + }, + params_map={ + "body": { + "required": True, + "openapi_types": (LLMObsAnnotationQueueRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + + self._create_llm_obs_annotation_queue_interactions_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueueInteractionsResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/interactions", + "operation_id": "create_llm_obs_annotation_queue_interactions", + "http_method": "POST", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsAnnotationQueueInteractionsRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._create_llm_obs_dataset_endpoint = _Endpoint( settings={ "response_type": (LLMObsDatasetResponse,), @@ -168,6 +228,55 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._delete_llm_obs_annotation_queue_endpoint = _Endpoint( + settings={ + "response_type": None, + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}", + "operation_id": "delete_llm_obs_annotation_queue", + "http_method": "DELETE", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + }, + headers_map={ + "accept": ["*/*"], + }, + api_client=api_client, + ) + + self._delete_llm_obs_annotation_queue_interactions_endpoint = _Endpoint( + settings={ + "response_type": None, + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/interactions/delete", + "operation_id": "delete_llm_obs_annotation_queue_interactions", + "http_method": "POST", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsDeleteAnnotationQueueInteractionsRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["*/*"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._delete_llm_obs_dataset_records_endpoint = _Endpoint( settings={ "response_type": None, @@ -266,6 +375,57 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._get_llm_obs_annotated_interactions_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotatedInteractionsResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/annotated-interactions", + "operation_id": "get_llm_obs_annotated_interactions", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + + self._list_llm_obs_annotation_queues_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueuesResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues", + "operation_id": "list_llm_obs_annotation_queues", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "project_id": { + "openapi_types": (str,), + "attribute": "projectId", + "location": "query", + }, + "queue_ids": { + "openapi_types": ([str],), + "attribute": "queueIds", + "location": "query", + "collection_format": "multi", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._list_llm_obs_dataset_records_endpoint = _Endpoint( settings={ "response_type": (LLMObsDatasetRecordsListResponse,), @@ -432,6 +592,32 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._update_llm_obs_annotation_queue_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueueResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}", + "operation_id": "update_llm_obs_annotation_queue", + "http_method": "PATCH", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsAnnotationQueueUpdateRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._update_llm_obs_dataset_endpoint = _Endpoint( settings={ "response_type": (LLMObsDatasetResponse,), @@ -548,6 +734,47 @@ def __init__(self, api_client=None): api_client=api_client, ) + def create_llm_obs_annotation_queue( + self, + body: LLMObsAnnotationQueueRequest, + ) -> LLMObsAnnotationQueueResponse: + """Create an LLM Observability annotation queue. + + Create a new annotation queue. Only ``name`` , ``project_id`` , and ``description`` are accepted. + Fields such as ``created_by`` , ``owned_by`` , ``created_at`` , ``modified_by`` , and ``modified_at`` are inferred by the backend. + + :param body: Create annotation queue payload. + :type body: LLMObsAnnotationQueueRequest + :rtype: LLMObsAnnotationQueueResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["body"] = body + + return self._create_llm_obs_annotation_queue_endpoint.call_with_http_info(**kwargs) + + def create_llm_obs_annotation_queue_interactions( + self, + queue_id: str, + body: LLMObsAnnotationQueueInteractionsRequest, + ) -> LLMObsAnnotationQueueInteractionsResponse: + """Add annotation queue interactions. + + Add one or more interactions (traces) to an annotation queue. + At least one interaction must be provided. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :param body: Add interactions payload. + :type body: LLMObsAnnotationQueueInteractionsRequest + :rtype: LLMObsAnnotationQueueInteractionsResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + kwargs["body"] = body + + return self._create_llm_obs_annotation_queue_interactions_endpoint.call_with_http_info(**kwargs) + def create_llm_obs_dataset( self, project_id: str, @@ -653,6 +880,45 @@ def create_llm_obs_project( return self._create_llm_obs_project_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_annotation_queue( + self, + queue_id: str, + ) -> None: + """Delete an LLM Observability annotation queue. + + Delete an annotation queue by its ID. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :rtype: None + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + return self._delete_llm_obs_annotation_queue_endpoint.call_with_http_info(**kwargs) + + def delete_llm_obs_annotation_queue_interactions( + self, + queue_id: str, + body: LLMObsDeleteAnnotationQueueInteractionsRequest, + ) -> None: + """Delete annotation queue interactions. + + Delete one or more interactions from an annotation queue. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :param body: Delete interactions payload. + :type body: LLMObsDeleteAnnotationQueueInteractionsRequest + :rtype: None + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + kwargs["body"] = body + + return self._delete_llm_obs_annotation_queue_interactions_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_dataset_records( self, project_id: str, @@ -736,6 +1002,49 @@ def delete_llm_obs_projects( return self._delete_llm_obs_projects_endpoint.call_with_http_info(**kwargs) + def get_llm_obs_annotated_interactions( + self, + queue_id: str, + ) -> LLMObsAnnotatedInteractionsResponse: + """Get annotated queue interactions. + + Retrieve all interactions and their annotations for a given annotation queue. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :rtype: LLMObsAnnotatedInteractionsResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + return self._get_llm_obs_annotated_interactions_endpoint.call_with_http_info(**kwargs) + + def list_llm_obs_annotation_queues( + self, + *, + project_id: Union[str, UnsetType] = unset, + queue_ids: Union[List[str], UnsetType] = unset, + ) -> LLMObsAnnotationQueuesResponse: + """List LLM Observability annotation queues. + + List annotation queues. Optionally filter by project ID or queue IDs. These parameters are mutually exclusive. + If neither is provided, all queues in the organization are returned. + + :param project_id: Filter annotation queues by project ID. Cannot be used together with ``queueIds``. + :type project_id: str, optional + :param queue_ids: Filter annotation queues by queue IDs (comma-separated). Cannot be used together with ``projectId``. + :type queue_ids: [str], optional + :rtype: LLMObsAnnotationQueuesResponse + """ + kwargs: Dict[str, Any] = {} + if project_id is not unset: + kwargs["project_id"] = project_id + + if queue_ids is not unset: + kwargs["queue_ids"] = queue_ids + + return self._list_llm_obs_annotation_queues_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_dataset_records( self, project_id: str, @@ -899,6 +1208,28 @@ def list_llm_obs_projects( return self._list_llm_obs_projects_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_annotation_queue( + self, + queue_id: str, + body: LLMObsAnnotationQueueUpdateRequest, + ) -> LLMObsAnnotationQueueResponse: + """Update an LLM Observability annotation queue. + + Partially update an annotation queue. Only ``name`` and ``description`` can be updated. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :param body: Update annotation queue payload. + :type body: LLMObsAnnotationQueueUpdateRequest + :rtype: LLMObsAnnotationQueueResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + kwargs["body"] = body + + return self._update_llm_obs_annotation_queue_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_dataset( self, project_id: str, diff --git a/src/datadog_api_client/v2/model/llm_obs_annotated_interaction_item.py b/src/datadog_api_client/v2/model/llm_obs_annotated_interaction_item.py new file mode 100644 index 0000000000..02da163588 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotated_interaction_item.py @@ -0,0 +1,62 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_item import LLMObsAnnotationItem + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + +class LLMObsAnnotatedInteractionItem(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_item import LLMObsAnnotationItem + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + return { + "annotations": ([LLMObsAnnotationItem],), + "content_id": (str,), + "id": (str,), + "type": (LLMObsInteractionType,), + } + + attribute_map = { + "annotations": "annotations", + "content_id": "content_id", + "id": "id", + "type": "type", + } + + def __init__( + self_, annotations: List[LLMObsAnnotationItem], content_id: str, id: str, type: LLMObsInteractionType, **kwargs + ): + """ + An interaction with its associated annotations. + + :param annotations: List of annotations for this interaction. + :type annotations: [LLMObsAnnotationItem] + + :param content_id: Identifier of the content for this interaction. + :type content_id: str + + :param id: Unique identifier of the interaction. + :type id: str + + :param type: Type of interaction in an annotation queue. + :type type: LLMObsInteractionType + """ + super().__init__(kwargs) + + self_.annotations = annotations + self_.content_id = content_id + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_attributes_response.py b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_attributes_response.py new file mode 100644 index 0000000000..440f89dae0 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_attributes_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotated_interaction_item import LLMObsAnnotatedInteractionItem + + +class LLMObsAnnotatedInteractionsDataAttributesResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotated_interaction_item import LLMObsAnnotatedInteractionItem + + return { + "annotated_interactions": ([LLMObsAnnotatedInteractionItem],), + } + + attribute_map = { + "annotated_interactions": "annotated_interactions", + } + + def __init__(self_, annotated_interactions: List[LLMObsAnnotatedInteractionItem], **kwargs): + """ + Attributes containing the list of annotated interactions. + + :param annotated_interactions: List of interactions with their annotations. + :type annotated_interactions: [LLMObsAnnotatedInteractionItem] + """ + super().__init__(kwargs) + + self_.annotated_interactions = annotated_interactions diff --git a/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_response.py b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_response.py new file mode 100644 index 0000000000..99e25e4465 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_data_response.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_attributes_response import ( + LLMObsAnnotatedInteractionsDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotated_interactions_type import LLMObsAnnotatedInteractionsType + + +class LLMObsAnnotatedInteractionsDataResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_attributes_response import ( + LLMObsAnnotatedInteractionsDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotated_interactions_type import LLMObsAnnotatedInteractionsType + + return { + "attributes": (LLMObsAnnotatedInteractionsDataAttributesResponse,), + "id": (str,), + "type": (LLMObsAnnotatedInteractionsType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsAnnotatedInteractionsDataAttributesResponse, + id: str, + type: LLMObsAnnotatedInteractionsType, + **kwargs, + ): + """ + Data object for annotated interactions. + + :param attributes: Attributes containing the list of annotated interactions. + :type attributes: LLMObsAnnotatedInteractionsDataAttributesResponse + + :param id: The queue ID. + :type id: str + + :param type: Resource type for annotated interactions. + :type type: LLMObsAnnotatedInteractionsType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_response.py b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_response.py new file mode 100644 index 0000000000..b82a08b663 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_response.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_response import ( + LLMObsAnnotatedInteractionsDataResponse, + ) + + +class LLMObsAnnotatedInteractionsResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_response import ( + LLMObsAnnotatedInteractionsDataResponse, + ) + + return { + "data": (LLMObsAnnotatedInteractionsDataResponse,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotatedInteractionsDataResponse, **kwargs): + """ + Response containing the annotated interactions for an annotation queue. + + :param data: Data object for annotated interactions. + :type data: LLMObsAnnotatedInteractionsDataResponse + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_type.py b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_type.py new file mode 100644 index 0000000000..420a4c9855 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotated_interactions_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsAnnotatedInteractionsType(ModelSimple): + """ + Resource type for annotated interactions. + + :param value: If omitted defaults to "annotated_interactions". Must be one of ["annotated_interactions"]. + :type value: str + """ + + allowed_values = { + "annotated_interactions", + } + ANNOTATED_INTERACTIONS: ClassVar["LLMObsAnnotatedInteractionsType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsAnnotatedInteractionsType.ANNOTATED_INTERACTIONS = LLMObsAnnotatedInteractionsType("annotated_interactions") diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_item.py b/src/datadog_api_client/v2/model/llm_obs_annotation_item.py new file mode 100644 index 0000000000..b7a523fe3e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_item.py @@ -0,0 +1,99 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Any, Dict + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + date, + datetime, + none_type, + UUID, +) + + +class LLMObsAnnotationItem(ModelNormal): + @cached_property + def openapi_types(_): + return { + "created_at": (datetime,), + "created_by": (str,), + "id": (str,), + "interaction_id": (str,), + "label_values": ( + { + str: ( + bool, + date, + datetime, + dict, + float, + int, + list, + str, + UUID, + none_type, + ) + }, + ), + "modified_at": (datetime,), + "modified_by": (str,), + } + + attribute_map = { + "created_at": "created_at", + "created_by": "created_by", + "id": "id", + "interaction_id": "interaction_id", + "label_values": "label_values", + "modified_at": "modified_at", + "modified_by": "modified_by", + } + + def __init__( + self_, + created_at: datetime, + created_by: str, + id: str, + interaction_id: str, + label_values: Dict[str, Any], + modified_at: datetime, + modified_by: str, + **kwargs, + ): + """ + A single annotation on an interaction. + + :param created_at: Timestamp when the annotation was created. + :type created_at: datetime + + :param created_by: Identifier of the user who created the annotation. + :type created_by: str + + :param id: Unique identifier of the annotation. + :type id: str + + :param interaction_id: Identifier of the interaction this annotation belongs to. + :type interaction_id: str + + :param label_values: The label values for this annotation. + :type label_values: {str: (bool, date, datetime, dict, float, int, list, str, UUID, none_type,)} + + :param modified_at: Timestamp when the annotation was last modified. + :type modified_at: datetime + + :param modified_by: Identifier of the user who last modified the annotation. + :type modified_by: str + """ + super().__init__(kwargs) + + self_.created_at = created_at + self_.created_by = created_by + self_.id = id + self_.interaction_id = interaction_id + self_.label_values = label_values + self_.modified_at = modified_at + self_.modified_by = modified_by diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py new file mode 100644 index 0000000000..3edc63e44e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py @@ -0,0 +1,49 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsAnnotationQueueDataAttributesRequest(ModelNormal): + @cached_property + def openapi_types(_): + return { + "description": (str,), + "name": (str,), + "project_id": (str,), + } + + attribute_map = { + "description": "description", + "name": "name", + "project_id": "project_id", + } + + def __init__(self_, name: str, project_id: str, description: Union[str, UnsetType] = unset, **kwargs): + """ + Attributes for creating an LLM Observability annotation queue. + + :param description: Description of the annotation queue. + :type description: str, optional + + :param name: Name of the annotation queue. + :type name: str + + :param project_id: Identifier of the project this queue belongs to. + :type project_id: str + """ + if description is not unset: + kwargs["description"] = description + super().__init__(kwargs) + + self_.name = name + self_.project_id = project_id diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py new file mode 100644 index 0000000000..989cabc59d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py @@ -0,0 +1,87 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, +) + + +class LLMObsAnnotationQueueDataAttributesResponse(ModelNormal): + @cached_property + def openapi_types(_): + return { + "created_at": (datetime,), + "created_by": (str,), + "description": (str,), + "modified_at": (datetime,), + "modified_by": (str,), + "name": (str,), + "owned_by": (str,), + "project_id": (str,), + } + + attribute_map = { + "created_at": "created_at", + "created_by": "created_by", + "description": "description", + "modified_at": "modified_at", + "modified_by": "modified_by", + "name": "name", + "owned_by": "owned_by", + "project_id": "project_id", + } + + def __init__( + self_, + created_at: datetime, + created_by: str, + description: str, + modified_at: datetime, + modified_by: str, + name: str, + owned_by: str, + project_id: str, + **kwargs, + ): + """ + Attributes of an LLM Observability annotation queue. + + :param created_at: Timestamp when the queue was created. + :type created_at: datetime + + :param created_by: Identifier of the user who created the queue. + :type created_by: str + + :param description: Description of the annotation queue. + :type description: str + + :param modified_at: Timestamp when the queue was last modified. + :type modified_at: datetime + + :param modified_by: Identifier of the user who last modified the queue. + :type modified_by: str + + :param name: Name of the annotation queue. + :type name: str + + :param owned_by: Identifier of the user who owns the queue. + :type owned_by: str + + :param project_id: Identifier of the project this queue belongs to. + :type project_id: str + """ + super().__init__(kwargs) + + self_.created_at = created_at + self_.created_by = created_by + self_.description = description + self_.modified_at = modified_at + self_.modified_by = modified_by + self_.name = name + self_.owned_by = owned_by + self_.project_id = project_id diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_request.py new file mode 100644 index 0000000000..7165513efd --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_request.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_request import ( + LLMObsAnnotationQueueDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + +class LLMObsAnnotationQueueDataRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_request import ( + LLMObsAnnotationQueueDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + return { + "attributes": (LLMObsAnnotationQueueDataAttributesRequest,), + "type": (LLMObsAnnotationQueueType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsAnnotationQueueDataAttributesRequest, type: LLMObsAnnotationQueueType, **kwargs + ): + """ + Data object for creating an LLM Observability annotation queue. + + :param attributes: Attributes for creating an LLM Observability annotation queue. + :type attributes: LLMObsAnnotationQueueDataAttributesRequest + + :param type: Resource type of an LLM Observability annotation queue. + :type type: LLMObsAnnotationQueueType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_response.py new file mode 100644 index 0000000000..07c83128c1 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_response.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_response import ( + LLMObsAnnotationQueueDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + +class LLMObsAnnotationQueueDataResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_response import ( + LLMObsAnnotationQueueDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + return { + "attributes": (LLMObsAnnotationQueueDataAttributesResponse,), + "id": (str,), + "type": (LLMObsAnnotationQueueType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsAnnotationQueueDataAttributesResponse, + id: str, + type: LLMObsAnnotationQueueType, + **kwargs, + ): + """ + Data object for an LLM Observability annotation queue. + + :param attributes: Attributes of an LLM Observability annotation queue. + :type attributes: LLMObsAnnotationQueueDataAttributesResponse + + :param id: Unique identifier of the annotation queue. + :type id: str + + :param type: Resource type of an LLM Observability annotation queue. + :type type: LLMObsAnnotationQueueType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_item.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_item.py new file mode 100644 index 0000000000..84f0311147 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_item.py @@ -0,0 +1,46 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + +class LLMObsAnnotationQueueInteractionItem(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + return { + "content_id": (str,), + "type": (LLMObsInteractionType,), + } + + attribute_map = { + "content_id": "content_id", + "type": "type", + } + + def __init__(self_, content_id: str, type: LLMObsInteractionType, **kwargs): + """ + A single interaction to add to an annotation queue. + + :param content_id: Identifier of the content (such as trace ID) for this interaction. + :type content_id: str + + :param type: Type of interaction in an annotation queue. + :type type: LLMObsInteractionType + """ + super().__init__(kwargs) + + self_.content_id = content_id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_response_item.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_response_item.py new file mode 100644 index 0000000000..55cf532e6b --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interaction_response_item.py @@ -0,0 +1,58 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + +class LLMObsAnnotationQueueInteractionResponseItem(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType + + return { + "already_existed": (bool,), + "content_id": (str,), + "id": (str,), + "type": (LLMObsInteractionType,), + } + + attribute_map = { + "already_existed": "already_existed", + "content_id": "content_id", + "id": "id", + "type": "type", + } + + def __init__(self_, already_existed: bool, content_id: str, id: str, type: LLMObsInteractionType, **kwargs): + """ + A single interaction result. + + :param already_existed: Whether this interaction already existed in the queue. + :type already_existed: bool + + :param content_id: Identifier of the content for this interaction. + :type content_id: str + + :param id: Unique identifier of the interaction. + :type id: str + + :param type: Type of interaction in an annotation queue. + :type type: LLMObsInteractionType + """ + super().__init__(kwargs) + + self_.already_existed = already_existed + self_.content_id = content_id + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_request.py new file mode 100644 index 0000000000..a08968ab54 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_request.py @@ -0,0 +1,50 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_item import ( + LLMObsAnnotationQueueInteractionItem, + ) + + +class LLMObsAnnotationQueueInteractionsDataAttributesRequest(ModelNormal): + validations = { + "interactions": { + "min_items": 1, + }, + } + + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_item import ( + LLMObsAnnotationQueueInteractionItem, + ) + + return { + "interactions": ([LLMObsAnnotationQueueInteractionItem],), + } + + attribute_map = { + "interactions": "interactions", + } + + def __init__(self_, interactions: List[LLMObsAnnotationQueueInteractionItem], **kwargs): + """ + Attributes for adding interactions to an annotation queue. + + :param interactions: List of interactions to add to the queue. Must contain at least one item. + :type interactions: [LLMObsAnnotationQueueInteractionItem] + """ + super().__init__(kwargs) + + self_.interactions = interactions diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_response.py new file mode 100644 index 0000000000..9bac3d7835 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_attributes_response.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_response_item import ( + LLMObsAnnotationQueueInteractionResponseItem, + ) + + +class LLMObsAnnotationQueueInteractionsDataAttributesResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_response_item import ( + LLMObsAnnotationQueueInteractionResponseItem, + ) + + return { + "interactions": ([LLMObsAnnotationQueueInteractionResponseItem],), + } + + attribute_map = { + "interactions": "interactions", + } + + def __init__(self_, interactions: List[LLMObsAnnotationQueueInteractionResponseItem], **kwargs): + """ + Attributes of the interaction addition response. + + :param interactions: List of interactions that were processed. + :type interactions: [LLMObsAnnotationQueueInteractionResponseItem] + """ + super().__init__(kwargs) + + self_.interactions = interactions diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_request.py new file mode 100644 index 0000000000..abf70d508e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_request.py @@ -0,0 +1,61 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_request import ( + LLMObsAnnotationQueueInteractionsDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + +class LLMObsAnnotationQueueInteractionsDataRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_request import ( + LLMObsAnnotationQueueInteractionsDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + return { + "attributes": (LLMObsAnnotationQueueInteractionsDataAttributesRequest,), + "type": (LLMObsAnnotationQueueInteractionsType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsAnnotationQueueInteractionsDataAttributesRequest, + type: LLMObsAnnotationQueueInteractionsType, + **kwargs, + ): + """ + Data object for adding interactions to an annotation queue. + + :param attributes: Attributes for adding interactions to an annotation queue. + :type attributes: LLMObsAnnotationQueueInteractionsDataAttributesRequest + + :param type: Resource type for annotation queue interactions. + :type type: LLMObsAnnotationQueueInteractionsType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_response.py new file mode 100644 index 0000000000..4d23dcc924 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_data_response.py @@ -0,0 +1,68 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_response import ( + LLMObsAnnotationQueueInteractionsDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + +class LLMObsAnnotationQueueInteractionsDataResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_response import ( + LLMObsAnnotationQueueInteractionsDataAttributesResponse, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + return { + "attributes": (LLMObsAnnotationQueueInteractionsDataAttributesResponse,), + "id": (str,), + "type": (LLMObsAnnotationQueueInteractionsType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsAnnotationQueueInteractionsDataAttributesResponse, + id: str, + type: LLMObsAnnotationQueueInteractionsType, + **kwargs, + ): + """ + Data object for the interaction addition response. + + :param attributes: Attributes of the interaction addition response. + :type attributes: LLMObsAnnotationQueueInteractionsDataAttributesResponse + + :param id: The queue ID the interactions were added to. + :type id: str + + :param type: Resource type for annotation queue interactions. + :type type: LLMObsAnnotationQueueInteractionsType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_request.py new file mode 100644 index 0000000000..1ed2e570b4 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_request.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_request import ( + LLMObsAnnotationQueueInteractionsDataRequest, + ) + + +class LLMObsAnnotationQueueInteractionsRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_request import ( + LLMObsAnnotationQueueInteractionsDataRequest, + ) + + return { + "data": (LLMObsAnnotationQueueInteractionsDataRequest,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueInteractionsDataRequest, **kwargs): + """ + Request to add interactions to an LLM Observability annotation queue. + + :param data: Data object for adding interactions to an annotation queue. + :type data: LLMObsAnnotationQueueInteractionsDataRequest + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_response.py new file mode 100644 index 0000000000..13a58afea0 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_response.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_response import ( + LLMObsAnnotationQueueInteractionsDataResponse, + ) + + +class LLMObsAnnotationQueueInteractionsResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_response import ( + LLMObsAnnotationQueueInteractionsDataResponse, + ) + + return { + "data": (LLMObsAnnotationQueueInteractionsDataResponse,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueInteractionsDataResponse, **kwargs): + """ + Response containing the result of adding interactions to an annotation queue. + + :param data: Data object for the interaction addition response. + :type data: LLMObsAnnotationQueueInteractionsDataResponse + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_type.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_type.py new file mode 100644 index 0000000000..1e3e6b4c1d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_interactions_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsAnnotationQueueInteractionsType(ModelSimple): + """ + Resource type for annotation queue interactions. + + :param value: If omitted defaults to "interactions". Must be one of ["interactions"]. + :type value: str + """ + + allowed_values = { + "interactions", + } + INTERACTIONS: ClassVar["LLMObsAnnotationQueueInteractionsType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsAnnotationQueueInteractionsType.INTERACTIONS = LLMObsAnnotationQueueInteractionsType("interactions") diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_request.py new file mode 100644 index 0000000000..bed46a1330 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_request import LLMObsAnnotationQueueDataRequest + + +class LLMObsAnnotationQueueRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_request import LLMObsAnnotationQueueDataRequest + + return { + "data": (LLMObsAnnotationQueueDataRequest,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueDataRequest, **kwargs): + """ + Request to create an LLM Observability annotation queue. + + :param data: Data object for creating an LLM Observability annotation queue. + :type data: LLMObsAnnotationQueueDataRequest + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_response.py new file mode 100644 index 0000000000..cba55de938 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_response import LLMObsAnnotationQueueDataResponse + + +class LLMObsAnnotationQueueResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_response import LLMObsAnnotationQueueDataResponse + + return { + "data": (LLMObsAnnotationQueueDataResponse,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueDataResponse, **kwargs): + """ + Response containing a single LLM Observability annotation queue. + + :param data: Data object for an LLM Observability annotation queue. + :type data: LLMObsAnnotationQueueDataResponse + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_type.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_type.py new file mode 100644 index 0000000000..2a7ee76199 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsAnnotationQueueType(ModelSimple): + """ + Resource type of an LLM Observability annotation queue. + + :param value: If omitted defaults to "queues". Must be one of ["queues"]. + :type value: str + """ + + allowed_values = { + "queues", + } + QUEUES: ClassVar["LLMObsAnnotationQueueType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsAnnotationQueueType.QUEUES = LLMObsAnnotationQueueType("queues") diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py new file mode 100644 index 0000000000..92dc720e76 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py @@ -0,0 +1,43 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsAnnotationQueueUpdateDataAttributesRequest(ModelNormal): + @cached_property + def openapi_types(_): + return { + "description": (str,), + "name": (str,), + } + + attribute_map = { + "description": "description", + "name": "name", + } + + def __init__(self_, description: Union[str, UnsetType] = unset, name: Union[str, UnsetType] = unset, **kwargs): + """ + Attributes for updating an LLM Observability annotation queue. All fields are optional. + + :param description: Updated description of the annotation queue. + :type description: str, optional + + :param name: Updated name of the annotation queue. + :type name: str, optional + """ + if description is not unset: + kwargs["description"] = description + if name is not unset: + kwargs["name"] = name + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_request.py new file mode 100644 index 0000000000..67cd20381d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_request.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_attributes_request import ( + LLMObsAnnotationQueueUpdateDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + +class LLMObsAnnotationQueueUpdateDataRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_attributes_request import ( + LLMObsAnnotationQueueUpdateDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + return { + "attributes": (LLMObsAnnotationQueueUpdateDataAttributesRequest,), + "type": (LLMObsAnnotationQueueType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsAnnotationQueueUpdateDataAttributesRequest, type: LLMObsAnnotationQueueType, **kwargs + ): + """ + Data object for updating an LLM Observability annotation queue. + + :param attributes: Attributes for updating an LLM Observability annotation queue. All fields are optional. + :type attributes: LLMObsAnnotationQueueUpdateDataAttributesRequest + + :param type: Resource type of an LLM Observability annotation queue. + :type type: LLMObsAnnotationQueueType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_request.py new file mode 100644 index 0000000000..df3c5ec826 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_request.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_request import ( + LLMObsAnnotationQueueUpdateDataRequest, + ) + + +class LLMObsAnnotationQueueUpdateRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_request import ( + LLMObsAnnotationQueueUpdateDataRequest, + ) + + return { + "data": (LLMObsAnnotationQueueUpdateDataRequest,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueUpdateDataRequest, **kwargs): + """ + Request to update an LLM Observability annotation queue. + + :param data: Data object for updating an LLM Observability annotation queue. + :type data: LLMObsAnnotationQueueUpdateDataRequest + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queues_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queues_response.py new file mode 100644 index 0000000000..aad072f9d9 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queues_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_response import LLMObsAnnotationQueueDataResponse + + +class LLMObsAnnotationQueuesResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_data_response import LLMObsAnnotationQueueDataResponse + + return { + "data": ([LLMObsAnnotationQueueDataResponse],), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: List[LLMObsAnnotationQueueDataResponse], **kwargs): + """ + Response containing a list of LLM Observability annotation queues. + + :param data: List of annotation queues. + :type data: [LLMObsAnnotationQueueDataResponse] + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_attributes_request.py new file mode 100644 index 0000000000..5d2eca403c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_attributes_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +class LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest(ModelNormal): + validations = { + "interaction_ids": { + "min_items": 1, + }, + } + + @cached_property + def openapi_types(_): + return { + "interaction_ids": ([str],), + } + + attribute_map = { + "interaction_ids": "interaction_ids", + } + + def __init__(self_, interaction_ids: List[str], **kwargs): + """ + Attributes for deleting interactions from an annotation queue. + + :param interaction_ids: List of interaction IDs to delete. Must contain at least one item. + :type interaction_ids: [str] + """ + super().__init__(kwargs) + + self_.interaction_ids = interaction_ids diff --git a/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_request.py b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_request.py new file mode 100644 index 0000000000..16dab2459c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_data_request.py @@ -0,0 +1,61 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_attributes_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + +class LLMObsDeleteAnnotationQueueInteractionsDataRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_attributes_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import ( + LLMObsAnnotationQueueInteractionsType, + ) + + return { + "attributes": (LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest,), + "type": (LLMObsAnnotationQueueInteractionsType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest, + type: LLMObsAnnotationQueueInteractionsType, + **kwargs, + ): + """ + Data object for deleting interactions from an annotation queue. + + :param attributes: Attributes for deleting interactions from an annotation queue. + :type attributes: LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest + + :param type: Resource type for annotation queue interactions. + :type type: LLMObsAnnotationQueueInteractionsType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_request.py b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_request.py new file mode 100644 index 0000000000..e827c965ad --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_delete_annotation_queue_interactions_request.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataRequest, + ) + + +class LLMObsDeleteAnnotationQueueInteractionsRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataRequest, + ) + + return { + "data": (LLMObsDeleteAnnotationQueueInteractionsDataRequest,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsDeleteAnnotationQueueInteractionsDataRequest, **kwargs): + """ + Request to delete interactions from an LLM Observability annotation queue. + + :param data: Data object for deleting interactions from an annotation queue. + :type data: LLMObsDeleteAnnotationQueueInteractionsDataRequest + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_interaction_type.py b/src/datadog_api_client/v2/model/llm_obs_interaction_type.py new file mode 100644 index 0000000000..af43da7b21 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_interaction_type.py @@ -0,0 +1,38 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsInteractionType(ModelSimple): + """ + Type of interaction in an annotation queue. + + :param value: Must be one of ["trace", "experiment_trace"]. + :type value: str + """ + + allowed_values = { + "trace", + "experiment_trace", + } + TRACE: ClassVar["LLMObsInteractionType"] + EXPERIMENT_TRACE: ClassVar["LLMObsInteractionType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsInteractionType.TRACE = LLMObsInteractionType("trace") +LLMObsInteractionType.EXPERIMENT_TRACE = LLMObsInteractionType("experiment_trace") diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index b3a7bf728c..3020453da0 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -2959,6 +2959,58 @@ from datadog_api_client.v2.model.kind_metadata import KindMetadata from datadog_api_client.v2.model.kind_obj import KindObj from datadog_api_client.v2.model.kind_response_meta import KindResponseMeta +from datadog_api_client.v2.model.llm_obs_annotated_interaction_item import LLMObsAnnotatedInteractionItem +from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_attributes_response import ( + LLMObsAnnotatedInteractionsDataAttributesResponse, +) +from datadog_api_client.v2.model.llm_obs_annotated_interactions_data_response import ( + LLMObsAnnotatedInteractionsDataResponse, +) +from datadog_api_client.v2.model.llm_obs_annotated_interactions_response import LLMObsAnnotatedInteractionsResponse +from datadog_api_client.v2.model.llm_obs_annotated_interactions_type import LLMObsAnnotatedInteractionsType +from datadog_api_client.v2.model.llm_obs_annotation_item import LLMObsAnnotationItem +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_request import ( + LLMObsAnnotationQueueDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_attributes_response import ( + LLMObsAnnotationQueueDataAttributesResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_request import LLMObsAnnotationQueueDataRequest +from datadog_api_client.v2.model.llm_obs_annotation_queue_data_response import LLMObsAnnotationQueueDataResponse +from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_item import LLMObsAnnotationQueueInteractionItem +from datadog_api_client.v2.model.llm_obs_annotation_queue_interaction_response_item import ( + LLMObsAnnotationQueueInteractionResponseItem, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_request import ( + LLMObsAnnotationQueueInteractionsDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_attributes_response import ( + LLMObsAnnotationQueueInteractionsDataAttributesResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_request import ( + LLMObsAnnotationQueueInteractionsDataRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_data_response import ( + LLMObsAnnotationQueueInteractionsDataResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_request import ( + LLMObsAnnotationQueueInteractionsRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_response import ( + LLMObsAnnotationQueueInteractionsResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import LLMObsAnnotationQueueInteractionsType +from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest +from datadog_api_client.v2.model.llm_obs_annotation_queue_response import LLMObsAnnotationQueueResponse +from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_attributes_request import ( + LLMObsAnnotationQueueUpdateDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_data_request import ( + LLMObsAnnotationQueueUpdateDataRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest +from datadog_api_client.v2.model.llm_obs_annotation_queues_response import LLMObsAnnotationQueuesResponse from datadog_api_client.v2.model.llm_obs_cursor_meta import LLMObsCursorMeta from datadog_api_client.v2.model.llm_obs_dataset_data_attributes_request import LLMObsDatasetDataAttributesRequest from datadog_api_client.v2.model.llm_obs_dataset_data_attributes_response import LLMObsDatasetDataAttributesResponse @@ -2991,6 +3043,15 @@ from datadog_api_client.v2.model.llm_obs_dataset_update_data_request import LLMObsDatasetUpdateDataRequest from datadog_api_client.v2.model.llm_obs_dataset_update_request import LLMObsDatasetUpdateRequest from datadog_api_client.v2.model.llm_obs_datasets_response import LLMObsDatasetsResponse +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_attributes_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest, +) +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_data_request import ( + LLMObsDeleteAnnotationQueueInteractionsDataRequest, +) +from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_request import ( + LLMObsDeleteAnnotationQueueInteractionsRequest, +) from datadog_api_client.v2.model.llm_obs_delete_dataset_records_data_attributes_request import ( LLMObsDeleteDatasetRecordsDataAttributesRequest, ) @@ -3040,6 +3101,7 @@ from datadog_api_client.v2.model.llm_obs_experiment_update_data_request import LLMObsExperimentUpdateDataRequest from datadog_api_client.v2.model.llm_obs_experiment_update_request import LLMObsExperimentUpdateRequest from datadog_api_client.v2.model.llm_obs_experiments_response import LLMObsExperimentsResponse +from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType from datadog_api_client.v2.model.llm_obs_metric_assessment import LLMObsMetricAssessment from datadog_api_client.v2.model.llm_obs_metric_score_type import LLMObsMetricScoreType from datadog_api_client.v2.model.llm_obs_project_data_attributes_request import LLMObsProjectDataAttributesRequest @@ -9078,6 +9140,32 @@ "KindMetadata", "KindObj", "KindResponseMeta", + "LLMObsAnnotatedInteractionItem", + "LLMObsAnnotatedInteractionsDataAttributesResponse", + "LLMObsAnnotatedInteractionsDataResponse", + "LLMObsAnnotatedInteractionsResponse", + "LLMObsAnnotatedInteractionsType", + "LLMObsAnnotationItem", + "LLMObsAnnotationQueueDataAttributesRequest", + "LLMObsAnnotationQueueDataAttributesResponse", + "LLMObsAnnotationQueueDataRequest", + "LLMObsAnnotationQueueDataResponse", + "LLMObsAnnotationQueueInteractionItem", + "LLMObsAnnotationQueueInteractionResponseItem", + "LLMObsAnnotationQueueInteractionsDataAttributesRequest", + "LLMObsAnnotationQueueInteractionsDataAttributesResponse", + "LLMObsAnnotationQueueInteractionsDataRequest", + "LLMObsAnnotationQueueInteractionsDataResponse", + "LLMObsAnnotationQueueInteractionsRequest", + "LLMObsAnnotationQueueInteractionsResponse", + "LLMObsAnnotationQueueInteractionsType", + "LLMObsAnnotationQueueRequest", + "LLMObsAnnotationQueueResponse", + "LLMObsAnnotationQueueType", + "LLMObsAnnotationQueueUpdateDataAttributesRequest", + "LLMObsAnnotationQueueUpdateDataRequest", + "LLMObsAnnotationQueueUpdateRequest", + "LLMObsAnnotationQueuesResponse", "LLMObsCursorMeta", "LLMObsDatasetDataAttributesRequest", "LLMObsDatasetDataAttributesResponse", @@ -9102,6 +9190,9 @@ "LLMObsDatasetUpdateDataRequest", "LLMObsDatasetUpdateRequest", "LLMObsDatasetsResponse", + "LLMObsDeleteAnnotationQueueInteractionsDataAttributesRequest", + "LLMObsDeleteAnnotationQueueInteractionsDataRequest", + "LLMObsDeleteAnnotationQueueInteractionsRequest", "LLMObsDeleteDatasetRecordsDataAttributesRequest", "LLMObsDeleteDatasetRecordsDataRequest", "LLMObsDeleteDatasetRecordsRequest", @@ -9135,6 +9226,7 @@ "LLMObsExperimentUpdateDataRequest", "LLMObsExperimentUpdateRequest", "LLMObsExperimentsResponse", + "LLMObsInteractionType", "LLMObsMetricAssessment", "LLMObsMetricScoreType", "LLMObsProjectDataAttributesRequest", diff --git a/tests/v2/features/llm_observability.feature b/tests/v2/features/llm_observability.feature index f1a56f818d..a390628998 100644 --- a/tests/v2/features/llm_observability.feature +++ b/tests/v2/features/llm_observability.feature @@ -1,13 +1,40 @@ @endpoint(llm-observability) @endpoint(llm-observability-v2) Feature: LLM Observability - Manage LLM Observability projects, datasets, dataset records, and - experiments via the Experiments API. + Manage LLM Observability projects, datasets, dataset records, experiments, + and annotations. Background: Given a valid "apiKeyAuth" key in the system And a valid "appKeyAuth" key in the system And an instance of "LLMObservability" API + @generated @skip @team:DataDog/ml-observability + Scenario: Add annotation queue interactions returns "Bad Request" response + Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + And new "CreateLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Add annotation queue interactions returns "Created" response + Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + And new "CreateLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} + When the request is sent + Then the response status is 201 Created + + @generated @skip @team:DataDog/ml-observability + Scenario: Add annotation queue interactions returns "Not Found" response + Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + And new "CreateLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} + When the request is sent + Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability Scenario: Append records to an LLM Observability dataset returns "Bad Request" response Given operation "CreateLLMObsDatasetRecords" enabled @@ -48,6 +75,22 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Create an LLM Observability annotation queue returns "Bad Request" response + Given operation "CreateLLMObsAnnotationQueue" enabled + And new "CreateLLMObsAnnotationQueue" request + And body with value {"data": {"attributes": {"description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Create an LLM Observability annotation queue returns "Created" response + Given operation "CreateLLMObsAnnotationQueue" enabled + And new "CreateLLMObsAnnotationQueue" request + And body with value {"data": {"attributes": {"description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} + When the request is sent + Then the response status is 201 Created + @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability dataset returns "Bad Request" response Given operation "CreateLLMObsDataset" enabled @@ -221,6 +264,87 @@ Feature: LLM Observability When the request is sent Then the response status is 204 No Content + @generated @skip @team:DataDog/ml-observability + Scenario: Delete an LLM Observability annotation queue returns "No Content" response + Given operation "DeleteLLMObsAnnotationQueue" enabled + And new "DeleteLLMObsAnnotationQueue" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 204 No Content + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete an LLM Observability annotation queue returns "Not Found" response + Given operation "DeleteLLMObsAnnotationQueue" enabled + And new "DeleteLLMObsAnnotationQueue" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete annotation queue interactions returns "Bad Request" response + Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + And new "DeleteLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete annotation queue interactions returns "No Content" response + Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + And new "DeleteLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} + When the request is sent + Then the response status is 204 No Content + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete annotation queue interactions returns "Not Found" response + Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + And new "DeleteLLMObsAnnotationQueueInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Get annotated queue interactions returns "Bad Request" response + Given operation "GetLLMObsAnnotatedInteractions" enabled + And new "GetLLMObsAnnotatedInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Get annotated queue interactions returns "Not Found" response + Given operation "GetLLMObsAnnotatedInteractions" enabled + And new "GetLLMObsAnnotatedInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Get annotated queue interactions returns "OK" response + Given operation "GetLLMObsAnnotatedInteractions" enabled + And new "GetLLMObsAnnotatedInteractions" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability annotation queues returns "Bad Request" response + Given operation "ListLLMObsAnnotationQueues" enabled + And new "ListLLMObsAnnotationQueues" request + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability annotation queues returns "OK" response + Given operation "ListLLMObsAnnotationQueues" enabled + And new "ListLLMObsAnnotationQueues" request + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset records returns "Bad Request" response Given operation "ListLLMObsDatasetRecords" enabled @@ -357,6 +481,33 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability annotation queue returns "Bad Request" response + Given operation "UpdateLLMObsAnnotationQueue" enabled + And new "UpdateLLMObsAnnotationQueue" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability annotation queue returns "Not Found" response + Given operation "UpdateLLMObsAnnotationQueue" enabled + And new "UpdateLLMObsAnnotationQueue" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability annotation queue returns "OK" response + Given operation "UpdateLLMObsAnnotationQueue" enabled + And new "UpdateLLMObsAnnotationQueue" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability dataset returns "Bad Request" response Given operation "UpdateLLMObsDataset" enabled diff --git a/tests/v2/features/undo.json b/tests/v2/features/undo.json index 02afc91a4b..078ced2bd8 100644 --- a/tests/v2/features/undo.json +++ b/tests/v2/features/undo.json @@ -3087,6 +3087,66 @@ "type": "idempotent" } }, + "ListLLMObsAnnotationQueues": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "CreateLLMObsAnnotationQueue": { + "tag": "LLM Observability", + "undo": { + "operationId": "DeleteLLMObsAnnotationQueue", + "parameters": [ + { + "name": "queue_id", + "source": "data.id" + } + ], + "type": "unsafe" + } + }, + "DeleteLLMObsAnnotationQueue": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, + "UpdateLLMObsAnnotationQueue": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, + "GetLLMObsAnnotatedInteractions": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "CreateLLMObsAnnotationQueueInteractions": { + "tag": "LLM Observability", + "undo": { + "operationId": "DeleteLLMObsAnnotationQueueInteractions", + "parameters": [ + { + "name": "queue_id", + "source": "path.queue_id" + }, + { + "name": "body", + "template": "{\"data\": {\"type\": \"interactions\", \"attributes\": {\"interaction_ids\": [\"{{ data.attributes.interactions[0].id }}\"]}}}" + } + ], + "type": "unsafe" + } + }, + "DeleteLLMObsAnnotationQueueInteractions": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListLLMObsExperiments": { "tag": "LLM Observability", "undo": {