From 80a9a80e4c29d6d40ec85a39f7c89904d7a8f7d8 Mon Sep 17 00:00:00 2001 From: Ian Streeter Date: Wed, 10 Jun 2026 10:50:58 +0100 Subject: [PATCH 1/5] Agent classification enrichment --- .../agent-classification-enrichment/index.md | 126 ++++++++++++++++++ .../available-enrichments/index.md | 1 + 2 files changed, 127 insertions(+) create mode 100644 docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md diff --git a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md new file mode 100644 index 000000000..31706f95a --- /dev/null +++ b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md @@ -0,0 +1,126 @@ +--- +title: "Agent classification enrichment" +sidebar_position: 9.5 +sidebar_label: Agent classification +description: "Identify AI crawlers and automated agents by classifying them by operator and purpose." +keywords: ["agent classification", "AI crawler", "bot classification", "YAUAA"] +date: "2026-06-10" +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import SchemaProperties from "@site/docs/reusable/schema-properties/_index.md" + +:::note[Availability] +This enrichment is available since version 6.12.0 of Enrich. +::: + +The agent classification enrichment identifies automated agents — AI crawlers, bots, and other non-human clients — by matching the agent name parsed by the [YAUAA enrichment](/docs/pipeline/enrichments/available-enrichments/yauaa-enrichment/index.md) against a lookup of known agents. For each match, it adds an [entity](/docs/fundamentals/entities/index.md) to the event identifying the operator (e.g. `OpenAI`) and, where known, the agent's purpose (e.g. `AI crawler`). + +## Prerequisites + +To use this enrichment, you need to enable the [YAUAA enrichment](/docs/pipeline/enrichments/available-enrichments/yauaa-enrichment/index.md). The agent classification enrichment reads the `agentName` field from the YAUAA entity. If YAUAA is not active, no agent classification entity is attached. + +## Configuration + +The enrichment takes these parameters: + +| Parameter | Required | Description | +| --- | --- | --- | +| `classificationFile` | ❌ | Location of a JSON file mapping YAUAA agent names to classification records. Already provided for CDI customers. | +| `classification` | ❌ | Inline classification records, keyed by YAUAA agent name. Takes precedence over matching entries in `classificationFile`. | + +At least one of `classificationFile` or `classification` must be present. + + + + +Configure the parameters in the Console enrichment editor. Keep the Console defaults for the `database` and `uri` fields. For example: + +```json +{ + "classificationFile": { + "uri": "", + "database": "" + }, + "classification": { + "MyInternalBot": {"operator": "Acme Corp", "purpose": "Monitoring"} + } +} +``` + + + + +For Self-Hosted, [provide a complete JSON](/docs/pipeline/enrichments/managing-enrichments/terraform/index.md). For example: + +```json +{ + "schema": "iglu:com.snowplowanalytics.snowplow.enrichments/agent_classification_enrichment_config/jsonschema/1-0-0", + "data": { + "name": "agent_classification_enrichment_config", + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "enabled": true, + "parameters": { + "classificationFile": { + "uri": "", + "database": "agent-classifications.json" + }, + "classification": { + "MyInternalBot": {"operator": "Acme Corp", "purpose": "Monitoring"} + } + } + } +} +``` + + + + +```mdx-code-block +import TestingWithMicro from "@site/docs/reusable/test-enrichment-with-micro/_index.md" + + +``` + +### `classificationFile` + +:::tip[Snowplow CDI] +If you're using Snowplow CDI, you don't need to configure this. Use the default values provided in Console. +::: + +Points to a JSON file containing the agent-to-classification mapping. The file must be a JSON object where the keys are YAUAA `agentName` values and the values are objects with `operator` (required) and `purpose` (optional). For example: + +```json +{ + "ChatGPT-User": {"operator": "OpenAI", "purpose": "AI data retrieval"}, + "GPTBot": {"operator": "OpenAI", "purpose": "AI crawler"}, + "ClaudeBot": {"operator": "Anthropic"} +} +``` + +| Field | Type | Description | +| --- | --- | --- | +| `uri` | string | Base URI where the file is hosted. Supports `http:`, `s3:`, and `gs:` schemes. Must not end with a trailing slash. | +| `database` | string | The JSON filename. | + +### `classification` + +An inline map of agent names to classification records, keyed by YAUAA `agentName`. Each record must include `operator` (the vendor operating the agent) and optionally `purpose` (the agent's purpose, if known). Records defined here take precedence over any matching entry in `classificationFile`. + +## Output + +This enrichment adds an `agent_classification` entity to events where the YAUAA `agentName` matches a known agent. + +This enrichment won't produce any output if: +* The YAUAA enrichment is not enabled +* The YAUAA entity does not contain an `agentName` field +* The `agentName` is not found in the classification lookup + + diff --git a/docs/pipeline/enrichments/available-enrichments/index.md b/docs/pipeline/enrichments/available-enrichments/index.md index bd6f0411b..7ae6ffaed 100644 --- a/docs/pipeline/enrichments/available-enrichments/index.md +++ b/docs/pipeline/enrichments/available-enrichments/index.md @@ -30,6 +30,7 @@ You can only configure one instance of each enrichment. | [Cookie Extractor](/docs/pipeline/enrichments/available-enrichments/cookie-extractor-enrichment/index.md) | Extract values of specific cookies into extra entities. | | [HTTP Header Extractor](/docs/pipeline/enrichments/available-enrichments/http-header-extractor-enrichment/index.md) | Extract values of specific HTTP headers into extra entities. | | [YAUAA](/docs/pipeline/enrichments/available-enrichments/yauaa-enrichment/index.md) | Parse the user agent string and attach an entity with detailed user agent information. | +| [Agent Classification](/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md) | Identify AI crawlers and automated agents by classifying them by operator and purpose using the YAUAA agent name. | | [IP Lookup](/docs/pipeline/enrichments/available-enrichments/ip-lookup-enrichment/index.md) | Look up useful data on the IP address in the MaxMind database. | | [ASN Lookup](/docs/pipeline/enrichments/available-enrichments/asn-lookup-enrichment/index.md) | Flag bot traffic by checking ASNs against known bad ASN lists. | | [Bot Detection](/docs/pipeline/enrichments/available-enrichments/bot-detection-enrichment/index.md) | Consolidate bot signals from YAUAA, IAB, and ASN lookup into a single entity. | From f464b644510ccf02ec57e3874ab0e45b5fb46439 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 31 Jul 2026 15:47:42 +0100 Subject: [PATCH 2/5] Apply suggestion from @benjben Co-authored-by: Benjamin BENOIST --- .../agent-classification-enrichment/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md index 31706f95a..f5e5d5f2e 100644 --- a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md +++ b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md @@ -44,7 +44,7 @@ Configure the parameters in the Console enrichment editor. Keep the Console defa "database": "" }, "classification": { - "MyInternalBot": {"operator": "Acme Corp", "purpose": "Monitoring"} + "SomeBot": {"operator": "Acme Corp", "purpose": "Monitoring"} } } ``` From 586ad9a31806328d36a1fa6e78649641d4b4e735 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 31 Jul 2026 15:48:22 +0100 Subject: [PATCH 3/5] Apply suggestion from @stanch --- .../agent-classification-enrichment/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md index f5e5d5f2e..24b3692b8 100644 --- a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md +++ b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md @@ -67,7 +67,7 @@ For Self-Hosted, [provide a complete JSON](/docs/pipeline/enrichments/managing-e "database": "agent-classifications.json" }, "classification": { - "MyInternalBot": {"operator": "Acme Corp", "purpose": "Monitoring"} + "SomeBot": {"operator": "Acme Corp", "purpose": "Monitoring"} } } } From a07b0240cd6e0fcd3060044a6b38494d74895e31 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 31 Jul 2026 16:01:07 +0100 Subject: [PATCH 4/5] Minor tweaks --- .../agent-classification-enrichment/index.md | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md index 24b3692b8..bc3b1e1ea 100644 --- a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md +++ b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md @@ -15,7 +15,7 @@ import SchemaProperties from "@site/docs/reusable/schema-properties/_index.md" This enrichment is available since version 6.12.0 of Enrich. ::: -The agent classification enrichment identifies automated agents — AI crawlers, bots, and other non-human clients — by matching the agent name parsed by the [YAUAA enrichment](/docs/pipeline/enrichments/available-enrichments/yauaa-enrichment/index.md) against a lookup of known agents. For each match, it adds an [entity](/docs/fundamentals/entities/index.md) to the event identifying the operator (e.g. `OpenAI`) and, where known, the agent's purpose (e.g. `AI crawler`). +The agent classification enrichment identifies automated agents — AI crawlers, bots, and other non-human clients — by matching the agent name parsed by the [YAUAA enrichment](/docs/pipeline/enrichments/available-enrichments/yauaa-enrichment/index.md) against a lookup of known agents. For each match, it adds an [entity](/docs/fundamentals/entities/index.md) to the event identifying the operator (e.g. `OpenAI`) and, where known, the agent's purpose (e.g. `AI_TRAINING`). ## Prerequisites @@ -49,6 +49,15 @@ Configure the parameters in the Console enrichment editor. Keep the Console defa } ``` +The agent categories in the default configuration are: + +* `AI_SEARCH_INDEX` — agents that power AI-based search features, e.g. `OAI-SearchBot` +* `AI_TRAINING` — agents used for crawling content and training AI models, e.g. `GPTBot` +* `AI_USER_FETCH` — agents that fetch content upon user request, e.g. `ChatGPT-User` +* `LINK_PREVIEW` — agents that visit links in order to provide a preview of the content, e.g. `BingPreview` +* `SEARCH_INDEX` — agents powering traditional search, e.g. `Bingbot` + + @@ -93,12 +102,18 @@ Points to a JSON file containing the agent-to-classification mapping. The file m ```json { - "ChatGPT-User": {"operator": "OpenAI", "purpose": "AI data retrieval"}, - "GPTBot": {"operator": "OpenAI", "purpose": "AI crawler"}, + "ChatGPT-User": {"operator": "OpenAI", "purpose": "AI_USER_FETCH"}, + "GPTBot": {"operator": "OpenAI", "purpose": "AI_TRAINING"}, "ClaudeBot": {"operator": "Anthropic"} } ``` +:::tip[Matching logic] + +The matching is not case-sensitive, i.e., both `GPTBot` and `GPTbot` (lowercase `b`) will match `GPTBot`. + +::: + | Field | Type | Description | | --- | --- | --- | | `uri` | string | Base URI where the file is hosted. Supports `http:`, `s3:`, and `gs:` schemes. Must not end with a trailing slash. | @@ -106,7 +121,7 @@ Points to a JSON file containing the agent-to-classification mapping. The file m ### `classification` -An inline map of agent names to classification records, keyed by YAUAA `agentName`. Each record must include `operator` (the vendor operating the agent) and optionally `purpose` (the agent's purpose, if known). Records defined here take precedence over any matching entry in `classificationFile`. +An inline classification using the same format as the file. Records defined here take precedence over any matching entry in `classificationFile`. ## Output @@ -121,6 +136,6 @@ This enrichment won't produce any output if: overview={{ entity: true }} example={{ operator: "OpenAI", - purpose: "AI crawler" + purpose: "AI_TRAINING" }} schema={{ "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", "description": "Schema for agent classification context generated by agent classification enrichment", "self": { "vendor": "com.snowplowanalytics.snowplow", "name": "agent_classification", "format": "jsonschema", "version": "1-0-0" }, "type": "object", "properties": { "operator": { "type": "string", "description": "The vendor operating the agent, e.g. OpenAI" }, "purpose": { "type": ["string", "null"], "description": "The purpose of the agent, e.g. AI crawler, if known" } }, "required": ["operator"], "additionalProperties": false }} /> From e1dd1b78f93cedf0eff3e591df2975f818ec73ee Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 31 Jul 2026 16:20:44 +0100 Subject: [PATCH 5/5] Fix formatting --- .../agent-classification-enrichment/index.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md index bc3b1e1ea..96bdf379e 100644 --- a/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md +++ b/docs/pipeline/enrichments/available-enrichments/agent-classification-enrichment/index.md @@ -57,9 +57,8 @@ The agent categories in the default configuration are: * `LINK_PREVIEW` — agents that visit links in order to provide a preview of the content, e.g. `BingPreview` * `SEARCH_INDEX` — agents powering traditional search, e.g. `Bingbot` - - - + + For Self-Hosted, [provide a complete JSON](/docs/pipeline/enrichments/managing-enrichments/terraform/index.md). For example: