diff --git a/api/openapi.yaml b/api/openapi.yaml index c95f37ba2..f471801b1 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -4985,6 +4985,13 @@ paths: schema: description: "List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only)." type: boolean + - description: "Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars." + explode: false + in: query + name: filter + schema: + description: "Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars." + type: string responses: "200": content: diff --git a/cli/src/__tests__/args.test.ts b/cli/src/__tests__/args.test.ts index 1ba730fa2..ef255c2d2 100644 --- a/cli/src/__tests__/args.test.ts +++ b/cli/src/__tests__/args.test.ts @@ -103,6 +103,12 @@ describe("getFlag", () => { // --to should still work expect(getFlag(args, "--to")).toBe("a@b.com"); }); + + it("extracts a messages list filter expression verbatim", () => { + expect( + getFlag(["messages", "list", "--filter", "label:urgent"], "--filter"), + ).toBe("label:urgent"); + }); }); describe("getFlags", () => { @@ -208,6 +214,16 @@ describe("checkFlags (FIX 1: single-dash flag typos)", () => { expect(() => checkFlags(["--json=true"], ["--json"])).toThrow("process.exit"); }); + it("rejects the retired legacy flag while accepting --filter", () => { + const retiredFilterFlag = `--${"q"}`; + expect(() => checkFlags([retiredFilterFlag, "label:urgent"], ["--filter"])).toThrow("process.exit"); + expect(mockExit).toHaveBeenCalledWith(2); + mockExit.mockClear(); + + expect(() => checkFlags(["--filter", "label:urgent"], ["--filter"])).not.toThrow(); + expect(mockExit).not.toHaveBeenCalled(); + }); + it("rejects the removed login --with-key flag", () => { expect(() => checkFlags(["--with-key"], [])).toThrow("process.exit"); expect(mockExit).toHaveBeenCalledWith(2); diff --git a/cli/src/__tests__/messages.test.ts b/cli/src/__tests__/messages.test.ts index 19cedd306..a3276d254 100644 --- a/cli/src/__tests__/messages.test.ts +++ b/cli/src/__tests__/messages.test.ts @@ -72,6 +72,27 @@ describe("messages commands", () => { ); }); + it("passes filter through verbatim and omits it when unset", async () => { + mockList.mockReturnValue(summaries()); + const { messagesList } = await import("../commands/messages.js"); + + await messagesList({ filter: "label:urgent" }); + expect(mockList).toHaveBeenCalledWith("bot@agents.e2a.dev", { + sort: "asc", + readStatus: "all", + filter: "label:urgent", + limit: 100, + }); + + mockList.mockClear(); + await messagesList({}); + expect(mockList).toHaveBeenCalledWith("bot@agents.e2a.dev", { + sort: "asc", + readStatus: "all", + limit: 100, + }); + }); + it("sanitizes TSV delimiters out of the sender-controlled From field", async () => { mockList.mockReturnValue( summaries({ diff --git a/cli/src/bin/e2a.ts b/cli/src/bin/e2a.ts index 7b0178fff..0a37c7784 100644 --- a/cli/src/bin/e2a.ts +++ b/cli/src/bin/e2a.ts @@ -74,6 +74,7 @@ Usage: --direction inbound|outbound|all --since Messages created AT or after this timestamp (inclusive — dedup by message id when cursoring) + --filter Boolean filter expression (see API filtering docs) --conversation Filter to one conversation (alias: --conversation-id) --read-status unread|read|all (default all — safe for poll loops) --limit Stop after n messages @@ -470,13 +471,14 @@ async function main() { if (sub === "list") { checkFlags(rest, [ "--direction", "--since", "--conversation", "--conversation-id", - "--read-status", "--limit", "--agent", "--json", + "--read-status", "--limit", "--agent", "--filter", "--json", ]); getPositionals(rest, 0, "usage: e2a messages list [options]"); await messagesList({ agent: getFlagChecked(rest, "--agent"), direction: getFlagChecked(rest, "--direction"), since: getFlagChecked(rest, "--since"), + filter: getFlagChecked(rest, "--filter"), conversation: getConversationId(rest), readStatus: getFlagChecked(rest, "--read-status"), limit: getFlagChecked(rest, "--limit"), diff --git a/cli/src/commands/messages.ts b/cli/src/commands/messages.ts index 3fb767244..abd106044 100644 --- a/cli/src/commands/messages.ts +++ b/cli/src/commands/messages.ts @@ -6,6 +6,7 @@ export interface MessagesListOptions { agent?: string; direction?: string; since?: string; + filter?: string; conversation?: string; limit?: string; readStatus?: string; @@ -26,7 +27,7 @@ export interface MessagesLifecycleOptions { } const LIST_USAGE = - "usage: e2a messages list [--direction inbound|outbound|all] [--since ] [--conversation ] [--read-status unread|read|all] [--limit ] [--agent ] [--json]"; + "usage: e2a messages list [--direction inbound|outbound|all] [--since ] [--filter ] [--conversation ] [--read-status unread|read|all] [--limit ] [--agent ] [--json]"; const GET_USAGE = "usage: e2a messages get [--text] [--agent ] [--json]"; const LIFECYCLE_USAGE = "usage: e2a messages lifecycle (beta) [--agent ] [--limit <1-100>] [--cursor ] [--json]"; @@ -61,6 +62,7 @@ export async function messagesList(opts: MessagesListOptions): Promise { params.readStatus = opts.readStatus as (typeof READ_STATUSES)[number]; } if (opts.since) params.since = opts.since; + if (opts.filter !== undefined) params.filter = opts.filter; if (opts.conversation) params.conversationId = opts.conversation; let max: number | undefined; diff --git a/docs/api.md b/docs/api.md index e72c07c54..60296103f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -415,8 +415,9 @@ single message. - `GET …/messages` — list inbound + outbound with filters (`direction`, `read_status`, `sort`, `from`, `subject_contains`, `conversation_id`, `labels`, - `since`, `until`) and cursor pagination. Held outbound drafts appear with - `status=pending_review`. + `since`, `until`) and cursor pagination. `filter` adds boolean composition; see + [message filtering](filtering.md) for its grammar and v1 fields. Held outbound + drafts appear with `status=pending_review`. - `POST …/messages` — send a new email (a new thread). Returns `202 Accepted` for every non-terminal outcome — `pending_review` when the agent's protection policy holds it for review, or `accepted` when the async pipeline durably queues it — diff --git a/docs/filtering.md b/docs/filtering.md new file mode 100644 index 000000000..378545e01 --- /dev/null +++ b/docs/filtering.md @@ -0,0 +1,82 @@ +# Message filtering (`filter`) + +`GET /v1/agents/{email}/messages?filter=…` accepts a small, boolean filter +language. It is an addition to—not a replacement for—the existing flat list +parameters. See the [filter-query design](superpowers/specs/2026-07-25-filter-query-language-design.md) +for rationale and rollout details. + +## Syntax + +Keywords are uppercase and case-sensitive: `AND`, `OR`, and `NOT`. + +```ebnf +query = expression [ WS ] ; +expression = sequence { WS* "AND" WS* sequence } ; +sequence = factor { WS factor } ; +factor = term { WS* "OR" WS* term } ; +term = [ "NOT" WS | "-" ] simple ; +simple = restriction | "(" WS* expression WS* ")" ; +restriction = member [ WS* comparator WS* value ] ; +member = ( TEXT | STRING ) { "." TEXT } ; +comparator = ":" | "=" | "!=" | "<" | "<=" | ">" | ">=" ; +value = STRING | TEXT { ( "." | ":" ) TEXT } ; +``` + +Whitespace between expressions is implicit `AND`. Binding, from tightest to +loosest, is **`NOT` > `OR` > implicit `AND` > explicit `AND`**. Use +parentheses when that is not what you mean. A restriction without a comparator +is syntactically a bare term, but v1 rejects it: qualify the value with a field. + +`STRING` is double-quoted. Quote values containing whitespace or reserved +operator/parenthesis characters; `.` and `:` may remain unquoted in values +(RFC3339 timestamps, email addresses, and `e2a:` labels). Only `\"` (a literal +quote) and `\\` (a literal backslash) are valid escapes. For example: +`subject:"build \"green\""`. A quoted field name is syntactically accepted, +but it must still resolve exactly to a supported v1 field name. + +## V1 fields + +| Field | Operators | Meaning | +| --- | --- | --- | +| `label` | `:` | Message has the label. Values match `[a-z0-9:_-]+` and are at most 64 characters. Use `NOT label:newsletter` to exclude it. | +| `from` | `:` `=` `!=` | Sender. `:` is case-insensitive substring matching; `=` and `!=` are case-insensitive exact matching. | +| `subject` | `:` `=` `!=` | Subject, with the same matching rules as `from`. A NULL subject matches none of these predicates. | +| `created` | `=` `!=` `<` `<=` `>` `>=` | Creation time. Value is RFC3339 or a `YYYY-MM-DD` UTC calendar date. | + +For `from:` and `subject:`, `*` matches any sequence of characters. Literal +`%`, `_`, and `\` are escaped, so they never become SQL wildcards. The wildcard +is only meaningful for `:`; `=` and `!=` remain exact comparisons. + +A date-only `created` value denotes the full UTC day `[00:00, next 00:00)`: + +- `created=2026-07-01` matches that day; `!=` matches outside it. +- `<` is before that day's midnight; `<=` is before the following midnight. +- `>` starts at the following midnight; `>=` starts at that day's midnight. + +Examples: + +```text +label:urgent OR label:follow-up +label:urgent AND NOT label:newsletter +(from:"alerts.example" OR subject:"release *") created>=2026-07-01 +created=2026-07-01 +``` + +## Composition, errors, and limits + +`filter` is ANDed with every flat list filter (`direction`, `read_status`, `from`, +`labels`, `since`, and so on). A pagination cursor is tied to the complete +filter identity, including `filter`; start a new query if any filter changes. + +Invalid syntax, unknown fields, unsupported operators, invalid values, and +limit violations return HTTP 400 with error code `invalid_filter`. Parse and +validation errors identify their source position, for example `at column 12`. + +The request accepts at most **500 Unicode code points**, nesting depth **64**, +and **512** AST nodes. Invalid UTF-8 and NUL are rejected. + +`has:attachment` is intentionally deferred. Inbound attachments are canonical +in raw MIME while outbound attachments currently use JSON, so a correct filter +needs a normalized attachment-count field. It will ship only after a +blue/green-safe expand, backfill, and contract rollout makes that field reliable +for every live writer. diff --git a/docs/superpowers/plans/2026-07-25-filter-query-language.md b/docs/superpowers/plans/2026-07-25-filter-query-language.md index af5b44c88..1516e4537 100644 --- a/docs/superpowers/plans/2026-07-25-filter-query-language.md +++ b/docs/superpowers/plans/2026-07-25-filter-query-language.md @@ -1,8 +1,14 @@ -# Filter Query Language (`q`) Implementation Plan +# Filter Query Language Implementation Plan + +> **Amendment (2026-07-28, PR #735):** The public message-list boolean-expression +> parameter is `filter`, not `q`. The grammar and implementation semantics in this +> historical plan are unchanged; its remaining `q` references describe the original +> proposed spelling only. The implemented API, SDKs, CLI, MCP tool, docs, and +> production E2E use `filter`. > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Add a boolean filter query language (`q` param) to `list_messages`, backed by a new dependency-free, schema-agnostic Go package `internal/filterquery` (parse → validate → emit parameterized Postgres SQL). +**Goal:** Add a boolean filter query language (`filter` param) to `list_messages`, backed by a new dependency-free, schema-agnostic Go package `internal/filterquery` (parse → validate → emit parameterized Postgres SQL). **Architecture:** Spec: `docs/superpowers/specs/2026-07-25-filter-query-language-design.md`. Hand-rolled recursive-descent parser over the AIP-160 EBNF; validation against an adopter-supplied `FieldRegistry`; emission through a `Dialect` interface. e2a's messages schema lives only in `internal/identity`'s registry. Ships as PR 1 (package only) + PR 2 (API surface + MCP + SDKs + CLI + docs). @@ -25,6 +31,7 @@ - Hard caps: input ≤ 500 chars (handler), nesting depth ≤ 64, nodes ≤ 512. Violations return positioned errors, never panics. - Placeholders are always 1-based `$n`, numbered from a caller-supplied start index; identifiers come only from registry constants; user values are always bound args. - DSL semantics MUST match flat params: `from:` = `m.sender ILIKE … ESCAPE '\'` (same as flat `from`), `subject:` = `m.subject ILIKE … ESCAPE '\'` (same as flat `subject_contains`), both via the existing `escapeLikePattern` helper; `*` in values maps to `%` (after escaping literals). +- Attachment filtering is deferred from v1. Inbound attachments are canonical in raw MIME while outbound attachments use `attachments_json`; an exact `has:attachment` needs a normalized count introduced by a blue/green-safe expand/backfill/contract rollout (nullable field + all writers, exact resumable Go backfill, then query exposure after old writers are gone). - `label:` values must match `^[a-z0-9:_-]{1,64}$`; `e2a:` system labels are filterable (read-only on writes, unchanged). - DB-backed tests need Postgres: `make docker-up` (pg at localhost:5433), then `go test ./internal/identity/ -run `. - Commit style: `feat(filterquery): …` / `feat(api): …` with trailer `Co-Authored-By: Kimi `. Commit at every task boundary. @@ -53,7 +60,7 @@ package filterquery import "testing" func TestLexBasics(t *testing.T) { - toks, err := lex(`label:urgent OR (from:"alice@x.com" AND NOT has:attachment)`) + toks, err := lex(`label:urgent OR (from:"alice@x.com" AND NOT subject:newsletter)`) if err != nil { t.Fatalf("lex: %v", err) } @@ -362,7 +369,7 @@ Expected: PASS (6 tests) ```bash git add internal/filterquery/ docs/superpowers/specs/2026-07-25-filter-query-language-design.md -git commit -m "feat(filterquery): lexer and error type for the q filter language +git commit -m "feat(filterquery): lexer and error type for the filter language Also corrects the spec's precedence note (NOT > OR > implicit AND > explicit AND, per the AIP-160 EBNF). @@ -443,7 +450,7 @@ func TestParsePrecedence(t *testing.T) { `-a:x`: `(not (a : x))`, `a:x AND (b:y OR c:z)`: `(and (a : x) (or (b : y) (c : z)))`, `(a:x OR b:y) AND NOT c:z`: `(and (or (a : x) (b : y)) (not (c : z)))`, - `label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01`: `(and (or (label : urgent) (and (from : alerts) (not (has : attachment)))) (created >= 2026-07-01))`, + `label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01`: `(and (or (label : urgent) (and (from : alerts) (not (subject : newsletter)))) (created >= 2026-07-01))`, } for q, want := range cases { if got := parseToString(t, q); got != want { @@ -2056,7 +2063,24 @@ Co-Authored-By: Kimi " **Interfaces:** - Consumes: `filterquery` public API (Tasks 1–4); existing `escapeLikePattern` (identity/store.go). -- Produces: `MessagesQRegistry() *filterquery.Registry` (lazy singleton); unexported `messagesFieldRegistry()`; value type `createdValue{at time.Time, dayRange bool}`. Task 10's differential tests and Task 11's handler consume these. +- Produces: `MessagesQRegistry() *filterquery.Registry` (lazy singleton); value type `createdValue{at time.Time, dayRange bool}`. Task 10's differential tests and Task 11's handler consume these. + +**Binding corrections and coverage requirements:** + +- Imports use the repository module path + `github.com/tokencanopy/e2a/internal/filterquery`. +- Date-only values denote the entire UTC calendar day. Unit tests must cover + every operator and its exact arguments: `=` is `[start,end)`, `!=` is + outside that range, `<` ends at `start`, `<=` ends at `end`, `>` begins at + `end`, and `>=` begins at `start`. In particular, + `created<=YYYY-MM-DD` includes that whole day and + `created>YYYY-MM-DD` starts at the following midnight. +- `from:` and `subject:` must preserve the flat-filter behavior exactly: + byte-counted 200-byte bound, case-insensitive substring, literal `%`, `_`, + and `\` escaping, and `*` translated to the ILIKE wildcard. Add boundary + tests for 200/201 bytes and exact tests showing `=`/`!=` do not translate + wildcards. +- Registry unit tests must be parallel-safe and must not mutate global state. - [ ] **Step 1: Write the failing tests** @@ -2068,7 +2092,7 @@ import ( "testing" "time" - "e2a/internal/filterquery" + "github.com/tokencanopy/e2a/internal/filterquery" ) func compileQ(t *testing.T, q string, start int) (string, []any) { @@ -2129,13 +2153,13 @@ func TestSubjectField(t *testing.T) { } } -func TestHasAttachment(t *testing.T) { - frag, args := compileQ(t, `has:attachment`, 1) - if frag != `(COALESCE(jsonb_array_length(m.attachments_json), 0) > 0)` || len(args) != 0 { - t.Errorf("frag=%s args=%v", frag, args) +func TestHasAttachmentIsDeferred(t *testing.T) { + _, _, err := filterquery.Compile(`has:attachment`, MessagesQRegistry(), filterquery.PostgresDialect{}, 1) + if err == nil { + t.Fatal("has:attachment: want unknown-field rejection") } - if _, _, err := filterquery.Compile(`has:body`, MessagesQRegistry(), filterquery.PostgresDialect{}, 1); err == nil { - t.Error("has:body: want rejection") + if got, want := err.Error(), `unknown field "has" — supported fields: created, from, label, subject`; !strings.Contains(got, want) { + t.Errorf("error = %q, want substring %q", got, want) } } @@ -2187,7 +2211,7 @@ import ( "sync" "time" - "e2a/internal/filterquery" + "github.com/tokencanopy/e2a/internal/filterquery" ) // q-language field registry for the messages table. Semantics MUST match the @@ -2210,7 +2234,6 @@ func MessagesQRegistry() *filterquery.Registry { labelQField(), fromQField(), subjectQField(), - hasQField(), createdQField(), ) if err != nil { @@ -2276,22 +2299,6 @@ func textQField(name, column string, maxLen int) filterquery.FieldSpec { func fromQField() filterquery.FieldSpec { return textQField("from", "m.sender", 200) } func subjectQField() filterquery.FieldSpec { return textQField("subject", "m.subject", 200) } -func hasQField() filterquery.FieldSpec { - return filterquery.FieldSpec{ - Name: "has", - Ops: []string{":"}, - Coerce: func(raw string, quoted bool) (any, error) { - if raw != "attachment" { - return nil, fmt.Errorf("unsupported has: value %q — v1 supports has:attachment", raw) - } - return raw, nil - }, - Emit: func(c *filterquery.Comparison, e *filterquery.EmitCtx) (string, error) { - return "COALESCE(jsonb_array_length(m.attachments_json), 0) > 0", nil - }, - } -} - // createdValue carries date-coercion semantics: a date-only input (dayRange) // makes "=" mean "that UTC day", not "that exact midnight second". type createdValue struct { @@ -2348,7 +2355,7 @@ Expected: PASS (unit tests need no DB) ```bash git add internal/identity/filter_registry.go internal/identity/filter_registry_test.go -git commit -m "feat(identity): messages field registry for the q filter language +git commit -m "feat(identity): messages field registry for the filter language Co-Authored-By: Kimi " ``` @@ -2363,25 +2370,47 @@ Co-Authored-By: Kimi " **Interfaces:** - Consumes: `MessagesQRegistry()` (Task 9), `filterquery.Expr`. -- Produces: `MessageListFilter.QEmit func(startIdx int) (fragment string, args []interface{})` — the handler (Task 11) sets it. +- Produces: `MessageListFilter.Q *filterquery.Expr` — the handler (Task 11) sets it and the Postgres store emits it at the correct placeholder offset. + +**Binding corrections (supersede the illustrative callback/evaluator/Root +sections below):** + +- Pass the validated expression as data (`Q *filterquery.Expr`), not an + errorless callback. `GetMessagesByAgent` calls `Q.Emit` with + `PostgresDialect{}` after all flat filters, propagates any emission error, + and appends its fragment/arguments. Add a store error-path test and a + flat-filter composition test that proves placeholder numbering. +- Do **not** add `Expr.Root()` or any other mutable-AST exposure. The generic + package deliberately keeps a validated expression encapsulated; production + API must not be widened solely for a test. +- Replace the AST-sharing "naive evaluator" below with an independent fixture + oracle: each query declares expected fixture keys/IDs explicitly. Cover + precedence, NOT, labels, case-insensitive exact/substring text matching, + escaped `%`/`_`/`\`, `*` wildcard behavior, Unicode, and all + date-only boundaries. Include fixtures immediately before the day, exactly + at its first midnight, exactly at the following midnight, and after it so + `<`, `<=`, `=`, `!=`, `>`, and `>=` cannot share a mistaken boundary. +- The old Step 2 evaluator and Step 3 `Expr.Root()` snippets are retained only + as historical illustration and must not be implemented. - [ ] **Step 1: Add the filter field and the store splice** In `MessageListFilter` (store.go ~3661, after `Labels []string`): ```go - // QEmit, when non-nil, emits the validated q-expression predicate with - // placeholders numbered from the given 1-based start index. The store - // invokes it after the built-in filters so $n numbering stays correct. - // Built by the handler from internal/filterquery.Expr. - QEmit func(startIdx int) (fragment string, args []interface{}) + // Q is a parsed and validated q-expression. The Postgres store emits it + // after built-in filters so placeholder numbering stays correct. + Q *filterquery.Expr ``` In `GetMessagesByAgent`, immediately after the `if len(f.Labels) > 0 { … }` block: ```go - if f.QEmit != nil { - frag, qargs := f.QEmit(len(args) + 1) + if f.Q != nil { + frag, qargs, err := f.Q.Emit(filterquery.PostgresDialect{}, len(args)+1) + if err != nil { + return nil, fmt.Errorf("emit filter: %w", err) + } if frag != "" { query += " AND " + frag args = append(args, qargs...) @@ -2404,8 +2433,8 @@ import ( "testing" "time" - "e2a/internal/filterquery" - "e2a/internal/testutil" + "github.com/tokencanopy/e2a/internal/filterquery" + "github.com/tokencanopy/e2a/internal/testutil" ) // Differential testing: for each q expression, the rows returned by the @@ -2420,7 +2449,6 @@ type fixtureMsg struct { subject string labels []string // nil = NULL labels column created time.Time - attachments int } func seedQFixtures(t *testing.T, store *Store, agentID string) []fixtureMsg { @@ -2429,7 +2457,7 @@ func seedQFixtures(t *testing.T, store *Store, agentID string) []fixtureMsg { day := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) fixtures := []fixtureMsg{ {sender: "alice@corp.com", subject: "Quarterly report", labels: []string{"urgent", "q3"}, created: day.Add(1 * time.Hour)}, - {sender: "bob@alerts.io", subject: "CPU alert", labels: []string{"alerts"}, created: day.Add(2 * time.Hour), attachments: 2}, + {sender: "bob@alerts.io", subject: "CPU alert", labels: []string{"alerts"}, created: day.Add(2 * time.Hour)}, {sender: "carol@news.net", subject: "Weekly digest", labels: []string{"newsletter"}, created: day.Add(3 * time.Hour)}, {sender: "ALICE@corp.com", subject: "Follow-up", labels: []string{"follow-up"}, created: day.Add(4 * time.Hour)}, {sender: "dave@x.com", subject: "", labels: nil, created: day.Add(5 * time.Hour)}, @@ -2450,12 +2478,6 @@ func seedQFixtures(t *testing.T, store *Store, agentID string) []fixtureMsg { t.Fatalf("labels %d: %v", i, err) } } - if fx.attachments > 0 { - att := fmt.Sprintf(`[{"filename":"a.pdf","content_type":"application/pdf","index":0,"size_bytes":10},{"filename":"b.pdf","content_type":"application/pdf","index":1,"size_bytes":10}][:{}]`, fx.attachments) - if _, err := store.pool.Exec(ctx, `UPDATE messages SET attachments_json = $1::jsonb WHERE id = $2`, att, m.ID); err != nil { - t.Fatalf("attachments %d: %v", i, err) - } - } if _, err := store.pool.Exec(ctx, `UPDATE messages SET created_at = $1 WHERE id = $2`, fx.created, m.ID); err != nil { t.Fatalf("created %d: %v", i, err) } @@ -2557,8 +2579,6 @@ func evalComparison(c *filterquery.Comparison, r fixtureMsg) bool { default: return !strings.EqualFold(r.subject, v) } - case "has": - return r.attachments > 0 case "created": cv := c.Value.(createdValue) at := cv.at @@ -2606,7 +2626,6 @@ func TestQDifferential(t *testing.T) { `label:urgent OR label:alerts`, `label:urgent AND NOT label:newsletter`, `NOT label:urgent`, - `(label:urgent OR label:follow-up) AND NOT has:attachment`, `from:alice`, `from:ALICE`, `from:*@corp.com`, @@ -2617,11 +2636,10 @@ func TestQDifferential(t *testing.T) { `subject:_now_`, // literal underscores `subject:"a*b literal"`, `subject:こんにちは`, - `has:attachment`, `created = "2026-07-01"`, // day range: all but the next-day row `created>=2026-07-02`, `created<2026-07-01T05:00:00Z`, - `label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01`, + `label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01`, } for _, q := range queries { expr, err := filterquery.Parse(q, MessagesQRegistry()) @@ -2708,9 +2726,28 @@ Co-Authored-By: Kimi " - Test: `internal/httpapi/messages_q_test.go` **Interfaces:** -- Consumes: `identity.MessagesQRegistry()` (Task 9), `filterquery.Parse/Expr`, `MessageListFilter.QEmit` (Task 10). +- Consumes: `identity.MessagesQRegistry()` (Task 9), `filterquery.Parse/Expr`, `MessageListFilter.Q` (Task 10). - Produces: public `q` query param on `GET /v1/agents/{email}/messages`; 400 `invalid_filter` contract. +**Binding corrections and coverage requirements:** + +- Count the 500-character `q` cap with `utf8.RuneCountInString`, consistent + with JSON Schema/OpenAPI character semantics. Test 500 Unicode code points + accepted and 501 rejected, in addition to ASCII oversize. +- Parse once in the handler and pass the resulting `*filterquery.Expr` + directly as `MessageListFilter.Q`. Do not preflight emission and do not + recreate an errorless callback; Task 10 made the store responsible for + dialect emission and error propagation. +- Pin the exact raw `q` string into pagination cursors and compare it on every + continuation. Tests must cover identical replay success, changed `q` + rejection as `invalid_cursor`, and adding/removing `q` across pages. +- Exercise the real HTTP/Huma stack. Invalid syntax, unknown fields, forbidden + operators, and length caps return 400 `invalid_filter`; positioned parser + errors retain their `(at column N)` text. `has:attachment` must be covered as + an unknown field whose diagnostic lists only `created, from, label, subject`. + Captured store filters must prove `Q` is non-nil and emits the expected + SQL/args, while flat filters remain set for AND composition. + - [ ] **Step 1: Write the failing handler tests** ```go @@ -2721,6 +2758,7 @@ package httpapi func TestQParamInvalid(t *testing.T) { // q with an unknown field → 400 invalid_filter naming the field + // q=has:attachment → 400 and supported fields are exactly created/from/label/subject // q longer than 500 chars → 400 invalid_filter // q with a parse error → 400 with "(at column N)" } @@ -2732,12 +2770,13 @@ func TestQParamCursorPinning(t *testing.T) { func TestQParamReachesStore(t *testing.T) { // q=label:urgent → stubbed ListMessages receives MessageListFilter with - // non-nil QEmit; invoking QEmit(1) yields "(m.labels @> $1)" and + // non-nil Q; invoking Q.Emit(PostgresDialect{}, 1) yields + // "(m.labels @> $1)" and // []interface{}{[]string{"urgent"}} } func TestQComposesWithFlatParams(t *testing.T) { - // q=label:urgent&from=alice → both From="alice" and QEmit set (AND + // q=label:urgent&from=alice → both From="alice" and Q set (AND // composition happens in the store, Task 10 covered the SQL) } ``` @@ -2754,7 +2793,7 @@ Expected: FAIL — `q` not a known query param / QEmit never set. In `ListMessagesInput` (messages.go ~368, after `Labels`): ```go - Q string `query:"q" doc:"Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, has, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR (e.g. 'label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01'). Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars."` + Q string `query:"q" doc:"Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR (e.g. 'label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01'). Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars."` ``` In `messagesCursor` (~395): @@ -2766,28 +2805,22 @@ In `messagesCursor` (~395): In `handleListMessages`, after the existing filter validation (~line 686, near `normalizeLabelFilter`): ```go - var qEmit func(int) (string, []interface{}) + var qExpr *filterquery.Expr if in.Q != "" { - if len(in.Q) > 500 { - return nil, NewError(http.StatusBadRequest, "invalid_filter", "q filter too long (max 500 chars)") + if utf8.RuneCountInString(in.Q) > 500 { + return nil, NewError(http.StatusBadRequest, "invalid_filter", "filter too long (max 500 chars)") } expr, err := filterquery.Parse(in.Q, identity.MessagesQRegistry()) if err != nil { return nil, NewError(http.StatusBadRequest, "invalid_filter", err.Error()) } - // Preflight: prove the expression emits (defensive — Validate already - // guarantees it) so the store closure can ignore the error. - if _, _, err := expr.Emit(filterquery.PostgresDialect{}, 1); err != nil { - return nil, NewError(http.StatusBadRequest, "invalid_filter", err.Error()) - } - qEmit = func(start int) (string, []interface{}) { - frag, args, _ := expr.Emit(filterquery.PostgresDialect{}, start) - return frag, args - } + qExpr = expr } ``` -Pass `QEmit: qEmit` in the `identity.MessageListFilter{…}` literal (~737), include `Q: in.Q` when encoding the cursor, and add to the cursor-mismatch check (~723) alongside the existing comparisons: +Pass `Q: qExpr` in the `identity.MessageListFilter{…}` literal (~737), include +the raw query as `Q: in.Q` when encoding the cursor, and add it to the +cursor-mismatch check (~723) alongside the existing comparisons: ```go cur.Q != in.Q || @@ -2795,7 +2828,7 @@ Pass `QEmit: qEmit` in the `identity.MessageListFilter{…}` literal (~737), inc (find the existing `if cur.AgentID != … || … !stringSlicesEqual(cur.Labels, labelsFilter)` block and add the one line; also set `Q: in.Q` on the cursor struct literal that encodes the next cursor.) -Import `"e2a/internal/filterquery"` in messages.go. +Import `"github.com/tokencanopy/e2a/internal/filterquery"` in messages.go. - [ ] **Step 4: Run tests to verify they pass** @@ -2806,7 +2839,7 @@ Expected: PASS ```bash git add internal/httpapi/ -git commit -m "feat(api): q filter param on list_messages with cursor pinning +git commit -m "feat(api): filter param on list_messages with cursor pinning Co-Authored-By: Kimi " ``` @@ -2848,6 +2881,11 @@ Co-Authored-By: Kimi " **Interfaces:** - Consumes: regenerated TS SDK `ListMessagesParams.q` (Task 14 runs `make generate-sdk`; if the MCP typecheck needs it first, run `make generate-sdk-ts` before this task's implementation step and commit it in Task 14). +**Binding correction:** Zod's plain `.max(500)` counts UTF-16 code units, while +the HTTP API counts Unicode code points. Use a schema refinement based on +`Array.from(value).length <= 500` with a clear max-length message. Test both +ASCII 500/501 and astral-Unicode 500/501 boundaries. + - [ ] **Step 1: Add the schema field + passthrough + test** In the `list_messages` `inputSchema` (after `labels`, ~line 435): @@ -2858,7 +2896,7 @@ In the `list_messages` `inputSchema` (after `labels`, ~line 435): .max(500) .optional() .describe( - "Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, has, created. Operators : = != < <= > >= with AND/OR/NOT and parentheses; whitespace is implicit AND (binds looser than OR). Example: 'label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01'. Composes (AND) with the flat filters (labels, from_, subject_contains, since, until). Invalid expressions are rejected with a positioned invalid_filter error.", + "Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators : = != < <= > >= with AND/OR/NOT and parentheses; whitespace is implicit AND (binds looser than OR). Example: 'label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01'. Composes (AND) with the flat filters (labels, from_, subject_contains, since, until). Invalid expressions are rejected with a positioned invalid_filter error.", ), ``` @@ -2887,14 +2925,14 @@ Expected: PASS ```bash git add mcp/ -git commit -m "feat(mcp): q filter on list_messages +git commit -m "feat(mcp): filter on list_messages Co-Authored-By: Kimi " ``` --- -### Task 14: SDK regen + CLI `--q` +### Task 14: SDK regen + CLI `--filter` **Files:** - Modify (generated): `sdks/python/src/e2a/v1/generated/**`, `sdks/typescript/src/v1/generated/**` @@ -2912,7 +2950,7 @@ Expected: `ListMessagesParams` (TS) and the messages API (Python) expose `q`; ch - [ ] **Step 2: CLI flag** -In `cli/src/commands/messages.ts` usage line, append ` [--q ]`; in the params building for `messages list`, add `q` when the flag is present (follow how `--since` maps into `params`). In `cli/src/__tests__/args.test.ts` add a case asserting `messages list --q "label:urgent"` puts `q: "label:urgent"` into the SDK params. +In `cli/src/commands/messages.ts` usage line, append ` [--filter ]`; in the params building for `messages list`, add `filter` when the flag is present (follow how `--since` maps into `params`). In `cli/src/__tests__/args.test.ts` add a case asserting `messages list --filter "label:urgent"` puts `filter: "label:urgent"` into the SDK params. - [ ] **Step 3: Run** @@ -2927,7 +2965,7 @@ Expected: PASS ```bash git add sdks/ cli/ -git commit -m "feat(sdks,cli): q filter param across python/ts SDKs and CLI +git commit -m "feat(sdks,cli): filter param across python/ts SDKs and CLI Co-Authored-By: Kimi " ``` diff --git a/docs/superpowers/specs/2026-07-25-filter-query-language-design.md b/docs/superpowers/specs/2026-07-25-filter-query-language-design.md index 4871009a9..65a6b56a2 100644 --- a/docs/superpowers/specs/2026-07-25-filter-query-language-design.md +++ b/docs/superpowers/specs/2026-07-25-filter-query-language-design.md @@ -4,6 +4,13 @@ Date: 2026-07-25 Status: Approved (design), pre-implementation Audience: e2a server, SDK, and MCP maintainers +## Amendment (2026-07-28): public parameter spelling + +PR #735 corrects the public message-list boolean-expression parameter from +`q` to `filter`. The grammar and semantics in this historical design are +unchanged. References to `q` below document the original proposal only; the +implemented API, SDKs, CLI, MCP tool, docs, and production E2E use `filter`. + ## Context and goals `GET /v1/agents/{email}/messages` today filters via flat params: `direction`, @@ -33,7 +40,7 @@ expression, backed by a small, dependency-free, schema-agnostic Go package Postgres SQL. **Non-goals (v1):** full-text/body search, `to`/`cc` fields, attachment -name/size filters, bare-term (unqualified) search, custom functions +presence/name/size filters, bare-term (unqualified) search, custom functions (`hasAttachment()`), SDK-side parsing or validation, Spanner/MySQL dialects (the `Dialect` interface admits them later), extraction as a standalone OSS library (the design only keeps that door open). @@ -88,8 +95,8 @@ Five files, five stages: identifier quoting, case-insensitive-match operator (ILIKE vs LOWER()). - e2a's registry lives in `internal/identity` (`messagesFieldRegistry()`): maps `label` → `m.labels` ops, `from` → `m.sender` (matching the existing - flat `from` filter), - `created` → `m.created_at`, `has:attachment` → attachment JSONB length. + flat `from` filter), `subject` → `m.subject`, and `created` → + `m.created_at`. - CI guard: a tiny test asserting `go list -deps ./internal/filterquery` contains no `internal/identity` (or any non-stdlib) import. @@ -100,25 +107,33 @@ Five files, five stages: | `label` | `:` | `label:urgent` = message carries label. Exclusion via `NOT label:x`. Value must pass the existing label charset rule (`[a-z0-9:_-]+`, ≤64). | | `from` | `:` `=` `!=` | `:` = case-insensitive substring (ILIKE) on `m.sender` — identical to the flat `from` filter. `=`/`!=` = case-insensitive exact. | | `subject` | `:` `=` `!=` | Same pattern on `subject`. NULL subjects never match `:`/`=` (SQL NULL semantics; document it). | -| `has` | `:` | Value `attachment` only: `has:attachment`. | | `created` | `=` `!=` `<` `<=` `>` `>=` | On `created_at`. Values: RFC3339 or `YYYY-MM-DD`. A date-only value denotes that whole UTC day: `=`/`!=` test membership in that day's `[midnight, next-midnight)` range, `<=` includes the whole day, and `>` begins at the next midnight. Full RFC3339 values are exact. | Everything else (`to:`, `body:`, bare text, …) parses but fails validation: -`unknown field 'to' — v1 supports: label, from, subject, has, created`. +`unknown field 'to' — supported fields: created, from, label, subject`. + +Attachment filtering is deliberately deferred. Inbound attachments are +canonical in the raw MIME message while outbound attachments use +`attachments_json`, so the existing JSON column cannot implement +`has:attachment` correctly. Shipping it requires a normalized attachment-count +field introduced with a blue/green-safe expand/backfill/contract rollout: +first make every live writer populate a nullable count and run an exact, +resumable Go backfill with `mailparse.Attachments`; only after old writers are +gone and the backfill is complete may the query field rely on that count. `*` inside quoted strings maps to ILIKE `%` for `from:`/`subject:`; literal user `%`, `_`, `\` are escaped and the predicate carries `ESCAPE '\'`. ## Emission pipeline (worked example) -Input: `label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01` +Input: `label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01` AST (precedence: NOT > OR > implicit AND > explicit AND): ``` And( Or( Has{label,"urgent"}, - And( Has{from,"alerts"}, Not(Has{has,"attachment"}) ) ), + And( Has{from,"alerts"}, Not(Has{subject,"newsletter"}) ) ), GreaterEq{created,2026-07-01T00:00:00Z} ) ``` @@ -127,9 +142,9 @@ Emitted (store already bound `$1`=agent_id, `$2..$3` flat filters): ```sql AND ( ( (m.labels @> $4) OR ( (m.sender ILIKE $5 ESCAPE '\') - AND NOT (COALESCE(jsonb_array_length(m.attachments_json),0) > 0) ) ) - AND (m.created_at >= $6) ) --- args += []string{"urgent"}, "%alerts%", 2026-07-01T00:00:00Z + AND NOT (m.subject ILIKE $6 ESCAPE '\') ) ) + AND (m.created_at >= $7) ) +-- args += []string{"urgent"}, "%alerts%", "%newsletter%", 2026-07-01T00:00:00Z ``` Rules that make this safe and correct: @@ -154,7 +169,7 @@ Rules that make this safe and correct: existing filter-identity rejection, same as other filters. - MCP `list_messages` gains optional `q` (pass-through, same validation). - SDKs regenerate from OpenAPI (Python `messages.list(..., q=...)`, TS - `{ q }`), CLI `messages list --q`, docs page documenting the grammar. + `{ filter }`), CLI `messages list --filter`, docs page documenting the grammar. ## Error model diff --git a/internal/filterquery/lexer_test.go b/internal/filterquery/lexer_test.go index 5b08755c2..3d408263a 100644 --- a/internal/filterquery/lexer_test.go +++ b/internal/filterquery/lexer_test.go @@ -3,7 +3,7 @@ package filterquery import "testing" func TestLexBasics(t *testing.T) { - toks, err := lex(`label:urgent OR (from:"alice@x.com" AND NOT has:attachment)`) + toks, err := lex(`label:urgent OR (from:"alice@x.com" AND NOT subject:newsletter)`) if err != nil { t.Fatalf("lex: %v", err) } diff --git a/internal/filterquery/parser_test.go b/internal/filterquery/parser_test.go index d82350ab5..4cd5375eb 100644 --- a/internal/filterquery/parser_test.go +++ b/internal/filterquery/parser_test.go @@ -27,7 +27,7 @@ func TestParsePrecedence(t *testing.T) { `-a:x`: `(not (a : x))`, `a:x AND (b:y OR c:z)`: `(and (a : x) (or (b : y) (c : z)))`, `(a:x OR b:y) AND NOT c:z`: `(and (or (a : x) (b : y)) (not (c : z)))`, - `label:urgent OR (from:alerts AND NOT has:attachment) created>=2026-07-01`: `(and (or (label : urgent) (and (from : alerts) (not (has : attachment)))) (created >= 2026-07-01))`, + `label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01`: `(and (or (label : urgent) (and (from : alerts) (not (subject : newsletter)))) (created >= 2026-07-01))`, } for q, want := range cases { if got := parseToString(t, q); got != want { diff --git a/internal/httpapi/messages.go b/internal/httpapi/messages.go index b643d7be2..08075b647 100644 --- a/internal/httpapi/messages.go +++ b/internal/httpapi/messages.go @@ -13,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/agent" "github.com/tokencanopy/e2a/internal/emailauth" "github.com/tokencanopy/e2a/internal/eventpayload" + "github.com/tokencanopy/e2a/internal/filterquery" "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/mailparse" ) @@ -371,6 +372,7 @@ type ListMessagesInput struct { Cursor string `query:"cursor"` Limit int `query:"limit" minimum:"1" maximum:"100" default:"100"` Deleted bool `query:"deleted" doc:"List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only)."` + Filter string `query:"filter" doc:"Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars."` } type listMessagesOutput struct { @@ -393,6 +395,7 @@ type messagesCursor struct { Since string `json:"sn,omitempty"` Until string `json:"un,omitempty"` Labels []string `json:"lb,omitempty"` + Filter string `json:"fl,omitempty"` Deleted bool `json:"dl,omitempty"` } @@ -688,6 +691,21 @@ func (s *Server) handleListMessages(ctx context.Context, in *ListMessagesInput) return nil, NewError(http.StatusBadRequest, "invalid_filter", err.Error()) } + var filterExpr *filterquery.Expr + if in.Filter != "" { + if !utf8.ValidString(in.Filter) || strings.IndexByte(in.Filter, 0) >= 0 { + return nil, NewError(http.StatusBadRequest, "invalid_filter", "filter must be valid UTF-8 and must not contain NUL") + } + if utf8.RuneCountInString(in.Filter) > 500 { + return nil, NewError(http.StatusBadRequest, "invalid_filter", "filter too long (max 500 chars)") + } + expr, err := filterquery.Parse(in.Filter, identity.MessagesFilterRegistry()) + if err != nil { + return nil, NewError(http.StatusBadRequest, "invalid_filter", err.Error()) + } + filterExpr = expr + } + // Time range. since, err := parseRFC3339Filter(in.Since, "since") if err != nil { @@ -719,6 +737,7 @@ func (s *Server) handleListMessages(ctx context.Context, in *ListMessagesInput) cur.From != in.From || cur.SubjectContains != in.SubjectContains || cur.ConversationID != in.ConversationID || cur.Since != rfc3339OrEmpty(since) || cur.Until != rfc3339OrEmpty(until) || + cur.Filter != in.Filter || cur.Deleted != in.Deleted || !stringSlicesEqual(cur.Labels, labelsFilter) { return nil, NewError(http.StatusBadRequest, "invalid_cursor", @@ -748,6 +767,7 @@ func (s *Server) handleListMessages(ctx context.Context, in *ListMessagesInput) Since: since, Until: until, Labels: labelsFilter, + Filter: filterExpr, Deleted: in.Deleted, }) if err != nil { @@ -771,6 +791,7 @@ func (s *Server) handleListMessages(ctx context.Context, in *ListMessagesInput) Status: status, Direction: direction, AgentID: ag.ID, Sort: sort, From: in.From, SubjectContains: in.SubjectContains, ConversationID: in.ConversationID, Since: rfc3339OrEmpty(since), Until: rfc3339OrEmpty(until), Labels: labelsFilter, + Filter: in.Filter, Deleted: in.Deleted, }) if err != nil { diff --git a/internal/httpapi/messages_filter_test.go b/internal/httpapi/messages_filter_test.go new file mode 100644 index 000000000..c59fabb57 --- /dev/null +++ b/internal/httpapi/messages_filter_test.go @@ -0,0 +1,245 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "testing" + "time" + "unicode/utf8" + + "github.com/tokencanopy/e2a/internal/filterquery" + "github.com/tokencanopy/e2a/internal/identity" +) + +func newMessagesFilterTestServer(t *testing.T, opts ...func(*Deps)) *httptest.Server { + t.Helper() + base := []func(*Deps){ + func(deps *Deps) { + deps.GetAgent = func(_ context.Context, address string) (*identity.AgentIdentity, error) { + if address != "bot@example.com" { + return nil, errors.New("not found") + } + return &identity.AgentIdentity{ID: "bot@example.com", Email: "bot@example.com", UserID: "u_1", DomainVerified: true}, nil + } + deps.GetAgentAnyState = deps.GetAgent + deps.ListMessages = func(_ context.Context, f identity.MessageListFilter) ([]identity.Message, error) { + if f.AgentID != "bot@example.com" { + return nil, errors.New("unexpected agent") + } + all := []identity.Message{ + {ID: "msg_b", Direction: "inbound", Sender: "b@x.com", Recipient: "bot@example.com", Subject: "B", InboxStatus: "unread", CreatedAt: time.Unix(1700000200, 0).UTC()}, + {ID: "msg_a", Direction: "inbound", Sender: "a@x.com", Recipient: "bot@example.com", Subject: "A", InboxStatus: "unread", CreatedAt: time.Unix(1700000100, 0).UTC()}, + } + if f.AfterID == "msg_b" { + return all[1:], nil + } + if f.Limit > 0 && len(all) > f.Limit { + return all[:f.Limit], nil + } + return all, nil + } + }, + } + return testServer(t, append(base, opts...)...) +} + +func messagesFilterURL(serverURL, filter string) string { + values := url.Values{ + "direction": {"inbound"}, + "read_status": {"all"}, + } + if filter != "" { + values.Set("filter", filter) + } + return serverURL + "/v1/agents/bot@example.com/messages?" + values.Encode() +} + +func filterError(t *testing.T, body map[string]any) map[string]any { + t.Helper() + errBody, ok := body["error"].(map[string]any) + if !ok { + t.Fatalf("expected error envelope, got %v", body) + } + if errBody["code"] != "invalid_filter" { + t.Fatalf("error code = %v, want invalid_filter (body %v)", errBody["code"], body) + } + return errBody +} + +func TestFilterParamInvalid(t *testing.T) { + valid500Unicode := "subject:" + strings.Repeat("界", 60) + valid500Unicode += strings.Repeat(" ", 500-utf8.RuneCountInString(valid500Unicode)) + if got := utf8.RuneCountInString(valid500Unicode); got != 500 { + t.Fatalf("valid Unicode query length = %d, want 500", got) + } + if len(strings.Repeat("界", 60)) >= 200 { + t.Fatal("test query subject value must remain under the 200-byte field cap") + } + + tests := []struct { + name string + filter string + wantInError string + wantOK bool + }{ + {name: "unknown field", filter: "unknown:thing", wantInError: `unknown field "unknown"`}, + {name: "attachment filtering deferred", filter: "has:attachment", wantInError: `unknown field "has" — supported fields: created, from, label, subject`}, + {name: "forbidden operator", filter: "label=urgent", wantInError: `operator "=" is not allowed on field "label"`}, + {name: "syntax retains column", filter: "label:", wantInError: "(at column 7)"}, + {name: "ASCII over code point limit", filter: "label:" + strings.Repeat("a", 501), wantInError: "filter too long (max 500 chars)"}, + {name: "Unicode over code point limit", filter: strings.Repeat("界", 501), wantInError: "filter too long (max 500 chars)"}, + {name: "NUL is invalid", filter: "subject:\x00", wantInError: "filter must be valid UTF-8 and must not contain NUL"}, + {name: "malformed UTF-8 is invalid", filter: string([]byte("subject:\xff")), wantInError: "filter must be valid UTF-8 and must not contain NUL"}, + {name: "Unicode code point limit accepts bytes over cap", filter: valid500Unicode, wantOK: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := newMessagesFilterTestServer(t) + code, body := getJSON(t, messagesFilterURL(srv.URL, tc.filter), "good") + if tc.wantOK { + if code != 200 { + t.Fatalf("status = %d, want 200 (body %v)", code, body) + } + return + } + if code != 400 { + t.Fatalf("status = %d, want 400 (body %v)", code, body) + } + errBody := filterError(t, body) + message, _ := errBody["message"].(string) + if !strings.Contains(message, tc.wantInError) { + t.Errorf("error message = %q, want it to contain %q", message, tc.wantInError) + } + }) + } +} + +func TestFilterParamCursorPinning(t *testing.T) { + srv := newMessagesFilterTestServer(t) + page1URL := messagesFilterURL(srv.URL, "label:urgent") + "&limit=1" + code, body := getJSON(t, page1URL, "good") + if code != 200 { + t.Fatalf("page 1 status = %d, want 200 (body %v)", code, body) + } + cursor, ok := body["next_cursor"].(string) + if !ok || cursor == "" { + t.Fatalf("page 1 next_cursor = %v, want non-empty string", body["next_cursor"]) + } + + continuation := func(filter string) (int, map[string]any) { + return getJSON(t, messagesFilterURL(srv.URL, filter)+"&limit=1&cursor="+url.QueryEscape(cursor), "good") + } + if code, body := continuation("label:urgent"); code != 200 { + t.Fatalf("identical filter continuation status = %d, want 200 (body %v)", code, body) + } + for _, filter := range []string{"label:other", ""} { + code, body := continuation(filter) + if code != 400 { + t.Fatalf("continuation filter=%q status = %d, want 400 (body %v)", filter, code, body) + } + if errBody, _ := body["error"].(map[string]any); errBody["code"] != "invalid_cursor" { + t.Fatalf("continuation filter=%q error = %v, want invalid_cursor", filter, body) + } + } + + // A cursor created without filter must also reject adding filter on page two. + code, body = getJSON(t, messagesFilterURL(srv.URL, "")+"&limit=1", "good") + if code != 200 { + t.Fatalf("no-filter page 1 status = %d, want 200 (body %v)", code, body) + } + noFilterCursor := body["next_cursor"].(string) + code, body = getJSON(t, messagesFilterURL(srv.URL, "label:urgent")+"&limit=1&cursor="+url.QueryEscape(noFilterCursor), "good") + if code != 400 { + t.Fatalf("adding filter continuation status = %d, want 400 (body %v)", code, body) + } + if errBody, _ := body["error"].(map[string]any); errBody["code"] != "invalid_cursor" { + t.Fatalf("adding filter continuation error = %v, want invalid_cursor", body) + } +} + +func TestListMessagesQIsNotAStructuredFilterAlias(t *testing.T) { + srv := newMessagesFilterTestServer(t) + code, body := getJSON(t, + srv.URL+"/v1/agents/bot@example.com/messages?direction=inbound&read_status=all&q=unknown:value", + "good", + ) + if code != http.StatusOK { + t.Fatalf("status = %d, body = %v; q must be ignored as an unknown query parameter", code, body) + } +} + +func TestFilterParamReachesStore(t *testing.T) { + var captured struct { + sync.Mutex + filter identity.MessageListFilter + } + srv := newMessagesFilterTestServer(t, func(deps *Deps) { + deps.ListMessages = func(_ context.Context, f identity.MessageListFilter) ([]identity.Message, error) { + captured.Lock() + captured.filter = f + captured.Unlock() + return nil, nil + } + }) + + code, body := getJSON(t, messagesFilterURL(srv.URL, "label:urgent"), "good") + if code != 200 { + t.Fatalf("status = %d, want 200 (body %v)", code, body) + } + captured.Lock() + filter := captured.filter + captured.Unlock() + if filter.Filter == nil { + t.Fatal("store Filter = nil, want parsed expression") + } + fragment, args, err := filter.Filter.Emit(filterquery.PostgresDialect{}, 1) + if err != nil { + t.Fatalf("Filter.Emit: %v", err) + } + if fragment != "(m.labels @> $1)" { + t.Errorf("Filter SQL = %q, want %q", fragment, "(m.labels @> $1)") + } + if want := []any{[]string{"urgent"}}; !reflect.DeepEqual(args, want) { + t.Errorf("Filter args = %#v, want %#v", args, want) + } +} + +func TestFilterComposesWithFlatParams(t *testing.T) { + var captured struct { + sync.Mutex + filter identity.MessageListFilter + } + srv := newMessagesFilterTestServer(t, func(deps *Deps) { + deps.ListMessages = func(_ context.Context, f identity.MessageListFilter) ([]identity.Message, error) { + captured.Lock() + captured.filter = f + captured.Unlock() + return nil, nil + } + }) + values := url.Values{} + values.Set("direction", "inbound") + values.Set("read_status", "all") + values.Set("from", "alice") + values.Set("filter", "label:urgent") + code, body := getJSON(t, srv.URL+"/v1/agents/bot@example.com/messages?"+values.Encode(), "good") + if code != 200 { + t.Fatalf("status = %d, want 200 (body %v)", code, body) + } + captured.Lock() + filter := captured.filter + captured.Unlock() + if filter.From != "alice" { + t.Errorf("store filter From = %q, want alice", filter.From) + } + if filter.Filter == nil { + t.Fatal("store Filter = nil, want parsed expression alongside From") + } +} diff --git a/internal/identity/filter_differential_test.go b/internal/identity/filter_differential_test.go new file mode 100644 index 000000000..de655d92f --- /dev/null +++ b/internal/identity/filter_differential_test.go @@ -0,0 +1,215 @@ +package identity_test + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/tokencanopy/e2a/internal/filterquery" + "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/testutil" +) + +type qFixture struct { + key string + sender string + subject string + labels []string + created time.Time + conversation string + id string +} + +func seedQAgent(t *testing.T, store *identity.Store, ctx context.Context) string { + t.Helper() + const domain = "qdiff.example.com" + user, err := store.CreateOrGetUser(ctx, "owner-qdiff@example.com", "Owner", "google-qdiff") + if err != nil { + t.Fatalf("CreateOrGetUser: %v", err) + } + if _, err := store.ClaimOrCreateDomain(ctx, domain, user.ID); err != nil { + t.Fatalf("ClaimOrCreateDomain: %v", err) + } + if err := store.VerifyDomain(ctx, domain, user.ID); err != nil { + t.Fatalf("VerifyDomain: %v", err) + } + agent, err := store.CreateAgent(ctx, "bot@"+domain, domain, "", "https://example.com/webhook", "", user.ID) + if err != nil { + t.Fatalf("CreateAgent: %v", err) + } + return agent.ID +} + +func seedQFixtures(t *testing.T, pool *pgxpool.Pool, store *identity.Store, agentID string) map[string]qFixture { + t.Helper() + ctx := context.Background() + day := time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC) + fixtures := []qFixture{ + {key: "before", sender: "before@example.com", subject: "xnowy", labels: []string{"archive"}, created: day.Add(-time.Nanosecond)}, + {key: "start", sender: "start@example.com", subject: "100 percent", labels: []string{"start"}, created: day}, + {key: "alice", sender: "alice@corp.com", subject: "Quarterly report", labels: []string{"urgent", "q3"}, created: day.Add(time.Hour), conversation: "qdiff-conversation"}, + {key: "alert", sender: "bob@alerts.io", subject: "CPU alert", labels: []string{"alerts"}, created: day.Add(2 * time.Hour)}, + {key: "newsletter", sender: "carol@news.net", subject: "Weekly digest", labels: []string{"newsletter"}, created: day.Add(3 * time.Hour)}, + {key: "follow", sender: "ALICE@corp.com", subject: "Follow-up", labels: []string{"follow-up"}, created: day.Add(4 * time.Hour)}, + {key: "empty", sender: "dave@x.com", subject: "", labels: []string{}, created: day.Add(5 * time.Hour)}, + {key: "literal", sender: "eve@percent.com", subject: "100% sure _now_ slash\\path", labels: []string{"urgent"}, created: day.Add(6 * time.Hour)}, + {key: "wildcard", sender: "frank@star.com", subject: "a*b literal", labels: []string{}, created: day.Add(7 * time.Hour)}, + {key: "unicode", sender: "日本語@例.jp", subject: "こんにちは 世界", labels: []string{"urgent", "日本"}, created: day.Add(8 * time.Hour)}, + {key: "end", sender: "end@example.com", subject: "At next day", labels: []string{"end"}, created: day.AddDate(0, 0, 1)}, + {key: "after", sender: "after@example.com", subject: "After next day", labels: []string{"after"}, created: day.AddDate(0, 0, 1).Add(time.Microsecond)}, + } + + for i := range fixtures { + fx := &fixtures[i] + message, err := store.CreateInboundMessage(ctx, "", agentID, fx.sender, "bot@qdiff.example.com", + fmt.Sprintf("", fx.key), fx.subject, fx.conversation, "", []byte("From: "+fx.sender+"\r\nSubject: "+fx.subject+"\r\n\r\nx"), + nil, nil, false, "", nil, nil, nil, identity.InboundScreening{}) + if err != nil { + t.Fatalf("seed %s: %v", fx.key, err) + } + fx.id = message.ID + + if _, err := pool.Exec(ctx, `UPDATE messages SET labels = $1, created_at = $2 WHERE id = $3`, fx.labels, fx.created, fx.id); err != nil { + t.Fatalf("set fixture %s: %v", fx.key, err) + } + } + + byKey := make(map[string]qFixture, len(fixtures)) + for _, fx := range fixtures { + byKey[fx.key] = fx + } + return byKey +} + +func fixtureIDs(t *testing.T, byKey map[string]qFixture, keys ...string) []string { + t.Helper() + ids := make([]string, 0, len(keys)) + for _, key := range keys { + fx, ok := byKey[key] + if !ok { + t.Fatalf("unknown fixture key %q", key) + } + ids = append(ids, fx.id) + } + sort.Strings(ids) + return ids +} + +func listedIDs(messages []identity.Message) []string { + ids := make([]string, 0, len(messages)) + for _, message := range messages { + ids = append(ids, message.ID) + } + sort.Strings(ids) + return ids +} + +func TestQFilterDifferential(t *testing.T) { + pool := testutil.TestDB(t) + store := identity.NewStore(pool) + ctx := context.Background() + agentID := seedQAgent(t, store, ctx) + byKey := seedQFixtures(t, pool, store, agentID) + + allDay := []string{"start", "alice", "alert", "newsletter", "follow", "empty", "literal", "wildcard", "unicode"} + allButUrgent := []string{"before", "start", "alert", "newsletter", "follow", "empty", "wildcard", "end", "after"} + allButAlice := []string{"before", "start", "alert", "newsletter", "empty", "literal", "wildcard", "unicode", "end", "after"} + allAfterStart := append(append([]string{}, allDay...), "end", "after") + allOutsideDay := []string{"before", "end", "after"} + queries := []struct { + q string + keys []string + }{ + {q: `label:urgent`, keys: []string{"alice", "literal", "unicode"}}, + {q: `label:urgent OR label:alerts`, keys: []string{"alice", "alert", "literal", "unicode"}}, + {q: `NOT label:urgent`, keys: allButUrgent}, + {q: `from:alice`, keys: []string{"alice", "follow"}}, + {q: `from:*@corp.com`, keys: []string{"alice", "follow"}}, + {q: `from = "alice@corp.com"`, keys: []string{"alice", "follow"}}, + {q: `from != "alice@corp.com"`, keys: allButAlice}, + {q: `subject:100%`, keys: []string{"literal"}}, + {q: `subject:_now_`, keys: []string{"literal"}}, + {q: `subject:"\\path"`, keys: []string{"literal"}}, + {q: `subject:"a*b literal"`, keys: []string{"wildcard"}}, + {q: `subject:こんにちは`, keys: []string{"unicode"}}, + {q: `created<2026-07-01`, keys: []string{"before"}}, + {q: `created<=2026-07-01`, keys: append([]string{"before"}, allDay...)}, + {q: `created=2026-07-01`, keys: allDay}, + {q: `created!=2026-07-01`, keys: allOutsideDay}, + {q: `created>2026-07-01`, keys: []string{"end", "after"}}, + {q: `created>=2026-07-01`, keys: allAfterStart}, + } + + for _, tc := range queries { + t.Run(tc.q, func(t *testing.T) { + expr, err := filterquery.Parse(tc.q, identity.MessagesFilterRegistry()) + if err != nil { + t.Fatalf("Parse(%q): %v", tc.q, err) + } + messages, err := store.GetMessagesByAgent(ctx, identity.MessageListFilter{ + AgentID: agentID, Direction: "inbound", Status: "all", Limit: 100, Filter: expr, + }) + if err != nil { + t.Fatalf("GetMessagesByAgent(%q): %v", tc.q, err) + } + if got, want := listedIDs(messages), fixtureIDs(t, byKey, tc.keys...); !reflect.DeepEqual(got, want) { + t.Errorf("q=%q IDs=%v, want %v", tc.q, got, want) + } + }) + } +} + +func TestQFilterComposesAfterFlatFilters(t *testing.T) { + pool := testutil.TestDB(t) + store := identity.NewStore(pool) + ctx := context.Background() + agentID := seedQAgent(t, store, ctx) + byKey := seedQFixtures(t, pool, store, agentID) + expr, err := filterquery.Parse(`from:corp.com`, identity.MessagesFilterRegistry()) + if err != nil { + t.Fatalf("Parse: %v", err) + } + day := time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC) + messages, err := store.GetMessagesByAgent(ctx, identity.MessageListFilter{ + AgentID: agentID, Direction: "inbound", Status: "all", Limit: 100, + From: "alice", SubjectContains: "quarter", ConversationID: "qdiff-conversation", + Since: day, Until: day.AddDate(0, 0, 1), Labels: []string{"urgent"}, Filter: expr, + }) + if err != nil { + t.Fatalf("GetMessagesByAgent: %v", err) + } + if got, want := listedIDs(messages), fixtureIDs(t, byKey, "alice"); !reflect.DeepEqual(got, want) { + t.Errorf("IDs=%v, want %v", got, want) + } +} + +func TestQFilterPropagatesEmissionErrorBeforeQuery(t *testing.T) { + sentinel := errors.New("emit failure") + registry, err := filterquery.NewRegistry(filterquery.FieldSpec{ + Name: "boom", Ops: []string{":"}, + Coerce: func(raw string, quoted bool) (any, error) { return raw, nil }, + Emit: func(*filterquery.Comparison, *filterquery.EmitCtx) (string, error) { return "", sentinel }, + }) + if err != nil { + t.Fatalf("NewRegistry: %v", err) + } + expr, err := filterquery.Parse(`boom:value`, registry) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + store := identity.NewStore(nil) + _, err = store.GetMessagesByAgent(context.Background(), identity.MessageListFilter{AgentID: "agent_not_queried", Filter: expr}) + if !errors.Is(err, sentinel) { + t.Fatalf("GetMessagesByAgent error = %v, want wrapped %v", err, sentinel) + } + if !strings.Contains(err.Error(), "emit message filter") { + t.Errorf("error = %q, want message-filter emission context", err) + } +} diff --git a/internal/identity/filter_registry.go b/internal/identity/filter_registry.go new file mode 100644 index 000000000..6700bcb45 --- /dev/null +++ b/internal/identity/filter_registry.go @@ -0,0 +1,134 @@ +package identity + +import ( + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/tokencanopy/e2a/internal/filterquery" +) + +// Everything messages-specific lives in this file — internal/filterquery +// stays schema-agnostic. +var ( + qRegistryOnce sync.Once + qRegistry *filterquery.Registry +) + +// MessagesFilterRegistry returns the shared field registry for list_messages filters. +func MessagesFilterRegistry() *filterquery.Registry { + qRegistryOnce.Do(func() { + reg, err := filterquery.NewRegistry( + labelQField(), + fromQField(), + subjectQField(), + createdQField(), + ) + if err != nil { + panic("filterquery: static messages registry is invalid: " + err.Error()) + } + qRegistry = reg + }) + return qRegistry +} + +var qLabelRe = regexp.MustCompile(`^[a-z0-9:_-]{1,64}$`) + +func labelQField() filterquery.FieldSpec { + return filterquery.FieldSpec{ + Name: "label", + Ops: []string{":"}, + Coerce: func(raw string, quoted bool) (any, error) { + if !qLabelRe.MatchString(raw) { + return nil, fmt.Errorf("labels must match [a-z0-9:_-]+ (max 64 chars), got %q", raw) + } + return raw, nil + }, + Emit: func(c *filterquery.Comparison, e *filterquery.EmitCtx) (string, error) { + return "m.labels @> " + e.PH([]string{c.Value.(string)}), nil + }, + } +} + +// likeSubstring builds the ILIKE pattern shared by from:/subject:. '*' maps +// to '%'; literal %, _, and \\ are escaped with the flat-filter helper first. +func likeSubstring(v string) string { + return "%" + strings.ReplaceAll(escapeLikePattern(v), "*", "%") + "%" +} + +func textQField(name, column string, maxLen int) filterquery.FieldSpec { + return filterquery.FieldSpec{ + Name: name, + Ops: []string{":", "=", "!="}, + Coerce: func(raw string, quoted bool) (any, error) { + if raw == "" { + return nil, fmt.Errorf("empty %s value", name) + } + if len(raw) > maxLen { + return nil, fmt.Errorf("%s filter too long (max %d bytes)", name, maxLen) + } + return raw, nil + }, + Emit: func(c *filterquery.Comparison, e *filterquery.EmitCtx) (string, error) { + v := c.Value.(string) + switch c.Op { + case ":": + return column + ` ILIKE ` + e.PH(likeSubstring(v)) + ` ESCAPE '\'`, nil + case "=": + return "LOWER(" + column + ") = LOWER(" + e.PH(v) + ")", nil + default: // "!=" + return "LOWER(" + column + ") != LOWER(" + e.PH(v) + ")", nil + } + }, + } +} + +func fromQField() filterquery.FieldSpec { return textQField("from", "m.sender", 200) } +func subjectQField() filterquery.FieldSpec { return textQField("subject", "m.subject", 200) } + +// createdValue carries date-coercion semantics: a date-only input (dayRange) +// makes comparisons cover the entire UTC calendar day. +type createdValue struct { + at time.Time + dayRange bool +} + +func createdQField() filterquery.FieldSpec { + return filterquery.FieldSpec{ + Name: "created", + Ops: []string{"=", "!=", "<", "<=", ">", ">="}, + Coerce: func(raw string, quoted bool) (any, error) { + if ts, err := time.Parse(time.RFC3339, raw); err == nil { + return createdValue{at: ts}, nil + } + if d, err := time.Parse("2006-01-02", raw); err == nil { + return createdValue{at: d, dayRange: true}, nil + } + return nil, fmt.Errorf("expected RFC3339 or YYYY-MM-DD, got %q", raw) + }, + Emit: func(c *filterquery.Comparison, e *filterquery.EmitCtx) (string, error) { + v := c.Value.(createdValue) + if !v.dayRange { + return "m.created_at " + c.Op + " " + e.PH(v.at), nil + } + + end := v.at.AddDate(0, 0, 1) + switch c.Op { + case "=": + return "(m.created_at >= " + e.PH(v.at) + " AND m.created_at < " + e.PH(end) + ")", nil + case "!=": + return "(m.created_at < " + e.PH(v.at) + " OR m.created_at >= " + e.PH(end) + ")", nil + case "<": + return "m.created_at < " + e.PH(v.at), nil + case "<=": + return "m.created_at < " + e.PH(end), nil + case ">": + return "m.created_at >= " + e.PH(end), nil + default: // ">=" + return "m.created_at >= " + e.PH(v.at), nil + } + }, + } +} diff --git a/internal/identity/filter_registry_test.go b/internal/identity/filter_registry_test.go new file mode 100644 index 000000000..0031067d6 --- /dev/null +++ b/internal/identity/filter_registry_test.go @@ -0,0 +1,191 @@ +package identity + +import ( + "reflect" + "strings" + "testing" + "time" + + "github.com/tokencanopy/e2a/internal/filterquery" +) + +func compileFilter(t *testing.T, filter string, start int) (string, []any) { + t.Helper() + frag, args, err := filterquery.Compile(filter, MessagesFilterRegistry(), filterquery.PostgresDialect{}, start) + if err != nil { + t.Fatalf("Compile(%q): %v", filter, err) + } + return frag, args +} + +func TestMessagesFilterRegistrySingleton(t *testing.T) { + t.Parallel() + + const callers = 32 + registries := make(chan *filterquery.Registry, callers) + for range callers { + go func() { registries <- MessagesFilterRegistry() }() + } + + first := <-registries + for range callers - 1 { + if got := <-registries; got != first { + t.Fatal("MessagesFilterRegistry returned distinct registries") + } + } +} + +func TestLabelField(t *testing.T) { + t.Parallel() + + frag, args := compileFilter(t, `label:urgent`, 1) + if frag != `(m.labels @> $1)` || !reflect.DeepEqual(args, []any{[]string{"urgent"}}) { + t.Errorf("frag=%s args=%v", frag, args) + } + if _, _, err := filterquery.Compile(`label:UPPER`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1); err == nil { + t.Error("uppercase label: want rejection (charset)") + } + if _, _, err := filterquery.Compile(`label = "urgent"`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1); err == nil { + t.Error("label=: want operator rejection") + } + if _, _, err := filterquery.Compile(`label:e2a:held`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1); err != nil { + t.Errorf("system label filter should work: %v", err) + } +} + +func TestFromFieldMatchesFlatParam(t *testing.T) { + t.Parallel() + + frag, args := compileFilter(t, `from:alice@x.com`, 1) + if frag != `(m.sender ILIKE $1 ESCAPE '\')` { + t.Errorf("frag = %s", frag) + } + if !reflect.DeepEqual(args, []any{"%alice@x.com%"}) { + t.Errorf("args = %v", args) + } + + frag, args = compileFilter(t, `from:"*@x_%.com\\tail"`, 1) + if !reflect.DeepEqual(args, []any{`%%@x\_\%.com\\tail%`}) { + t.Errorf("wildcard args = %v", args) + } + + for _, op := range []string{"=", "!="} { + frag, args = compileFilter(t, `from `+op+` "a*b%_"`, 1) + wantFrag := `(LOWER(m.sender) ` + op + ` LOWER($1))` + if frag != wantFrag || !reflect.DeepEqual(args, []any{"a*b%_"}) { + t.Errorf("exact %s: frag=%s args=%v", op, frag, args) + } + } +} + +func TestTextFieldLengthBounds(t *testing.T) { + t.Parallel() + + for _, field := range []string{"from", "subject"} { + field := field + t.Run(field, func(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + value string + valid bool + }{ + {"200 ASCII bytes", strings.Repeat("a", 200), true}, + {"201 ASCII bytes", strings.Repeat("a", 201), false}, + {"200 UTF-8 bytes", strings.Repeat("é", 100), true}, + {"202 UTF-8 bytes", strings.Repeat("é", 101), false}, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + _, _, err := filterquery.Compile(field+`:"`+tc.value+`"`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1) + if tc.valid && err != nil { + t.Fatalf("%s %s value: %v", tc.name, field, err) + } + if !tc.valid && err == nil { + t.Errorf("%s %s value: want rejection", tc.name, field) + } + }) + } + }) + } +} + +func TestSubjectField(t *testing.T) { + t.Parallel() + + frag, args := compileFilter(t, `subject:quarterly`, 1) + if frag != `(m.subject ILIKE $1 ESCAPE '\')` || !reflect.DeepEqual(args, []any{"%quarterly%"}) { + t.Errorf("frag=%s args=%v", frag, args) + } + + frag, args = compileFilter(t, `subject:"*@x_%.com\\tail"`, 1) + if !reflect.DeepEqual(args, []any{`%%@x\_\%.com\\tail%`}) { + t.Errorf("wildcard args = %v", args) + } + + for _, op := range []string{"=", "!="} { + frag, args = compileFilter(t, `subject `+op+` "a*b%_"`, 1) + wantFrag := `(LOWER(m.subject) ` + op + ` LOWER($1))` + if frag != wantFrag || !reflect.DeepEqual(args, []any{"a*b%_"}) { + t.Errorf("exact %s: frag=%s args=%v", op, frag, args) + } + } +} + +func TestHasAttachmentIsDeferred(t *testing.T) { + t.Parallel() + + _, _, err := filterquery.Compile(`has:attachment`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1) + if err == nil { + t.Fatal("has:attachment: want unknown-field rejection") + } + if got, want := err.Error(), `unknown field "has" — supported fields: created, from, label, subject`; !strings.Contains(got, want) { + t.Errorf("error = %q, want substring %q", got, want) + } +} + +func TestCreatedField(t *testing.T) { + t.Parallel() + + start := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 0, 1) + for _, tc := range []struct { + name string + q string + frag string + args []any + }{ + {"equal", `created = "2026-07-01"`, `((m.created_at >= $1 AND m.created_at < $2))`, []any{start, end}}, + {"not equal", `created != "2026-07-01"`, `((m.created_at < $1 OR m.created_at >= $2))`, []any{start, end}}, + {"before", `created<2026-07-01`, `(m.created_at < $1)`, []any{start}}, + {"through day", `created<=2026-07-01`, `(m.created_at < $1)`, []any{end}}, + {"after day", `created>2026-07-01`, `(m.created_at >= $1)`, []any{end}}, + {"on or after", `created>=2026-07-01`, `(m.created_at >= $1)`, []any{start}}, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + frag, args := compileFilter(t, tc.q, 1) + if frag != tc.frag || !reflect.DeepEqual(args, tc.args) { + t.Errorf("frag=%s args=%v, want frag=%s args=%v", frag, args, tc.frag, tc.args) + } + }) + } + + ts := "2026-07-25T10:30:00Z" + frag, args := compileFilter(t, `created<`+ts, 1) + want, err := time.Parse(time.RFC3339, ts) + if err != nil { + t.Fatal(err) + } + if frag != `(m.created_at < $1)` || !reflect.DeepEqual(args, []any{want}) { + t.Errorf("rfc3339 frag=%s args=%v", frag, args) + } + frag, args = compileFilter(t, `created=`+ts, 1) + if frag != `(m.created_at = $1)` || !reflect.DeepEqual(args, []any{want}) { + t.Errorf("rfc3339 equality frag=%s args=%v", frag, args) + } + if _, _, err := filterquery.Compile(`created>yesterday`, MessagesFilterRegistry(), filterquery.PostgresDialect{}, 1); err == nil { + t.Error("bad date: want rejection") + } +} diff --git a/internal/identity/store.go b/internal/identity/store.go index f40c07ef7..3a11c7f3e 100644 --- a/internal/identity/store.go +++ b/internal/identity/store.go @@ -20,6 +20,7 @@ import ( "github.com/tokencanopy/e2a/internal/dkim" "github.com/tokencanopy/e2a/internal/emailauth" "github.com/tokencanopy/e2a/internal/eventpayload" + "github.com/tokencanopy/e2a/internal/filterquery" "github.com/tokencanopy/e2a/internal/inboundpolicy" "github.com/tokencanopy/e2a/internal/messagelifecycle" "golang.org/x/net/idna" @@ -3676,6 +3677,9 @@ type MessageListFilter struct { // rows. Handler-layer validates each entry against the same charset // rule used on writes so callers can't smuggle SQL through here. Labels []string + // Filter is a parsed and registry-validated expression. The store emits it + // after built-in filters so placeholder numbering remains contiguous. + Filter *filterquery.Expr // Deleted flips the query to the TRASH view: only soft-deleted rows. // False (default) lists indefinitely retained live rows only. Deleted bool @@ -3804,6 +3808,19 @@ func (s *Store) GetMessagesByAgent(ctx context.Context, f MessageListFilter) ([] query += fmt.Sprintf(` AND m.labels @> $%d`, len(args)+1) args = append(args, f.Labels) } + if f.Filter != nil { + fragment, filterArgs, err := f.Filter.Emit( + filterquery.PostgresDialect{}, + len(args)+1, + ) + if err != nil { + return nil, fmt.Errorf("emit message filter: %w", err) + } + if fragment != "" { + query += " AND " + fragment + args = append(args, filterArgs...) + } + } cursorCmp := ">" sortDir := "ASC" diff --git a/mcp/src/client.ts b/mcp/src/client.ts index c736b887c..d66cd3df0 100644 --- a/mcp/src/client.ts +++ b/mcp/src/client.ts @@ -46,12 +46,20 @@ import type { DeleteApiKeyResult, Page, PageMessageLifecycleTransition, + ListMessagesParams, } from "@e2a/sdk/v1"; import type { McpConfig } from "./config.js"; import type { Scope } from "./tools/tiers.js"; const DEFAULT_LIST_LIMIT = 1000; +// Keep the SDK-owned message-filter contract in one place. MCP adds only the +// pagination cursor (consumed by AutoPager) and the agent-address selector. +type McpListMessagesParams = ListMessagesParams & { + cursor?: string; + explicitAddress?: string; +}; + /** Per-call options for unsafe writes. */ export interface SendOpts { idempotencyKey?: string; @@ -273,23 +281,14 @@ export class McpClient { // Cursor pagination (§6a #3): returns ONE page + next_cursor. `limit` is the // page size; pass a prior response's next_cursor as `cursor` for the next page. - listMessages(params: { - direction?: "inbound" | "outbound" | "all"; - readStatus?: "unread" | "read" | "all"; - sort?: "asc" | "desc"; - from_?: string; - subjectContains?: string; - conversationId?: string; - since?: string; - until?: string; - labels?: Array; - cursor?: string; - limit?: number; - deleted?: boolean; - explicitAddress?: string; - }): Promise> { - const { explicitAddress, cursor, ...rest } = params; - return this.sdk.messages.list(this.resolveAddress(explicitAddress), rest).page(cursor); + listMessages(params: McpListMessagesParams): Promise> { + const { explicitAddress, cursor, filter, ...rest } = params; + return this.sdk.messages + .list(this.resolveAddress(explicitAddress), { + ...rest, + ...(filter !== undefined ? { filter } : {}), + }) + .page(cursor); } restoreMessage(messageId: string, explicitAddress?: string): Promise { diff --git a/mcp/src/tools/messages.ts b/mcp/src/tools/messages.ts index e349f2c60..7a2042bc1 100644 --- a/mcp/src/tools/messages.ts +++ b/mcp/src/tools/messages.ts @@ -385,7 +385,7 @@ export function registerMessageTools(server: McpServer, client: McpClient): void title: "List messages (inbox or sent)", annotations: { readOnlyHint: true }, description: - "List the agent's messages, newest first by default. Use `direction` to pick the folder: `inbound` (the Inbox — received mail, the default), `outbound` (the Sent folder — mail this agent sent, including held drafts), or `all` (both). Filter received mail by `read_status` (unread/read/all; default unread — applies to inbound only; sent mail has no read-state). **Cursor-paginated:** returns one page in `messages` plus a `next_cursor` when more remain — pass it back as `cursor` for the next page (keep the same filters + sort). Pass `sort: \"asc\"` for FIFO order (oldest first) to drain in arrival order. **Search filters** (`from_`, `subject_contains`, `conversation_id`, `since`, `until`) narrow server-side — use them instead of paging the whole folder. In inbound summaries, `header_from` is the claimed RFC 5322 From address; a non-null `verified_domain` means DMARC passed for that From domain. It does not authenticate the mailbox local part, a person, or message content. Returns summaries only — use `get_message` for the full body and SPF/DKIM/DMARC evidence.", + "List the agent's messages, newest first by default. Use `direction` to pick the folder: `inbound` (the Inbox — received mail, the default), `outbound` (the Sent folder — mail this agent sent, including held drafts), or `all` (both). Filter received mail by `read_status` (unread/read/all; default unread — applies to inbound only; sent mail has no read-state). **Cursor-paginated:** returns one page in `messages` plus a `next_cursor` when more remain — pass it back as `cursor` for the next page (keep the same filters + sort). Pass `sort: \"asc\"` for FIFO order (oldest first) to drain in arrival order. **Search filters** (`from_`, `subject_contains`, `conversation_id`, `since`, `until`, `filter` (boolean expression)) narrow server-side — use them instead of paging the whole folder. In inbound summaries, `header_from` is the claimed RFC 5322 From address; a non-null `verified_domain` means DMARC passed for that From domain. It does not authenticate the mailbox local part, a person, or message content. Returns summaries only — use `get_message` for the full body and SPF/DKIM/DMARC evidence.", inputSchema: strictInputSchema({ direction: z .enum(["inbound", "outbound", "all"]) @@ -442,6 +442,15 @@ export function registerMessageTools(server: McpServer, client: McpClient): void .boolean() .optional() .describe("List the message trash instead of live messages."), + filter: z + .string() + .refine((value) => [...value].length <= 500, { + message: "filter must contain at most 500 Unicode code points", + }) + .optional() + .describe( + "Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators : = != < <= > >= with AND/OR/NOT and parentheses; whitespace is implicit AND (binds looser than OR). Example: 'label:urgent OR (from:alerts AND NOT subject:newsletter) created>=2026-07-01'. Composes (AND) with the flat filters (labels, from_, subject_contains, since, until). Invalid expressions are rejected with a positioned invalid_filter error.", + ), email: emailSelector, }), }, @@ -464,6 +473,7 @@ export function registerMessageTools(server: McpServer, client: McpClient): void ...(args.until !== undefined ? { until: args.until } : {}), ...(args.labels !== undefined ? { labels: args.labels } : {}), ...(args.deleted !== undefined ? { deleted: args.deleted } : {}), + ...(args.filter !== undefined ? { filter: args.filter } : {}), ...(args.email !== undefined ? { explicitAddress: args.email } : {}), }); return { messages: page.items, ...(page.next_cursor ? { next_cursor: page.next_cursor } : {}) }; diff --git a/mcp/tests/client.types.ts b/mcp/tests/client.types.ts index 7fae3befd..f128c2b66 100644 --- a/mcp/tests/client.types.ts +++ b/mcp/tests/client.types.ts @@ -6,6 +6,12 @@ type ListMessagesParams = Parameters[0]; const senderFilter: ListMessagesParams = { from_: "alice@example.com" }; void senderFilter; +// The MCP wrapper must preserve generated SDK filter expressions while adding only its +// cursor/address transport fields. This assignment fails to typecheck if the +// handwritten wrapper parameter shape drifts from the SDK again. +const expressionFilter: ListMessagesParams = { filter: "label:urgent" }; +void expressionFilter; + // The MCP tool and its wrapper follow the SDK's reserved-word-safe spelling. // @ts-expect-error `from` is the REST wire name, not the MCP input name. const removedSenderFilter: ListMessagesParams = { from: "alice@example.com" }; diff --git a/mcp/tests/list-messages-filter.test.ts b/mcp/tests/list-messages-filter.test.ts new file mode 100644 index 000000000..821c7c2d4 --- /dev/null +++ b/mcp/tests/list-messages-filter.test.ts @@ -0,0 +1,76 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { McpClient } from "../src/client.js"; +import { buildServer } from "../src/server.js"; + +function makeStubClient() { + const client = new McpClient({} as never, "bot@example.com", "account"); + const listMessages = vi + .spyOn(client, "listMessages") + .mockResolvedValue({ items: [], next_cursor: undefined }); + return { client, listMessages }; +} + +async function connect(stub: McpClient): Promise { + const server = buildServer({ client: stub, version: "0.0.0-test" }); + const client = new Client({ name: "list-messages-filter-test", version: "0.0.0" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + return client; +} + +describe("list_messages filter", () => { + let stub: ReturnType; + let client: Client; + + beforeEach(async () => { + stub = makeStubClient(); + client = await connect(stub.client); + }); + + it("documents only the v1 filter fields", async () => { + const { tools } = await client.listTools(); + const tool = tools.find((candidate) => candidate.name === "list_messages"); + const properties = (tool?.inputSchema as { + properties?: Record; + }).properties; + const filter = properties?.filter; + + expect(filter?.description).toContain("v1 fields: label, from, subject, created."); + expect(filter?.description).not.toContain("has"); + }); + + it("accepts filter at 500 Unicode code points and rejects 501", async () => { + const accepted = ["a".repeat(500), "😀".repeat(500)]; + const rejected = ["a".repeat(501), "😀".repeat(501)]; + + for (const filter of accepted) { + expect(Array.from(filter)).toHaveLength(500); + const result = await client.callTool({ name: "list_messages", arguments: { filter } }); + expect(result.isError).not.toBe(true); + expect(stub.listMessages).toHaveBeenLastCalledWith({ filter }); + } + + for (const filter of rejected) { + expect(Array.from(filter)).toHaveLength(501); + const result = await client.callTool({ name: "list_messages", arguments: { filter } }); + expect(result.isError).toBe(true); + expect((result.content as Array<{ text: string }>)[0]?.text).toMatch(/500 Unicode code points/i); + } + }); + + it("passes filter through verbatim and omits it when absent", async () => { + const filter = "label:urgent"; + await client.callTool({ name: "list_messages", arguments: { filter } }); + expect(stub.listMessages).toHaveBeenLastCalledWith( + expect.objectContaining({ filter: "label:urgent" }), + ); + + stub.listMessages.mockClear(); + await client.callTool({ name: "list_messages", arguments: {} }); + const params = stub.listMessages.mock.calls[0]?.[0]; + expect(params).toEqual({}); + expect(params).not.toHaveProperty("filter"); + }); +}); diff --git a/sdks/python/scripts/generate-oag.sh b/sdks/python/scripts/generate-oag.sh index 1fe22f828..ec930b2a3 100644 --- a/sdks/python/scripts/generate-oag.sh +++ b/sdks/python/scripts/generate-oag.sh @@ -73,6 +73,12 @@ perl -0pi -e 's/\n+\z/\n/' "$DEST/models/event_envelope.py" # Keep the expanded agents surface deterministic and diff-check clean. perl -pi -e 's/[ \t]+$//' "$DEST/api/agents_api.py" +# The newly appended message filter query block otherwise receives whitespace- +# only blank lines from OpenAPI Generator. Normalize this exact block without +# rewriting the historical generated API file. +perl -0pi -e 's/( if filter is not None:\n)[ \t]*\n( _query_params\.append\(\(\x27filter\x27, filter\)\)\n)[ \t]*\n/$1\n$2\n/' \ + "$DEST/api/messages_api.py" + perl -0pi -e 's/\n+\z/\n/' \ "$DEST/models/agent_suppression_added_data.py" \ "$DEST/models/agent_suppression_view.py" \ diff --git a/sdks/python/src/e2a/v1/client.py b/sdks/python/src/e2a/v1/client.py index e0cbae329..c9e0baea6 100644 --- a/sdks/python/src/e2a/v1/client.py +++ b/sdks/python/src/e2a/v1/client.py @@ -452,6 +452,7 @@ def list( until: Optional[str] = None, limit: Optional[int] = None, deleted: Optional[bool] = None, + filter: Optional[str] = None, ) -> AutoPager[MessageSummaryView]: # `from` is a Python keyword; the generator is configured (via # --name-mappings/--parameter-name-mappings in generate-oag.sh) to expose @@ -473,6 +474,7 @@ async def fetch(cursor: Optional[str]) -> Page: cursor=cursor, limit=limit, deleted=deleted, + filter=filter, _headers=h, ) ) diff --git a/sdks/python/src/e2a/v1/generated/api/messages_api.py b/sdks/python/src/e2a/v1/generated/api/messages_api.py index 7fdd0fd9c..d747baaa4 100644 --- a/sdks/python/src/e2a/v1/generated/api/messages_api.py +++ b/sdks/python/src/e2a/v1/generated/api/messages_api.py @@ -1629,6 +1629,7 @@ async def list_messages( cursor: Optional[StrictStr] = None, limit: Optional[Annotated[int, Field(le=100, strict=True, ge=1)]] = None, deleted: Annotated[Optional[StrictBool], Field(description="List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only).")] = None, + filter: Annotated[Optional[StrictStr], Field(description="Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars.")] = None, _request_timeout: Union[ None, Annotated[StrictFloat, Field(gt=0)], @@ -1672,6 +1673,8 @@ async def list_messages( :type limit: int :param deleted: List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). :type deleted: bool + :param filter: Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. + :type filter: str :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of @@ -1708,6 +1711,7 @@ async def list_messages( cursor=cursor, limit=limit, deleted=deleted, + filter=filter, _request_auth=_request_auth, _content_type=_content_type, _headers=_headers, @@ -1744,6 +1748,7 @@ async def list_messages_with_http_info( cursor: Optional[StrictStr] = None, limit: Optional[Annotated[int, Field(le=100, strict=True, ge=1)]] = None, deleted: Annotated[Optional[StrictBool], Field(description="List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only).")] = None, + filter: Annotated[Optional[StrictStr], Field(description="Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars.")] = None, _request_timeout: Union[ None, Annotated[StrictFloat, Field(gt=0)], @@ -1787,6 +1792,8 @@ async def list_messages_with_http_info( :type limit: int :param deleted: List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). :type deleted: bool + :param filter: Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. + :type filter: str :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of @@ -1823,6 +1830,7 @@ async def list_messages_with_http_info( cursor=cursor, limit=limit, deleted=deleted, + filter=filter, _request_auth=_request_auth, _content_type=_content_type, _headers=_headers, @@ -1859,6 +1867,7 @@ async def list_messages_without_preload_content( cursor: Optional[StrictStr] = None, limit: Optional[Annotated[int, Field(le=100, strict=True, ge=1)]] = None, deleted: Annotated[Optional[StrictBool], Field(description="List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only).")] = None, + filter: Annotated[Optional[StrictStr], Field(description="Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars.")] = None, _request_timeout: Union[ None, Annotated[StrictFloat, Field(gt=0)], @@ -1902,6 +1911,8 @@ async def list_messages_without_preload_content( :type limit: int :param deleted: List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). :type deleted: bool + :param filter: Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. + :type filter: str :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of @@ -1938,6 +1949,7 @@ async def list_messages_without_preload_content( cursor=cursor, limit=limit, deleted=deleted, + filter=filter, _request_auth=_request_auth, _content_type=_content_type, _headers=_headers, @@ -1969,6 +1981,7 @@ def _list_messages_serialize( cursor, limit, deleted, + filter, _request_auth, _content_type, _headers, @@ -2042,6 +2055,10 @@ def _list_messages_serialize( _query_params.append(('deleted', deleted)) + if filter is not None: + + _query_params.append(('filter', filter)) + # process the header parameters # process the form parameters # process the body parameter diff --git a/sdks/python/tests/test_v1_client.py b/sdks/python/tests/test_v1_client.py index 4dbd2c34e..7c639bf9c 100644 --- a/sdks/python/tests/test_v1_client.py +++ b/sdks/python/tests/test_v1_client.py @@ -770,6 +770,19 @@ async def test_messages_list_threads_cursor(httpx_mock): assert "cursor=cur_2" in str(reqs[1].url) +@pytest.mark.anyio +async def test_messages_list_forwards_filter(httpx_mock): + httpx_mock.add_response(json={"items": [], "next_cursor": None}) + async with _client() as c: + await c.messages.list( + "bot@test.dev", deleted=True, filter="label:urgent" + ).to_list(limit=10) + request = httpx_mock.get_requests()[-1] + assert request.url.params["filter"] == "label:urgent" + assert "q" not in request.url.params + assert request.url.params["deleted"] == "true" + + @pytest.mark.anyio async def test_messages_get_lifecycle_uses_read_path_and_parses_contract(httpx_mock): httpx_mock.add_response( diff --git a/sdks/typescript/src/v1/client.ts b/sdks/typescript/src/v1/client.ts index 2b90494f7..d4708badd 100644 --- a/sdks/typescript/src/v1/client.ts +++ b/sdks/typescript/src/v1/client.ts @@ -345,6 +345,7 @@ export interface ListMessagesParams { limit?: number; /** List soft-deleted messages in the trash instead of live messages. */ deleted?: boolean; + filter?: string; } class MessagesResource { @@ -352,11 +353,22 @@ class MessagesResource { list(email: string, params: ListMessagesParams = {}): AutoPager { return new AutoPager(async (cursor) => { - const page = await call(() => - this.api.listMessages(email, params.direction, params.readStatus, params.sort, params.from_, - params.subjectContains, params.conversationId, params.labels, params.since, params.until, - cursor, params.limit, params.deleted), - ); + const page = await call(() => this.api.listMessages( + email, + params.direction, + params.readStatus, + params.sort, + params.from_, + params.subjectContains, + params.conversationId, + params.labels, + params.since, + params.until, + cursor, + params.limit, + params.deleted, + params.filter, + )); return { items: page.items ?? [], next_cursor: page.nextCursor }; }); } diff --git a/sdks/typescript/src/v1/generated/apis/MessagesApi.ts b/sdks/typescript/src/v1/generated/apis/MessagesApi.ts index 4ef4edc84..06a6de674 100644 --- a/sdks/typescript/src/v1/generated/apis/MessagesApi.ts +++ b/sdks/typescript/src/v1/generated/apis/MessagesApi.ts @@ -347,8 +347,9 @@ export class MessagesApiRequestFactory extends BaseAPIRequestFactory { * @param cursor * @param limit * @param deleted List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). + * @param filter Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. */ - public async listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, _options?: Configuration): Promise { + public async listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, filter?: string, _options?: Configuration): Promise { let _config = _options || this.configuration; // verify required parameter 'email' is not null or undefined @@ -369,6 +370,7 @@ export class MessagesApiRequestFactory extends BaseAPIRequestFactory { + // Path Params const localVarPath = '/v1/agents/{email}/messages' .replace('{' + 'email' + '}', encodeURIComponent(String(email))); @@ -437,6 +439,11 @@ export class MessagesApiRequestFactory extends BaseAPIRequestFactory { requestContext.setQueryParam("deleted", ObjectSerializer.serialize(deleted, "boolean", "")); } + // Query Params + if (filter !== undefined) { + requestContext.setQueryParam("filter", ObjectSerializer.serialize(filter, "string", "")); + } + let authMethod: SecurityAuthentication | undefined; // Apply auth methods diff --git a/sdks/typescript/src/v1/generated/types/ObjectParamAPI.ts b/sdks/typescript/src/v1/generated/types/ObjectParamAPI.ts index 04f62e6d9..b3447cb04 100644 --- a/sdks/typescript/src/v1/generated/types/ObjectParamAPI.ts +++ b/sdks/typescript/src/v1/generated/types/ObjectParamAPI.ts @@ -1490,6 +1490,13 @@ export interface MessagesApiListMessagesRequest { * @memberof MessagesApilistMessages */ deleted?: boolean + /** + * Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. + * Defaults to: undefined + * @type string + * @memberof MessagesApilistMessages + */ + filter?: string } export interface MessagesApiReplyToMessageRequest { @@ -1702,7 +1709,7 @@ export class ObjectMessagesApi { * @param param the request object */ public listMessagesWithHttpInfo(param: MessagesApiListMessagesRequest, options?: ConfigurationOptions): Promise> { - return this.api.listMessagesWithHttpInfo(param.email, param.direction, param.readStatus, param.sort, param.from_, param.subjectContains, param.conversationId, param.labels, param.since, param.until, param.cursor, param.limit, param.deleted, options).toPromise(); + return this.api.listMessagesWithHttpInfo(param.email, param.direction, param.readStatus, param.sort, param.from_, param.subjectContains, param.conversationId, param.labels, param.since, param.until, param.cursor, param.limit, param.deleted, param.filter, options).toPromise(); } /** @@ -1711,7 +1718,7 @@ export class ObjectMessagesApi { * @param param the request object */ public listMessages(param: MessagesApiListMessagesRequest, options?: ConfigurationOptions): Promise { - return this.api.listMessages(param.email, param.direction, param.readStatus, param.sort, param.from_, param.subjectContains, param.conversationId, param.labels, param.since, param.until, param.cursor, param.limit, param.deleted, options).toPromise(); + return this.api.listMessages(param.email, param.direction, param.readStatus, param.sort, param.from_, param.subjectContains, param.conversationId, param.labels, param.since, param.until, param.cursor, param.limit, param.deleted, param.filter, options).toPromise(); } /** diff --git a/sdks/typescript/src/v1/generated/types/ObservableAPI.ts b/sdks/typescript/src/v1/generated/types/ObservableAPI.ts index 622dad1ae..e0fa19ab0 100644 --- a/sdks/typescript/src/v1/generated/types/ObservableAPI.ts +++ b/sdks/typescript/src/v1/generated/types/ObservableAPI.ts @@ -1532,11 +1532,12 @@ export class ObservableMessagesApi { * @param [cursor] * @param [limit] * @param [deleted] List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). + * @param [filter] Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. */ - public listMessagesWithHttpInfo(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, _options?: ConfigurationOptions): Observable> { + public listMessagesWithHttpInfo(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, filter?: string, _options?: ConfigurationOptions): Observable> { const _config = mergeConfiguration(this.configuration, _options); - const requestContextPromise = this.requestFactory.listMessages(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, _config); + const requestContextPromise = this.requestFactory.listMessages(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, filter, _config); // build promise chain let middlewarePreObservable = from(requestContextPromise); for (const middleware of _config.middleware) { @@ -1569,9 +1570,10 @@ export class ObservableMessagesApi { * @param [cursor] * @param [limit] * @param [deleted] List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). + * @param [filter] Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. */ - public listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, _options?: ConfigurationOptions): Observable { - return this.listMessagesWithHttpInfo(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, _options).pipe(map((apiResponse: HttpInfo) => apiResponse.data)); + public listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, filter?: string, _options?: ConfigurationOptions): Observable { + return this.listMessagesWithHttpInfo(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, filter, _options).pipe(map((apiResponse: HttpInfo) => apiResponse.data)); } /** diff --git a/sdks/typescript/src/v1/generated/types/PromiseAPI.ts b/sdks/typescript/src/v1/generated/types/PromiseAPI.ts index 926f64583..d84cdd82f 100644 --- a/sdks/typescript/src/v1/generated/types/PromiseAPI.ts +++ b/sdks/typescript/src/v1/generated/types/PromiseAPI.ts @@ -1113,10 +1113,11 @@ export class PromiseMessagesApi { * @param [cursor] * @param [limit] * @param [deleted] List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). + * @param [filter] Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. */ - public listMessagesWithHttpInfo(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, _options?: PromiseConfigurationOptions): Promise> { + public listMessagesWithHttpInfo(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, filter?: string, _options?: PromiseConfigurationOptions): Promise> { const observableOptions = wrapOptions(_options); - const result = this.api.listMessagesWithHttpInfo(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, observableOptions); + const result = this.api.listMessagesWithHttpInfo(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, filter, observableOptions); return result.toPromise(); } @@ -1136,10 +1137,11 @@ export class PromiseMessagesApi { * @param [cursor] * @param [limit] * @param [deleted] List the trash instead: messages that were soft-deleted and are restorable until purged (30 days after deletion by default, deployment-configurable). Defaults to false (live messages only). + * @param [filter] Boolean filter expression (AIP-160-derived). v1 fields: label, from, subject, created. Operators: : = != < <= > >= with AND / OR / NOT and parentheses; whitespace is implicit AND and binds looser than OR. Composes with (ANDs) the flat filters. Unknown fields/operators are rejected with a positioned invalid_filter error. Max 500 chars. */ - public listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, _options?: PromiseConfigurationOptions): Promise { + public listMessages(email: string, direction?: 'inbound' | 'outbound' | 'all', readStatus?: 'unread' | 'read' | 'all', sort?: 'asc' | 'desc', from_?: string, subjectContains?: string, conversationId?: string, labels?: Array, since?: string, until?: string, cursor?: string, limit?: number, deleted?: boolean, filter?: string, _options?: PromiseConfigurationOptions): Promise { const observableOptions = wrapOptions(_options); - const result = this.api.listMessages(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, observableOptions); + const result = this.api.listMessages(email, direction, readStatus, sort, from_, subjectContains, conversationId, labels, since, until, cursor, limit, deleted, filter, observableOptions); return result.toPromise(); } diff --git a/sdks/typescript/test/v1/client.test.ts b/sdks/typescript/test/v1/client.test.ts index 61c496c9e..48bd1f7dc 100644 --- a/sdks/typescript/test/v1/client.test.ts +++ b/sdks/typescript/test/v1/client.test.ts @@ -407,6 +407,41 @@ describe("E2AClient", () => { expect(url.searchParams.has("from_")).toBe(false); }); + it("sends the structured filter after all existing list parameters", async () => { + globalThis.fetch = mockFetch(200, { items: [], next_cursor: null }); + const messages = client.messages as unknown as { + api: { listMessages: (...args: unknown[]) => Promise }; + }; + const listMessagesSpy = vi.spyOn(messages.api, "listMessages"); + + await client.messages.list("bot@test.dev", { + direction: "all", + deleted: true, + filter: "label:urgent", + }).toArray({ limit: 10 }); + + const url = new URL(lastCall().url); + expect(url.searchParams.get("filter")).toBe("label:urgent"); + expect(url.searchParams.has("q")).toBe(false); + expect(url.searchParams.get("deleted")).toBe("true"); + expect(listMessagesSpy).toHaveBeenCalledWith( + "bot@test.dev", + "all", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + true, + "label:urgent", + ); + }); + it("messages.list({ deleted: true }) lists the trash", async () => { globalThis.fetch = mockFetch(200, { items: [], next_cursor: null }); await client.messages.list("bot@test.dev", { deleted: true }).toArray({ limit: 10 }); diff --git a/tests/e2e-prod/suites/14-v030-features.test.ts b/tests/e2e-prod/suites/14-v030-features.test.ts index 8fa0368ef..dab9468fa 100644 --- a/tests/e2e-prod/suites/14-v030-features.test.ts +++ b/tests/e2e-prod/suites/14-v030-features.test.ts @@ -23,7 +23,8 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { ApiClient } from "../harness/client.ts"; -import { cleanup } from "../harness/cleanup.ts"; +import { cleanup, track, untrack } from "../harness/cleanup.ts"; +import { uniqueSlug, uniqueSubject } from "../harness/fixtures.ts"; import { info, warn, writeReport } from "../harness/report.ts"; const client = new ApiClient(); @@ -228,6 +229,73 @@ test("agent messages: ?labels= filter accepted without error", async () => { assert.equal(r.status, 200, `expected 200, got ${r.status}: ${r.raw.slice(0, 200)}`); }); +test("agent messages: filter OR union and AND NOT difference on isolated loopback messages", async () => { + // Do not use the shared primary inbox: this creates a throwaway agent, sends + // only to itself (the providerless loopback path), and deletes the agent in + // finally so its messages cascade away even if an assertion fails. + const email = `${uniqueSlug("filter")}@${client.env.sharedDomain}`; + const created = await client.post<{ email: string }>("/v1/agents", { + body: { email, name: "filter e2e" }, + expect: 201, + }); + const agentEmail = created.body!.email; + track("agent", agentEmail); + + try { + const receiveLoopback = async (subject: string): Promise => { + await client.post(`/v1/agents/${encodeURIComponent(agentEmail)}/messages`, { + body: { to: [agentEmail], subject, text: "filter loopback fixture" }, + expect: [200, 202], + }); + // The inbound copy can arrive asynchronously. Explicitly request all + // read states and a 100-item page so defaults, pagination, and unrelated + // messages cannot hide either unique fixture. + for (let attempt = 0; attempt < 12; attempt++) { + const listed = await client.get<{ items?: Array<{ id: string; subject?: string }> }>( + `/v1/agents/${encodeURIComponent(agentEmail)}/messages`, + { query: { direction: "inbound", read_status: "all", limit: 100 } }, + ); + assert.equal(listed.status, 200, `list loopback inbox: ${listed.status} ${listed.raw.slice(0, 200)}`); + const message = listed.body?.items?.find((item) => item.subject === subject); + if (message?.id) return message.id; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`loopback message ${JSON.stringify(subject)} did not arrive for ${agentEmail}`); + }; + + const idA = await receiveLoopback(uniqueSubject("filter a")); + const idB = await receiveLoopback(uniqueSubject("filter b")); + const labelA = uniqueSlug("filter-a"); + const labelB = uniqueSlug("filter-b"); + for (const [id, label] of [[idA, labelA], [idB, labelB]] as const) { + const updated = await client.patch<{ labels?: string[] }>( + `/v1/agents/${encodeURIComponent(agentEmail)}/messages/${encodeURIComponent(id)}`, + { body: { add_labels: [label] }, expect: 200 }, + ); + assert.ok(updated.body?.labels?.includes(label), `message ${id} received label ${label}`); + } + + const matchedIDs = async (filter: string): Promise => { + const listed = await client.get<{ items?: Array<{ id: string }> }>( + `/v1/agents/${encodeURIComponent(agentEmail)}/messages`, + { query: { filter, direction: "inbound", read_status: "all", limit: 100 }, expect: 200 }, + ); + return (listed.body?.items ?? []).map((item) => item.id).sort(); + }; + + assert.deepEqual(await matchedIDs(`label:${labelA} OR label:${labelB}`), [idA, idB].sort(), "filter OR returns the label union"); + assert.deepEqual(await matchedIDs(`label:${labelA} AND NOT label:${labelB}`), [idA], "filter AND NOT returns only label A"); + } finally { + const deleted = await client.delete(`/v1/agents/${encodeURIComponent(agentEmail)}?confirm=DELETE&permanent=true`); + const cleanupSucceeded = [200, 204, 404].includes(deleted.status); + assert.ok( + cleanupSucceeded, + `delete throwaway filter agent: ${deleted.status} ${deleted.raw.slice(0, 200)}`, + ); + if (cleanupSucceeded) untrack("agent", agentEmail); + } +}); + test("agent messages: ?labels= filter enforces the 50-value cap (400)", async () => { const email = client.env.primaryAgentEmail; // The ?labels= filter is capped server-side at 50 values (maxLabelsPerOp);