From ed34c725213f825c7d640df5a2e780c8d248893e Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 29 Jan 2026 20:32:01 +1100 Subject: [PATCH 1/3] docs: update documentation for encryptQuery API - Update all reference docs for encryptQuery API - Add searchableJson schema documentation - Update README files with new examples - Add searchable-json.test.ts for schema validation - Add changeset for version bump - Update helpers with new utility functions --- .changeset/searchable-json-query-api.md | 6 + .gitignore | 7 + AGENTS.md | 2 +- docs/README.md | 1 + .../aws-kms-vs-cipherstash-comparison.md | 7 +- docs/concepts/searchable-encryption.md | 11 +- docs/getting-started.md | 8 + docs/reference/model-operations.md | 4 +- docs/reference/schema.md | 34 +- .../searchable-encryption-postgres.md | 317 ++++++++++++++++-- docs/reference/supabase-sdk.md | 12 +- local/create-ci-table.sql | 3 +- packages/drizzle/README.md | 2 +- packages/protect-dynamodb/README.md | 14 +- packages/protect/README.md | 101 +++++- packages/protect/src/helpers/index.ts | 56 +++- packages/schema/__tests__/schema.test.ts | 4 +- .../schema/__tests__/searchable-json.test.ts | 41 +++ 18 files changed, 560 insertions(+), 70 deletions(-) create mode 100644 .changeset/searchable-json-query-api.md create mode 100644 packages/schema/__tests__/searchable-json.test.ts diff --git a/.changeset/searchable-json-query-api.md b/.changeset/searchable-json-query-api.md new file mode 100644 index 00000000..c543b8c5 --- /dev/null +++ b/.changeset/searchable-json-query-api.md @@ -0,0 +1,6 @@ +--- +"@cipherstash/protect": major +"@cipherstash/schema": major +--- + +Add searchable JSON query API with path and containment query support diff --git a/.gitignore b/.gitignore index fc7ee438..599d7e98 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,10 @@ mise.local.toml cipherstash.toml cipherstash.secret.toml sql/cipherstash-*.sql + +# work files +.claude/ +.serena/ +.work/ +**/.work/ +PR_REVIEW.md diff --git a/AGENTS.md b/AGENTS.md index a4d2b0c6..8860b8bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ If these variables are missing, tests that require live encryption will fail or - `encryptModel(model, table)` / `decryptModel(model)` - `bulkEncrypt(plaintexts[], { table, column })` / `bulkDecrypt(encrypted[])` - `bulkEncryptModels(models[], table)` / `bulkDecryptModels(models[])` - - `createSearchTerms(terms)` for searchable queries + - `encryptQuery(terms)` for searchable queries (note: `createSearchTerms` is deprecated, use `encryptQuery` instead) - **Identity-aware encryption**: Use `LockContext` from `@cipherstash/protect/identify` and chain `.withLockContext()` on operations. Same context must be used for both encrypt and decrypt. ## Critical Gotchas (read before coding) diff --git a/docs/README.md b/docs/README.md index 0e1192c9..19494b47 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ The documentation for Protect.js is organized into the following sections: ## Concepts - [Searchable encryption](./concepts/searchable-encryption.md) +- [Searchable JSON](./reference/schema.md#searchable-json) - Query encrypted JSON documents ## Reference diff --git a/docs/concepts/aws-kms-vs-cipherstash-comparison.md b/docs/concepts/aws-kms-vs-cipherstash-comparison.md index d0a52f19..86f6eeee 100644 --- a/docs/concepts/aws-kms-vs-cipherstash-comparison.md +++ b/docs/concepts/aws-kms-vs-cipherstash-comparison.md @@ -165,11 +165,12 @@ const encryptResult = await protectClient.encrypt( ); // Create search terms and query directly in PostgreSQL -const searchTerms = await protectClient.createSearchTerms({ - terms: ['secret'], +const searchTerms = await protectClient.encryptQuery([{ + value: 'secret', column: users.email, table: users, -}); + queryType: 'freeTextSearch', +}]); // Use with your ORM (Drizzle integration included) ``` diff --git a/docs/concepts/searchable-encryption.md b/docs/concepts/searchable-encryption.md index 56ca41fa..23dc4382 100644 --- a/docs/concepts/searchable-encryption.md +++ b/docs/concepts/searchable-encryption.md @@ -69,14 +69,16 @@ CipherStash uses [EQL](https://github.com/cipherstash/encrypt-query-language) to // 1) Encrypt the search term const searchTerm = 'alice.johnson@example.com' -const encryptedParam = await protectClient.createSearchTerms([{ +const encryptedParam = await protectClient.encryptQuery([{ value: searchTerm, table: protectedUsers, // Reference to the Protect table schema column: protectedUsers.email, // Your Protect column definition + queryType: 'equality', // Use 'equality' for exact match queries }]) if (encryptedParam.failure) { // Handle the failure + throw new Error(encryptedParam.failure.message) } // 2) Build an equality query noting that EQL must be installed in order for the operation to work successfully @@ -86,10 +88,9 @@ const equalitySQL = ` WHERE email = $1 ` -// 3) Execute the query, passing in the Postgres column name -// and the encrypted search term as the second parameter +// 3) Execute the query, passing in the encrypted search term // (client is an arbitrary Postgres client) -const result = await client.query(equalitySQL, [ protectedUser.email.getName(), encryptedParam.data ]) +const result = await client.query(equalitySQL, [encryptedParam.data[0]]) ``` Using the above approach, Protect.js is generating the EQL payloads and which means you never have to drop down to writing complex SQL queries. @@ -132,7 +133,7 @@ With searchable encryption, you can: With searchable encryption: - Data can be encrypted, stored, and searched in your existing PostgreSQL database. -- Encrypted data can be searched using equality, free text search, and range queries. +- Encrypted data can be searched using equality, free text search, range queries, and JSON path/containment queries. - Data remains encrypted, and will be decrypted using the Protect.js library in your application. - Queries are blazing fast, and won't slow down your application experience. - Every decryption event is logged, giving you an audit trail of data access events. diff --git a/docs/getting-started.md b/docs/getting-started.md index 84c154c0..a51f2414 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -254,6 +254,14 @@ CREATE TABLE users ( ); ``` +## Next steps + +Now that you have the basics working, explore these advanced features: + +- **[Searchable Encryption](./reference/searchable-encryption-postgres.md)** - Learn how to search encrypted data using `encryptQuery()` with PostgreSQL and EQL +- **[Model Operations](./reference/model-operations.md)** - Encrypt and decrypt entire objects with bulk operations +- **[Schema Configuration](./reference/schema.md)** - Configure indexes for equality, free text search, range queries, and JSON search + --- ### Didn't find what you wanted? diff --git a/docs/reference/model-operations.md b/docs/reference/model-operations.md index 5a241214..bf62d076 100644 --- a/docs/reference/model-operations.md +++ b/docs/reference/model-operations.md @@ -75,7 +75,7 @@ For better performance when working with multiple models, use these bulk encrypt ### Bulk encryption ```typescript -const users = [ +const usersList = [ { id: "1", email: "user1@example.com", @@ -88,7 +88,7 @@ const users = [ }, ]; -const encryptedResult = await protectClient.bulkEncryptModels(users, users); +const encryptedResult = await protectClient.bulkEncryptModels(usersList, usersSchema); if (encryptedResult.failure) { console.error("Bulk encryption failed:", encryptedResult.failure.message); diff --git a/docs/reference/schema.md b/docs/reference/schema.md index b828bdf4..9977cc39 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -76,9 +76,29 @@ export const protectedUsers = csTable("users", { }); ``` +### Searchable JSON + +To enable searching within JSON columns, use the `searchableJson()` method. This automatically sets the column data type to `json` and configures the necessary indexes for path and containment queries. + +```ts +import { csTable, csColumn } from "@cipherstash/protect"; + +export const protectedUsers = csTable("users", { + metadata: csColumn("metadata").searchableJson(), +}); +``` + +> [!WARNING] +> `searchableJson()` is mutually exclusive with other index types (`equality()`, `freeTextSearch()`, `orderAndRange()`) on the same column. Combining them will result in runtime errors. This is enforced by the encryption backend, not at the TypeScript type level. + + ### Nested objects -Protect.js supports nested objects in your schema, allowing you to encrypt **but not search on** nested properties. You can define nested objects up to 3 levels deep. +Protect.js supports nested objects in your schema, allowing you to encrypt nested properties. You can define nested objects up to 3 levels deep using `csValue`. For **searchable** JSON data, use `.searchableJson()` on a JSON column instead. + +> [!TIP] +> If you need to search within JSON data, use `.searchableJson()` on the column instead of nested `csValue` definitions. See [Searchable JSON](#searchable-json) above. + This is useful for data stores that have less structured data, like NoSQL databases. You can define nested objects by using the `csValue` function to define a value in a nested object. The value naming convention of the `csValue` function is a dot-separated string of the nested object path, e.g. `profile.name` or `profile.address.street`. @@ -105,15 +125,15 @@ export const protectedUsers = csTable("users", { ``` When working with nested objects: -- Searchable encryption is not supported on nested objects +- Searchable encryption is not supported on nested `csValue` objects (use `.searchableJson()` for searchable JSON) - Each level can have its own encrypted fields - The maximum nesting depth is 3 levels - Null and undefined values are supported at any level - Optional nested objects are supported > [!WARNING] -> TODO: The schema builder does not validate the values you supply to the `csValue` or `csColumn` functions. -> These values are meant to be unique, and and cause unexpected behavior if they are not defined correctly. +> The schema builder does not currently validate the values you supply to the `csValue` or `csColumn` functions. +> These values must be unique within your schema - duplicate values may cause unexpected behavior. ## Available index options @@ -124,8 +144,12 @@ The following index options are available for your schema: | equality | Enables a exact index for equality queries. | `WHERE email = 'example@example.com'` | | freeTextSearch | Enables a match index for free text queries. | `WHERE description LIKE '%example%'` | | orderAndRange | Enables an sorting and range queries index. | `ORDER BY price ASC` | +| searchableJson | Enables searching inside JSON columns. | `WHERE data->'user'->>'email' = '...'` | -You can chain these methods to your column to configure them in any combination. +You can chain `equality()`, `freeTextSearch()`, and `orderAndRange()` methods in any combination. + +> [!WARNING] +> `searchableJson()` is **mutually exclusive** with other index types. Do not combine `searchableJson()` with `equality()`, `freeTextSearch()`, or `orderAndRange()` on the same column. ## Initializing the Protect client diff --git a/docs/reference/searchable-encryption-postgres.md b/docs/reference/searchable-encryption-postgres.md index 74ead6a4..12fcee79 100644 --- a/docs/reference/searchable-encryption-postgres.md +++ b/docs/reference/searchable-encryption-postgres.md @@ -7,6 +7,11 @@ This reference guide outlines the different query patterns you can use to search - [Prerequisites](#prerequisites) - [What is EQL?](#what-is-eql) - [Setting up your schema](#setting-up-your-schema) +- [Deprecated Functions](#deprecated-functions) +- [Unified Query Encryption API](#unified-query-encryption-api) +- [JSON Search](#json-search) + - [Creating JSON Search Terms](#creating-json-search-terms) + - [Using JSON Search Terms in PostgreSQL](#using-json-search-terms-in-postgresql) - [Search capabilities](#search-capabilities) - [Exact matching](#exact-matching) - [Free text search](#free-text-search) @@ -15,7 +20,6 @@ This reference guide outlines the different query patterns you can use to search - [Using Raw PostgreSQL Client (pg)](#using-raw-postgresql-client-pg) - [Using Supabase SDK](#using-supabase-sdk) - [Best practices](#best-practices) -- [Common use cases](#common-use-cases) ## Prerequisites @@ -60,49 +64,304 @@ const schema = csTable('users', { }) ``` -## The `createSearchTerms` function +## Deprecated Functions -The `createSearchTerms` function is used to create search terms used in the SQL query. - -The function takes an array of objects, each with the following properties: - -| Property | Description | -|----------|-------------| -| `value` | The value to search for | -| `column` | The column to search in | -| `table` | The table to search in | -| `returnType` | The type of return value to expect from the SQL query. Required for PostgreSQL composite types. | - -**Return types:** +> [!WARNING] +> The `createSearchTerms` and `createQuerySearchTerms` functions are deprecated and will be removed in v2.0. Use the unified `encryptQuery` function instead. See [Unified Query Encryption API](#unified-query-encryption-api). -- `eql` (default) - EQL encrypted payload -- `composite-literal` - EQL encrypted payload wrapped in a composite literal -- `escaped-composite-literal` - EQL encrypted payload wrapped in an escaped composite literal +### `createSearchTerms` (deprecated) -Example: +The `createSearchTerms` function was the original API for creating search terms. It has been superseded by `encryptQuery`. ```typescript +// DEPRECATED - use encryptQuery instead const term = await protectClient.createSearchTerms([{ value: 'user@example.com', column: schema.email, table: schema, returnType: 'composite-literal' -}, { - value: '18', - column: schema.age, +}]) + +// NEW - use encryptQuery with queryType +const term = await protectClient.encryptQuery([{ + value: 'user@example.com', + column: schema.email, table: schema, + queryType: 'equality', returnType: 'composite-literal' }]) +``` + +### `createQuerySearchTerms` (deprecated) + +The `createQuerySearchTerms` function provided explicit index type control. It has been superseded by `encryptQuery`. + +```typescript +// DEPRECATED - use encryptQuery instead +const term = await protectClient.createQuerySearchTerms([{ + value: 'user@example.com', + column: schema.email, + table: schema, + indexType: 'unique' // Note: indexType was the old parameter name, now use queryType +}]) + +// NEW - similar API with encryptQuery +const term = await protectClient.encryptQuery([{ + value: 'user@example.com', + column: schema.email, + table: schema, + queryType: 'equality' +}]) +``` + +See [Migration from Deprecated Functions](#migration-from-deprecated-functions) for a complete migration guide. + +## Unified Query Encryption API + +The `encryptQuery` function handles both single values and batch operations: + +### Single Value + +```typescript +// Encrypt a single value with explicit query type +const term = await protectClient.encryptQuery('admin@example.com', { + column: usersSchema.email, + table: usersSchema, + queryType: 'equality', +}) if (term.failure) { // Handle the error } -console.log(term.data) // array of search terms +// Use the encrypted term in your query +console.log(term.data) // encrypted search term ``` +### Batch Operations + +```typescript +// Encrypt multiple terms in one call +const terms = await protectClient.encryptQuery([ + // Scalar term with explicit query type + { value: 'admin@example.com', column: users.email, table: users, queryType: 'equality' }, + + // JSON path query (ste_vec implicit) + { path: 'user.email', value: 'test@example.com', column: jsonSchema.metadata, table: jsonSchema }, + + // JSON containment query (ste_vec implicit) + { contains: { role: 'admin' }, column: jsonSchema.metadata, table: jsonSchema }, +]) + +if (terms.failure) { + // Handle the error +} + +// Access encrypted terms +console.log(terms.data) // array of encrypted terms +``` + +### Migration from Deprecated Functions + +| Old API | New API | +|---------|---------| +| `createSearchTerms([{ value, column, table }])` | `encryptQuery([{ value, column, table, queryType }])` with `ScalarQueryTerm` | +| `createQuerySearchTerms([{ value, column, table, indexType }])` | `encryptQuery([{ value, column, table, queryType }])` with `ScalarQueryTerm` | +| `createSearchTerms([{ path, value, column, table }])` | `encryptQuery([{ path, value, column, table }])` with `JsonPathQueryTerm` | +| `createSearchTerms([{ containmentType: 'contains', value, ... }])` | `encryptQuery([{ contains: {...}, column, table }])` with `JsonContainsQueryTerm` | +| `createSearchTerms([{ containmentType: 'contained_by', value, ... }])` | `encryptQuery([{ containedBy: {...}, column, table }])` with `JsonContainedByQueryTerm` | + > [!NOTE] -> As a developer, you must track the index of the search term in the array when using the `createSearchTerms` function. +> Both `createSearchTerms` and `createQuerySearchTerms` are deprecated. Use `encryptQuery` for all query encryption needs. + +> [!TIP] +> The `queryType` parameter is optional when the column has only one index type configured. + +### Query Term Types + +The `encryptQuery` function accepts different query term types. These types are exported from `@cipherstash/protect`: + +```typescript +import { + // Query term types + type QueryTerm, + type ScalarQueryTerm, + type JsonPathQueryTerm, + type JsonContainsQueryTerm, + type JsonContainedByQueryTerm, + // Type guards for runtime type checking + isScalarQueryTerm, + isJsonPathQueryTerm, + isJsonContainsQueryTerm, + isJsonContainedByQueryTerm, +} from '@cipherstash/protect' +``` + +**Type definitions:** + +| Type | Properties | Use Case | +|------|------------|----------| +| `ScalarQueryTerm` | `value`, `column`, `table`, `queryType`, `queryOp?` | Scalar value queries using queryType: 'equality', 'freeTextSearch', or 'orderAndRange' | +| `JsonPathQueryTerm` | `path`, `value?`, `column`, `table` | JSON path access queries | +| `JsonContainsQueryTerm` | `contains`, `column`, `table` | JSON containment (`@>`) queries | +| `JsonContainedByQueryTerm` | `containedBy`, `column`, `table` | JSON contained-by (`<@`) queries | + +**Type guards:** + +Type guards are useful when working with mixed query results: + +```typescript +const terms = await protectClient.encryptQuery([ + { value: 'user@example.com', column: schema.email, table: schema, queryType: 'equality' }, + { contains: { role: 'admin' }, column: schema.metadata, table: schema }, +]) + +if (terms.failure) { + // Handle error +} + +for (const term of terms.data) { + if (isScalarQueryTerm(term)) { + // Handle scalar term + } else if (isJsonContainsQueryTerm(term)) { + // Handle containment term - access term.sv + } +} +``` + +## JSON Search + +For querying encrypted JSON columns configured with `.searchableJson()`, use the `encryptQuery` function with JSON-specific term types. + +### Creating JSON Search Terms + +#### Path Queries + +Used for finding records where a specific path in the JSON equals a value. + +| Property | Description | +|----------|-------------| +| `path` | The path to the field (e.g., `'user.email'` or `['user', 'email']`) | +| `value` | The value to match at that path | +| `column` | The column definition from the schema | +| `table` | The table definition | + +```typescript +// Path query - SQL equivalent: WHERE metadata->'user'->>'email' = 'alice@example.com' +const pathTerms = await protectClient.encryptQuery([{ + path: 'user.email', + value: 'alice@example.com', + column: schema.metadata, + table: schema +}]) + +if (pathTerms.failure) { + // Handle the error +} +``` + +#### Containment Queries + +Used for finding records where the JSON column contains a specific JSON structure (subset). + +**Contains Query (`@>` operator)** - Find records where JSON contains the specified structure: + +| Property | Description | +|----------|-------------| +| `contains` | The JSON object/array structure to search for | +| `column` | The column definition from the schema | +| `table` | The table definition | + +```typescript +// Containment query - SQL equivalent: WHERE metadata @> '{"roles": ["admin"]}' +const containmentTerms = await protectClient.encryptQuery([{ + contains: { roles: ['admin'] }, + column: schema.metadata, + table: schema +}]) + +if (containmentTerms.failure) { + // Handle the error +} +``` + +**Contained-By Query (`<@` operator)** - Find records where JSON is contained by the specified structure: + +| Property | Description | +|----------|-------------| +| `containedBy` | The JSON superset to check against | +| `column` | The column definition from the schema | +| `table` | The table definition | + +```typescript +// Contained-by query - SQL equivalent: WHERE metadata <@ '{"permissions": ["read", "write", "admin"]}' +const containedByTerms = await protectClient.encryptQuery([{ + containedBy: { permissions: ['read', 'write', 'admin'] }, + column: schema.metadata, + table: schema +}]) + +if (containedByTerms.failure) { + // Handle the error +} +``` + +### Using JSON Search Terms in PostgreSQL + +When searching encrypted JSON columns, you use the `ste_vec` index type which supports both path access and containment operators. + +#### Path Search (Access Operator) + +Equivalent to `data->'path'->>'field' = 'value'`. + +```typescript +const terms = await protectClient.encryptQuery([{ + path: 'user.email', + value: 'alice@example.com', + column: schema.metadata, + table: schema +}]) + +if (terms.failure) { + // Handle the error +} + +// The generated term contains a selector and the encrypted term +const term = terms.data[0] + +// EQL function equivalent to: metadata->'user'->>'email' = 'alice@example.com' +const query = ` + SELECT * FROM users + WHERE eql_ste_vec_u64_8_128_access(metadata, $1) = $2 +` +// Bind parameters: [term.s, term.c] +``` + +#### Containment Search + +Equivalent to `data @> '{"key": "value"}'`. + +```typescript +const terms = await protectClient.encryptQuery([{ + contains: { tags: ['premium'] }, + column: schema.metadata, + table: schema +}]) + +if (terms.failure) { + // Handle the error +} + +// Containment terms return a vector of terms to match +const termVector = terms.data[0].sv + +// EQL function equivalent to: metadata @> '{"tags": ["premium"]}' +const query = ` + SELECT * FROM users + WHERE eql_ste_vec_u64_8_128_contains(metadata, $1) +` +// Bind parameter: [JSON.stringify(termVector)] +``` ## Search capabilities @@ -112,10 +371,11 @@ Use `.equality()` when you need to find exact matches: ```typescript // Find user with specific email -const term = await protectClient.createSearchTerms([{ +const term = await protectClient.encryptQuery([{ value: 'user@example.com', column: schema.email, table: schema, + queryType: 'equality', // Use 'equality' for exact match queries returnType: 'composite-literal' // Required for PostgreSQL composite types }]) @@ -136,10 +396,11 @@ Use `.freeTextSearch()` for text-based searches: ```typescript // Search for users with emails containing "example" -const term = await protectClient.createSearchTerms([{ +const term = await protectClient.encryptQuery([{ value: 'example', column: schema.email, table: schema, + queryType: 'freeTextSearch', // Use 'freeTextSearch' for text search queries returnType: 'composite-literal' }]) @@ -206,10 +467,11 @@ await client.query( ) // Search encrypted data -const searchTerm = await protectClient.createSearchTerms([{ +const searchTerm = await protectClient.encryptQuery([{ value: 'example.com', column: schema.email, table: schema, + queryType: 'freeTextSearch', // Use 'freeTextSearch' for text search returnType: 'composite-literal' }]) @@ -259,7 +521,8 @@ For Supabase users, we provide a specific implementation guide. [Read more about ## Performance optimization -TODO: make docs for creating Postgres Indexes on columns that require searches. At the moment EQL v2 doesn't support creating indexes while also using the out-of-the-box operator and operator families. The solution is to create an index using the EQL functions and then using the EQL functions directly in your SQL statments, which isn't the best experience. +> [!NOTE] +> Documentation for creating PostgreSQL indexes on encrypted columns is coming soon. Currently, EQL v2 requires using EQL functions directly in SQL statements when creating indexes. ### Didn't find what you wanted? diff --git a/docs/reference/supabase-sdk.md b/docs/reference/supabase-sdk.md index 594c3122..330370be 100644 --- a/docs/reference/supabase-sdk.md +++ b/docs/reference/supabase-sdk.md @@ -174,7 +174,7 @@ ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA eql_v2 GRANT ALL ON SEQUENC When searching encrypted data, you need to convert the encrypted payload into a format that PostgreSQL and the Supabase SDK can understand. The encrypted payload needs to be converted to a raw composite type format by double stringifying the JSON: ```typescript -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'billy@example.com', column: users.email, @@ -189,7 +189,7 @@ const searchTerm = searchTerms.data[0] For certain queries, when including the encrypted search term with an operator that uses the string logic syntax, you need to use the 'escaped-composite-literal' return type: ```typescript -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'billy@example.com', column: users.email, @@ -208,7 +208,7 @@ Here are examples of different ways to search encrypted data using the Supabase ### Equality Search ```typescript -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'billy@example.com', column: users.email, @@ -226,7 +226,7 @@ const { data, error } = await supabase ### Pattern Matching Search ```typescript -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'example.com', column: users.email, @@ -247,7 +247,7 @@ When you need to search for multiple encrypted values, you can use the IN operat ```typescript // Encrypt multiple search terms -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'value1', column: users.name, @@ -275,7 +275,7 @@ You can combine multiple encrypted search conditions using the `.or()` syntax. T ```typescript // Encrypt search terms for different columns -const searchTerms = await protectClient.createSearchTerms([ +const searchTerms = await protectClient.encryptQuery([ { value: 'user@example.com', column: users.email, diff --git a/local/create-ci-table.sql b/local/create-ci-table.sql index d61dfabd..842f37ec 100644 --- a/local/create-ci-table.sql +++ b/local/create-ci-table.sql @@ -4,5 +4,6 @@ CREATE TABLE "protect-ci" ( age eql_v2_encrypted, score eql_v2_encrypted, profile eql_v2_encrypted, - created_at TIMESTAMP DEFAULT NOW() + created_at TIMESTAMP DEFAULT NOW(), + test_run_id TEXT ); \ No newline at end of file diff --git a/packages/drizzle/README.md b/packages/drizzle/README.md index 2f4e82ff..f673316e 100644 --- a/packages/drizzle/README.md +++ b/packages/drizzle/README.md @@ -248,7 +248,7 @@ const results = await db ``` > [!TIP] -> **Performance Tip**: Using `protectOps.and()` batches all encryption operations into a single `createSearchTerms` call, which is more efficient than awaiting each operator individually. +> **Performance Tip**: Using `protectOps.and()` batches all encryption operations into a single `encryptQuery` call, which is more efficient than awaiting each operator individually. ## Available Operators diff --git a/packages/protect-dynamodb/README.md b/packages/protect-dynamodb/README.md index e52ffe66..ffd2e84c 100644 --- a/packages/protect-dynamodb/README.md +++ b/packages/protect-dynamodb/README.md @@ -55,7 +55,7 @@ await docClient.send(new PutCommand({ })) // Create search terms for querying -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, @@ -119,10 +119,10 @@ if (result.failure) { Create search terms for querying encrypted data: -- `createSearchTerms`: Creates search terms for one or more columns +- `encryptQuery`: Creates search terms for one or more columns ```typescript -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, @@ -165,7 +165,7 @@ if (encryptResult.failure) { } // Query using search terms -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, @@ -199,7 +199,7 @@ const table = { } // Create search terms for querying -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, @@ -243,7 +243,7 @@ const table = { } // Create search terms for querying -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, @@ -298,7 +298,7 @@ const table = { } // Create search terms for querying -const searchTermsResult = await protectDynamo.createSearchTerms([ +const searchTermsResult = await protectDynamo.encryptQuery([ { value: 'user@example.com', column: users.email, diff --git a/packages/protect/README.md b/packages/protect/README.md index fab455d0..0461ac45 100644 --- a/packages/protect/README.md +++ b/packages/protect/README.md @@ -846,7 +846,7 @@ CREATE TABLE users ( > [!WARNING] > The `eql_v2_encrypted` type is a [composite type](https://www.postgresql.org/docs/current/rowtypes.html) and each ORM/client has a different way of handling inserts and selects. -> We've documented how to handle inserts and selects for the different ORMs/clients in the [docs](./docs/reference/working-with-composite-types.md). +> Handling inserts and selects varies by ORM/client. See the [Drizzle integration guide](./docs/reference/drizzle/drizzle.md) for examples. Read more about [how to search encrypted data](./docs/reference/searchable-encryption-postgres.md) in the docs. @@ -986,15 +986,106 @@ const bulkDecryptedResult = await protectClient ## Supported data types -Protect.js currently supports encrypting and decrypting text. -Other data types like booleans, dates, ints, floats, and JSON are well-supported in other CipherStash products, and will be coming to Protect.js soon. +Protect.js supports a number of different data types with support for additional types on the roadmap. + +| JS/TS Type | Available | Notes | +|--|--|--| +| `string` | ✅ | +| `number` | ✅ | +| `json` (opaque) | ✅ | | +| `json` (searchable) | ✅ | | +| `bigint` | ⚙️ | Coming soon | +| `boolean`| ⚙️ | Coming soon | +| `date` | ⚙️ | Coming soon | + +If you need support for ther data types please [raise an issue](https://github.com/cipherstash/protectjs/issues) and we'll do our best to add it to Protect.js. + +### Type casting + +When encrypting types other than `string`, Protect requires the data type to be specified explicitly using the `dataType` function on the column definition. + +For example, to handle encryption of a `number` field called `score`: + +```ts +const users = csTable('users', { + score: csColumn('score').dataType('number') +}) +``` + +This means that any JavaScript/TypeScript `number` will encrypt correctly but if an attempt to encrypt a value of a different type is made the operation will fail with an error. +This is particularly important for searchable index schemes that require data types (and their encodings) to be consistent. + +In an unencrypted setup, this type checking is usually handled by the database (the column type in a table) but when the data is encrypted, the database can't determine what type the plaintext value should be so we must specify it in the Protect schema instead. + +> [!IMPORTANT] +> If the data type of a column is set to `bigint`, floating point numbers will be converted to integers (via truncation). + +### Handling of null and special values + +There are some important special cases to be aware of when encrypting values with Protect.js. +For example, encrypting `null` or `undefined` will just return a `null`/`undefined` value. + +When `dataType` is `number`, attempting to encrypt `NaN`, `Infinity` or `-Infinity` will fail with an error. +Encrypting `-0.0` will coerce the value into `0.0`. + +The table below summarizes these cases. + +| Data type | Plaintext | Encryption | +|--|--|--| +|`any`| `null` | `null` | +| `any` | `undefined` | `undefined` | +| `number` | `-0.0` | Encryption of `0.0` | +| `number` | `NaN` | _Error_ | +| `number` | `Infinity` | _Error_| +| `number` | `-Infinity` | _Error_| -Until support for other data types are available, you can express interest in this feature by adding a :+1: on this [GitHub Issue](https://github.com/cipherstash/protectjs/issues/48). ## Searchable encryption Read more about [searching encrypted data](./docs/concepts/searchable-encryption.md) in the docs. +### Searchable JSON + +Protect.js allows you to perform deep searches within encrypted JSON documents. You can query nested fields, arrays, and objects without decrypting the entire document. + +To enable searchable JSON, configure your schema: + +```ts +// schema.ts +import { csTable, csColumn } from "@cipherstash/protect"; + +export const users = csTable("users", { + metadata: csColumn("metadata").searchableJson(), +}); +``` + +Then generate search terms for your queries: + +```ts +// index.ts +// Path query: find users with metadata.role = 'admin' +const searchTerms = await protectClient.encryptQuery([ + { + path: "role", // or "user.role" or ["user", "role"] + value: "admin", + column: users.metadata, + table: users, + } +]); + +// Containment query: find users where metadata contains { tags: ['premium'] } +const containmentTerms = await protectClient.encryptQuery([ + { + contains: { tags: ["premium"] }, + column: users.metadata, + table: users, + } +]); +``` + +These search terms can then be used in your database query (e.g., using SQL or an ORM). + + ## Multi-tenant encryption Protect.js supports multi-tenant encryption by using keysets. @@ -1073,7 +1164,7 @@ Here are a few resources to help based on your tool set: - [SST and AWS serverless functions](./docs/how-to/sst-external-packages.md). > [!TIP] -> Deploying to Linux (e.g., AWS Lambda) with npm lockfile v3 and seeing runtime module load errors? See the troubleshooting guide: [`docs/how-to/npm-lockfile-v3`](./docs/how-to/npm-lockfile-v3-linux-deployments.md). +> Deploying to Linux (e.g., AWS Lambda) with npm lockfile v3 and seeing runtime module load errors? See the troubleshooting guide: [`docs/how-to/npm-lockfile-v3.md`](./docs/how-to/npm-lockfile-v3.md). ## Contributing diff --git a/packages/protect/src/helpers/index.ts b/packages/protect/src/helpers/index.ts index 037d27df..379e246e 100644 --- a/packages/protect/src/helpers/index.ts +++ b/packages/protect/src/helpers/index.ts @@ -1,12 +1,29 @@ import type { KeysetIdentifier as KeysetIdentifierFfi } from '@cipherstash/protect-ffi' import type { Encrypted, KeysetIdentifier } from '../types' +/** + * Represents an encrypted payload formatted for a PostgreSQL composite type (`eql_v2_encrypted`). + */ export type EncryptedPgComposite = { + /** The raw encrypted data object. */ data: Encrypted } /** - * Helper function to transform an encrypted payload into a PostgreSQL composite type + * Transforms an encrypted payload into a PostgreSQL composite type format. + * + * This is required when inserting encrypted data into a column defined as `eql_v2_encrypted` + * using a PostgreSQL client or SDK (like Supabase). + * + * @param obj - The encrypted payload object. + * + * @example + * **Supabase SDK Integration** + * ```typescript + * const { data, error } = await supabase + * .from('users') + * .insert([encryptedToPgComposite(encryptedResult.data)]) + * ``` */ export function encryptedToPgComposite(obj: Encrypted): EncryptedPgComposite { return { @@ -15,7 +32,21 @@ export function encryptedToPgComposite(obj: Encrypted): EncryptedPgComposite { } /** - * Helper function to transform a model's encrypted fields into PostgreSQL composite types + * Transforms all encrypted fields within a model into PostgreSQL composite types. + * + * Automatically detects fields that look like encrypted payloads and wraps them + * in the structure expected by PostgreSQL's `eql_v2_encrypted` composite type. + * + * @param model - An object containing one or more encrypted fields. + * + * @example + * **Supabase Model Integration** + * ```typescript + * const encryptedModel = await protectClient.encryptModel(user, usersTable); + * const { data, error } = await supabase + * .from('users') + * .insert([modelToEncryptedPgComposites(encryptedModel.data)]) + * ``` */ export function modelToEncryptedPgComposites>( model: T, @@ -34,7 +65,17 @@ export function modelToEncryptedPgComposites>( } /** - * Helper function to transform multiple models' encrypted fields into PostgreSQL composite types + * Transforms multiple models' encrypted fields into PostgreSQL composite types. + * + * @param models - An array of objects containing encrypted fields. + * + * @example + * ```typescript + * const encryptedModels = await protectClient.bulkEncryptModels(users, usersTable); + * await supabase + * .from('users') + * .insert(bulkModelsToEncryptedPgComposites(encryptedModels.data)) + * ``` */ export function bulkModelsToEncryptedPgComposites< T extends Record, @@ -42,6 +83,9 @@ export function bulkModelsToEncryptedPgComposites< return models.map((model) => modelToEncryptedPgComposites(model)) } +/** + * @internal + */ export function toFfiKeysetIdentifier( keyset: KeysetIdentifier | undefined, ): KeysetIdentifierFfi | undefined { @@ -55,7 +99,9 @@ export function toFfiKeysetIdentifier( } /** - * Helper function to check if a value is an encrypted payload + * Checks if a value is an encrypted payload object. + * + * @param value - The value to check. */ export function isEncryptedPayload(value: unknown): value is Encrypted { if (value === null) return false @@ -69,4 +115,4 @@ export function isEncryptedPayload(value: unknown): value is Encrypted { } return false -} +} \ No newline at end of file diff --git a/packages/schema/__tests__/schema.test.ts b/packages/schema/__tests__/schema.test.ts index d1d99a51..7d2a117f 100644 --- a/packages/schema/__tests__/schema.test.ts +++ b/packages/schema/__tests__/schema.test.ts @@ -131,7 +131,7 @@ describe('Schema with nested columns', () => { }) // NOTE: Leaving this test commented out until stevec indexing for JSON is supported. - /*it('should handle ste_vec index for JSON columns', () => { + it('should handle ste_vec index for JSON columns', () => { const users = csTable('users', { json: csColumn('json').dataType('jsonb').searchableJson(), } as const) @@ -142,5 +142,5 @@ describe('Schema with nested columns', () => { expect(config.tables.users.json.indexes.ste_vec?.prefix).toEqual( 'users/json', ) - })*/ + }) }) diff --git a/packages/schema/__tests__/searchable-json.test.ts b/packages/schema/__tests__/searchable-json.test.ts new file mode 100644 index 00000000..ec8187cb --- /dev/null +++ b/packages/schema/__tests__/searchable-json.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest' +import { buildEncryptConfig, csColumn, csTable } from '../src' + +describe('searchableJson schema method', () => { + it('should configure ste_vec index with correct prefix', () => { + const users = csTable('users', { + metadata: csColumn('metadata').searchableJson(), + }) + + const config = buildEncryptConfig(users) + + expect(config.tables.users.metadata.cast_as).toBe('json') + expect(config.tables.users.metadata.indexes.ste_vec).toBeDefined() + expect(config.tables.users.metadata.indexes.ste_vec?.prefix).toBe( + 'users/metadata', + ) + }) + + it('should allow chaining with other column methods', () => { + const users = csTable('users', { + data: csColumn('data').searchableJson(), + }) + + const config = buildEncryptConfig(users) + + expect(config.tables.users.data.cast_as).toBe('json') + expect(config.tables.users.data.indexes.ste_vec?.prefix).toBe('users/data') + }) + + it('should work alongside regular encrypted columns', () => { + const users = csTable('users', { + email: csColumn('email').equality(), + metadata: csColumn('metadata').searchableJson(), + }) + + const config = buildEncryptConfig(users) + + expect(config.tables.users.email.indexes.unique).toBeDefined() + expect(config.tables.users.metadata.indexes.ste_vec).toBeDefined() + }) +}) From c08b534332e042956d56c8fbfad7547de9412e43 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 29 Jan 2026 20:33:16 +1100 Subject: [PATCH 2/3] fix(protect): update test assertions for FFI 0.20.1 error messages --- packages/protect/__tests__/number-protect.test.ts | 2 +- packages/protect/tsup.config.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/protect/__tests__/number-protect.test.ts b/packages/protect/__tests__/number-protect.test.ts index 3ade327a..81891ed5 100644 --- a/packages/protect/__tests__/number-protect.test.ts +++ b/packages/protect/__tests__/number-protect.test.ts @@ -882,7 +882,7 @@ describe('Invalid or uncoercable values', () => { }) expect(result.failure).toBeDefined() - expect(result.failure?.message).toContain('Unsupported conversion') + expect(result.failure?.message).toContain('Cannot convert') }, 30000, ) diff --git a/packages/protect/tsup.config.ts b/packages/protect/tsup.config.ts index 8fdee46c..59390bbe 100644 --- a/packages/protect/tsup.config.ts +++ b/packages/protect/tsup.config.ts @@ -24,7 +24,5 @@ export default defineConfig([ }, dts: false, sourcemap: true, - external: ['dotenv'], - noExternal: [], }, ]) From 31a72892462a394aa77653abfdeb967a76c1a10d Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 29 Jan 2026 20:33:48 +1100 Subject: [PATCH 3/3] refactor(protect): remove obsolete search-terms.test.ts (replaced by search-terms-deprecated.test.ts) --- .../protect/__tests__/search-terms.test.ts | 90 ------------------- 1 file changed, 90 deletions(-) delete mode 100644 packages/protect/__tests__/search-terms.test.ts diff --git a/packages/protect/__tests__/search-terms.test.ts b/packages/protect/__tests__/search-terms.test.ts deleted file mode 100644 index f3cef7fe..00000000 --- a/packages/protect/__tests__/search-terms.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import 'dotenv/config' -import { csColumn, csTable } from '@cipherstash/schema' -import { describe, expect, it } from 'vitest' -import { type SearchTerm, protect } from '../src' - -const users = csTable('users', { - email: csColumn('email').freeTextSearch().equality().orderAndRange(), - address: csColumn('address').freeTextSearch(), -}) - -describe('create search terms', () => { - it('should create search terms with default return type', async () => { - const protectClient = await protect({ schemas: [users] }) - - const searchTerms = [ - { - value: 'hello', - column: users.email, - table: users, - }, - { - value: 'world', - column: users.address, - table: users, - }, - ] as SearchTerm[] - - const searchTermsResult = await protectClient.createSearchTerms(searchTerms) - - if (searchTermsResult.failure) { - throw new Error(`[protect]: ${searchTermsResult.failure.message}`) - } - - expect(searchTermsResult.data).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - c: expect.any(String), - }), - ]), - ) - }, 30000) - - it('should create search terms with composite-literal return type', async () => { - const protectClient = await protect({ schemas: [users] }) - - const searchTerms = [ - { - value: 'hello', - column: users.email, - table: users, - returnType: 'composite-literal', - }, - ] as SearchTerm[] - - const searchTermsResult = await protectClient.createSearchTerms(searchTerms) - - if (searchTermsResult.failure) { - throw new Error(`[protect]: ${searchTermsResult.failure.message}`) - } - - const result = searchTermsResult.data[0] as string - expect(result).toMatch(/^\(.*\)$/) - expect(() => JSON.parse(result.slice(1, -1))).not.toThrow() - }, 30000) - - it('should create search terms with escaped-composite-literal return type', async () => { - const protectClient = await protect({ schemas: [users] }) - - const searchTerms = [ - { - value: 'hello', - column: users.email, - table: users, - returnType: 'escaped-composite-literal', - }, - ] as SearchTerm[] - - const searchTermsResult = await protectClient.createSearchTerms(searchTerms) - - if (searchTermsResult.failure) { - throw new Error(`[protect]: ${searchTermsResult.failure.message}`) - } - - const result = searchTermsResult.data[0] as string - expect(result).toMatch(/^".*"$/) - const unescaped = JSON.parse(result) - expect(unescaped).toMatch(/^\(.*\)$/) - expect(() => JSON.parse(unescaped.slice(1, -1))).not.toThrow() - }, 30000) -})