From 9c25abd114a1453dc43cd62b27271d7a2e37fdd6 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 8 Sep 2026 16:21:46 -0400 Subject: [PATCH 1/2] feat(extension-pgvector): support variable length embeddings Signed-off-by: Ryan --- packages/3-extensions/pgvector/README.md | 2 +- .../pgvector/src/core/authoring.ts | 9 +++- .../3-extensions/pgvector/src/core/codecs.ts | 46 +++++++++++------ .../pgvector/src/exports/column-types.ts | 19 ++++++- .../codec-conformance.integration.test.ts | 10 ++++ .../test/codec-render-output-type.test.ts | 6 ++- .../pgvector/test/variable-dimensions.test.ts | 49 +++++++++++++++++++ .../@prisma/orm-extension-pgvector/README.md | 4 +- .../test/packaging/extension-tarball.test.ts | 20 ++++++++ 9 files changed, 142 insertions(+), 23 deletions(-) create mode 100644 packages/3-extensions/pgvector/test/variable-dimensions.test.ts diff --git a/packages/3-extensions/pgvector/README.md b/packages/3-extensions/pgvector/README.md index 4e7ef91e265d..571546bffd9c 100644 --- a/packages/3-extensions/pgvector/README.md +++ b/packages/3-extensions/pgvector/README.md @@ -83,7 +83,7 @@ export const contract = defineContract({ }); ``` -The `vector(N)` factory is registered through the unified `CodecDescriptor<{ length: number }>` shape — `paramsSchema` validates the dimension at the contract boundary, `renderOutputType: ({ length }) => 'Vector<' + length + '>'` produces the column's TS type for `contract.d.ts`, and the curried `factory` materializes the runtime codec at context construction. See [ADR 208 — Higher-order codecs for parameterized types](../../../docs/architecture%20docs/adrs/ADR%20208%20-%20Higher-order%20codecs%20for%20parameterized%20types.md) for the descriptor model. Every pgvector column must declare an explicit dimension via `vector(N)`; the runtime codec is constructed against `{ length: N }`, so an undimensioned form has no honest descriptor signature. +The `vector(N?)` factory is registered through the unified `CodecDescriptor<{ length?: number }>` shape — `paramsSchema` validates the dimension at the contract boundary when specified, `renderOutputType: ({ length? }) => 'Vector' | 'Vector<' + length + '>'` produces the column's TS type for `contract.d.ts`, and the curried `factory` materializes the runtime codec at context construction. See [ADR 208 — Higher-order codecs for parameterized types](../../../docs/architecture%20docs/adrs/ADR%20208%20-%20Higher-order%20codecs%20for%20parameterized%20types.md) for the descriptor model. Any pgvector column can declare an explicit dimension via `vector(N)` for runtime validation and `Vector` typing, or use a standard `vector` to support variable dimensions. In either case, Postgres still enforces its own vector limits and requires compatible dimensions for distance operations. ### Runtime Setup diff --git a/packages/3-extensions/pgvector/src/core/authoring.ts b/packages/3-extensions/pgvector/src/core/authoring.ts index 6e95ef5e8d54..2da8afe20d7f 100644 --- a/packages/3-extensions/pgvector/src/core/authoring.ts +++ b/packages/3-extensions/pgvector/src/core/authoring.ts @@ -6,7 +6,14 @@ export const pgvectorAuthoringTypes = { Vector: { kind: 'typeConstructor', args: [ - { kind: 'number', name: 'length', integer: true, minimum: 1, maximum: VECTOR_MAX_DIM }, + { + kind: 'number', + name: 'length', + optional: true, + integer: true, + minimum: 1, + maximum: VECTOR_MAX_DIM, + }, ], output: { codecId: 'pg/vector@1', diff --git a/packages/3-extensions/pgvector/src/core/codecs.ts b/packages/3-extensions/pgvector/src/core/codecs.ts index 0a43e3e58d07..6cec208a5521 100644 --- a/packages/3-extensions/pgvector/src/core/codecs.ts +++ b/packages/3-extensions/pgvector/src/core/codecs.ts @@ -4,10 +4,10 @@ * Mirrors the patterns in `postgres/codecs-class.ts` and `sqlite/codecs-class.ts` for the single `pg/vector@1` codec. Three artifacts: * * 1. `PgVectorCodec` extends {@link CodecImpl} with the runtime encode/decode/encodeJson/decodeJson conversions inline. Conversions are simple enough (PostgreSQL `[1,2,3]` text format) that no shared helper module is warranted; the class body is the source of truth. - * 2. `PgVectorDescriptor` extends {@link PostgresCodecDescriptor} with the codec id, traits, target types, params schema (`{ length: number }`, validated against {@link VECTOR_MAX_DIM}), the postgres native type `vector`, explicit target behavior, and the emit-path `renderOutputType` producing `Vector<${length}>`. - * 3. `pgVectorColumn(length)` per-codec column helper invoking `descriptor.factory({ length })` directly + passing the bare `nativeType: 'vector'`. The family-layer {@link expandNativeType} hook renders the parameterized form (`vector(1536)`) at emit/verify time from `nativeType` + `typeParams`. + * 2. `PgVectorDescriptor` extends {@link PostgresCodecDescriptor} with the codec id, traits, target types, params schema (`{ length?: number }`, validated against {@link VECTOR_MAX_DIM}), the postgres native type `vector`, explicit target behavior, and the emit-path `renderOutputType` producing `Vector` or `Vector<${length}>`. + * 3. `pgVectorColumn(length?)` per-codec column helper invoking `descriptor.factory({ length })` directly + passing the bare `nativeType: 'vector'`. The family-layer {@link expandNativeType} hook renders the parameterized form (`vector` or `vector(1536)`) at emit/verify time from `nativeType` + `typeParams`. * - * `length` threads into the runtime codec via the constructor so encode/decode/encodeJson/decodeJson enforce the declared dimension at every ingress path. Without this, `vector(3)` and `vector(1536)` would produce codecs with identical behaviour and a dimension-mismatched value would round-trip undetected. + * When provided, `length` threads into the runtime codec via the constructor so encode/decode/encodeJson/decodeJson enforce the declared dimension at every ingress path. Without this, `vector(3)` and `vector(1536)` would produce codecs with identical behaviour and a dimension-mismatched value would round-trip undetected. */ import type { JsonValue } from '@internal/contract/types'; @@ -33,12 +33,13 @@ import { pgVectorError } from './errors'; type VectorConversionCode = 'RUNTIME.ENCODE_FAILED' | 'RUNTIME.DECODE_FAILED'; -type VectorParams = { readonly length: number }; +type VectorParams = { readonly length?: number }; const vectorParamsSchema = arktype({ - length: 'number', + 'length?': 'number', }).narrow((params, ctx) => { const { length } = params; + if (length === undefined) return true; if (!Number.isInteger(length)) { return ctx.mustBe('an integer'); } @@ -86,9 +87,9 @@ export class PgVectorCodec extends CodecImpl< string, number[] > { - readonly length: number; + readonly length: number | undefined; - constructor(descriptor: AnyCodecDescriptor, length: number) { + constructor(descriptor: AnyCodecDescriptor, length: number | undefined) { super(descriptor); this.length = length; } @@ -106,7 +107,7 @@ export class PgVectorCodec extends CodecImpl< throw pgVectorError(code, 'Vector value must contain only finite numbers', { meta }); } } - if (value.length !== this.length) { + if (this.length !== undefined && value.length !== this.length) { throw pgVectorError( code, `Vector length mismatch: expected ${this.length}, got ${value.length}`, @@ -183,7 +184,7 @@ export class PgVectorDescriptor extends PostgresCodecDescriptor { override readonly targetTypes = ['vector'] as const; override readonly paramsSchema: StandardSchemaV1 = vectorParamsSchema; override renderOutputType(params: VectorParams): string { - return `Vector<${params.length}>`; + return params.length === undefined ? 'Vector' : `Vector<${params.length}>`; } override factory(params: VectorParams): (ctx: CodecInstanceContext) => PgVectorCodec { return () => new PgVectorCodec(this, params.length); @@ -192,13 +193,26 @@ export class PgVectorDescriptor extends PostgresCodecDescriptor { export const pgVectorDescriptor = new PgVectorDescriptor(); -/** - * Per-codec column helper for `pg/vector@1`. Generic over `N extends number` so the column site preserves the dimension literal in `typeParams` (e.g. `pgVectorColumn(1536)` packs `typeParams: { length: 1536 }`). - * - * Passes the bare `nativeType: 'vector'`; the family-layer `expandNativeType` hook renders the parameterized form (`vector(1536)`) at emit/verify time from `nativeType` + `typeParams`. - */ -export const pgVectorColumn = (length: N) => - column(pgVectorDescriptor.factory({ length }), pgVectorDescriptor.codecId, { length }, 'vector'); +export function pgVectorColumn(): ReturnType; +export function pgVectorColumn( + length: N, +): ReturnType>; +export function pgVectorColumn(length?: number) { + return length === undefined ? variableVectorColumn() : fixedVectorColumn(length); +} + +function variableVectorColumn() { + return column(pgVectorDescriptor.factory({}), pgVectorDescriptor.codecId, {}, 'vector'); +} + +function fixedVectorColumn(length: N) { + return column( + pgVectorDescriptor.factory({ length }), + pgVectorDescriptor.codecId, + { length }, + 'vector', + ); +} pgVectorColumn satisfies ColumnHelperFor; pgVectorColumn satisfies ColumnHelperForStrict; diff --git a/packages/3-extensions/pgvector/src/exports/column-types.ts b/packages/3-extensions/pgvector/src/exports/column-types.ts index cb4de6c9077e..cd7721ed1e98 100644 --- a/packages/3-extensions/pgvector/src/exports/column-types.ts +++ b/packages/3-extensions/pgvector/src/exports/column-types.ts @@ -1,11 +1,22 @@ /** - * Column type descriptor factory for pgvector extension. `vector(N)` is the canonical authoring surface; every pgvector column must declare a dimension via this factory. The dimension threads into the runtime codec through `paramsSchema.length` and into the DDL via the family-layer `expandNativeType` hook (e.g. `vector(1536)`). + * Column type descriptor factory for pgvector extension. `vector(N?)` is the canonical authoring surface; each pgvector column may optionally declare a dimension via this factory. When provided, the dimension threads into the runtime codec through `paramsSchema.length` and into the DDL via the family-layer `expandNativeType` hook (e.g. `vector(1536)`). */ import type { ColumnTypeDescriptor } from '@internal/framework-components/codec'; import { VECTOR_CODEC_ID, VECTOR_MAX_DIM } from '../core/constants'; import { pgVectorError } from '../core/errors'; +/** + * Factory for creating non-dimensioned vector column descriptors. + * + * @example + * ```typescript + * .column('embedding', { type: vector(), nullable: false }) + * // Produces: nativeType: 'vector', typeParams: {} + * ``` + * @returns A column type descriptor without `typeParams.length` set + */ +export function vector(): ColumnTypeDescriptor & { readonly typeParams: Record }; /** * Factory for creating dimensioned vector column descriptors. * @@ -20,7 +31,11 @@ import { pgVectorError } from '../core/errors'; */ export function vector( length: N, -): ColumnTypeDescriptor & { readonly typeParams: { readonly length: N } } { +): ColumnTypeDescriptor & { readonly typeParams: { readonly length: N } }; +export function vector(length?: number): ColumnTypeDescriptor { + if (length === undefined) { + return { codecId: VECTOR_CODEC_ID, nativeType: 'vector', typeParams: {} }; + } if (!Number.isInteger(length) || length < 1 || length > VECTOR_MAX_DIM) { throw pgVectorError( 'CONTRACT.ARGUMENT_INVALID', diff --git a/packages/3-extensions/pgvector/test/codec-conformance.integration.test.ts b/packages/3-extensions/pgvector/test/codec-conformance.integration.test.ts index b6efe279f0b8..7382592454ae 100644 --- a/packages/3-extensions/pgvector/test/codec-conformance.integration.test.ts +++ b/packages/3-extensions/pgvector/test/codec-conformance.integration.test.ts @@ -45,6 +45,16 @@ function vectorCase( } const cases: readonly PostgresCodecConformanceCase[] = [ + ...[[1], [1, 2, 3]].map( + (value): PostgresCodecConformanceCase => ({ + codecId: 'pg/vector@1', + descriptor: pgVectorDescriptor, + label: `variable vector with ${value.length} dimensions`, + value, + typeParams: {}, + setupSql: INSTALL_VECTOR, + }), + ), vectorCase('three dimensions', [1, 2, 3]), // A vector's text form separates elements with commas and wraps them in // brackets, so a value has to carry negatives and fractions before the diff --git a/packages/3-extensions/pgvector/test/codec-render-output-type.test.ts b/packages/3-extensions/pgvector/test/codec-render-output-type.test.ts index 6e746b8cb2a1..a11e9ee61563 100644 --- a/packages/3-extensions/pgvector/test/codec-render-output-type.test.ts +++ b/packages/3-extensions/pgvector/test/codec-render-output-type.test.ts @@ -6,7 +6,11 @@ describe('pgvector codec renderOutputType', () => { | ((typeParams: Record) => string | undefined) | undefined; - // The descriptor's `renderOutputType` runs *after* `paramsSchema` validation so it can assume a well-formed `length`. Negative-shape inputs (missing / NaN / non-integer) are rejected upstream by `paramsSchema` and never reach this renderer. + // The descriptor's `renderOutputType` runs *after* `paramsSchema` validation so it can assume either a well-formed `length` or none at all. Negative-shape inputs (missing / NaN / non-integer) are rejected upstream by `paramsSchema` and never reach this renderer. + + it('renders Vector when when length is not present', () => { + expect(renderOutputType?.({})).toBe('Vector'); + }); it('renders Vector when length is present', () => { expect(renderOutputType?.({ length: 1536 })).toBe('Vector<1536>'); diff --git a/packages/3-extensions/pgvector/test/variable-dimensions.test.ts b/packages/3-extensions/pgvector/test/variable-dimensions.test.ts new file mode 100644 index 000000000000..0e1f6f77c120 --- /dev/null +++ b/packages/3-extensions/pgvector/test/variable-dimensions.test.ts @@ -0,0 +1,49 @@ +import { extractCodecControlHooks } from '@internal/family-sql/control'; +import { + instantiateAuthoringTypeConstructor, + validateAuthoringHelperArguments, +} from '@internal/framework-components/authoring'; +import { describe, expect, it } from 'vitest'; +import { pgvectorAuthoringTypes } from '../src/core/authoring'; +import { pgVectorColumn, pgVectorDescriptor } from '../src/core/codecs'; +import { vector } from '../src/exports/column-types'; +import control from '../src/exports/control'; + +describe('variable dimensions', () => { + it('authors a vector without dimension metadata', () => { + expect(vector()).toEqual({ codecId: 'pg/vector@1', nativeType: 'vector', typeParams: {} }); + expect(pgVectorColumn().typeParams).toEqual({}); + }); + + it('authors PSL vectors without a length argument', () => { + validateAuthoringHelperArguments( + 'pgvector.Vector', + pgvectorAuthoringTypes.pgvector.Vector.args, + [], + ); + expect( + instantiateAuthoringTypeConstructor(pgvectorAuthoringTypes.pgvector.Vector, []), + ).toMatchObject({ + codecId: 'pg/vector@1', + nativeType: 'vector', + }); + }); + + it('validates and renders undimensioned contracts', async () => { + expect(await pgVectorDescriptor.paramsSchema['~standard'].validate({})).toEqual({ value: {} }); + expect(pgVectorDescriptor.renderOutputType({})).toBe('Vector'); + const hooks = extractCodecControlHooks([control]).get('pg/vector@1'); + expect(hooks?.expandNativeType?.({ nativeType: 'vector', typeParams: {} })).toBe('vector'); + }); + + it('accepts different lengths through every codec path', async () => { + const codec = pgVectorColumn().codecFactory({ name: 'embedding' }); + for (const value of [[1], [1, 2, 3], [1, 2]]) { + const wire = `[${value.join(',')}]`; + expect(await codec.encode(value, {})).toBe(wire); + expect(await codec.decode(wire, {})).toEqual(value); + expect(codec.encodeJson(value)).toEqual(value); + expect(codec.decodeJson(value)).toEqual(value); + } + }); +}); diff --git a/packages/9-public/@prisma/orm-extension-pgvector/README.md b/packages/9-public/@prisma/orm-extension-pgvector/README.md index 52c2ac1ac320..51177af4c85c 100644 --- a/packages/9-public/@prisma/orm-extension-pgvector/README.md +++ b/packages/9-public/@prisma/orm-extension-pgvector/README.md @@ -11,11 +11,11 @@ pnpm add @prisma/orm-extension-pgvector | Namespace | Surface | | --- | --- | | `/pack` | the extension pack an application composes into `extensions: [...]` — pure, no runtime imports | -| `/column-types` | the `Vector(n)` column author | +| `/column-types` | the `vector()` or `vector(n)` column author | | `/codec-types`, `/operation-types` | types emitted contracts reference | | `/runtime` | the runtime extension that registers the codec and operations | | `/control` | the control descriptor and baseline migration that install the server extension | ## Responsibilities -Dimensioned vector storage and search: the `pg/vector@1` codec (`number[]` at runtime, `Vector` in `contract.d.ts`), similarity operations such as `cosineDistance`, and a baseline migration that runs `CREATE EXTENSION IF NOT EXISTS vector` when the pack is composed into an application. +Variable or fixed-dimension vector storage and search: the `pg/vector@1` codec (`number[]` at runtime, `Vector` in `contract.d.ts`), similarity operations such as `cosineDistance`, and a baseline migration that runs `CREATE EXTENSION IF NOT EXISTS vector` when the pack is composed into an application. diff --git a/test/integration/test/packaging/extension-tarball.test.ts b/test/integration/test/packaging/extension-tarball.test.ts index 77754855cc36..af60a8f40e55 100644 --- a/test/integration/test/packaging/extension-tarball.test.ts +++ b/test/integration/test/packaging/extension-tarball.test.ts @@ -81,6 +81,26 @@ describe('an extension pack installed next to the facade it extends', () => { expect(runInScratch(scratch, script)).toContain(`resolved ${subpaths.length}`); }); + it('supports variable dimensions from the installed package', () => { + expect( + runInScratch( + scratch, + ` + import { strict as assert } from 'node:assert'; + import { vector } from '${extension}/column-types'; + import runtime from '${extension}/runtime'; + assert.deepEqual(vector().typeParams, {}); + const descriptor = runtime.codecs().find(codec => codec.codecId === 'pg/vector@1'); + const codec = descriptor.factory({})({ name: 'embedding' }); + for (const value of [[1], [1, 2, 3]]) { + assert.deepEqual(await codec.decode(await codec.encode(value, {}), {}), value); + } + console.log('variable dimensions ok'); + `, + ), + ).toContain('variable dimensions ok'); + }); + it('requires its target shell as an exact-pinned peer, not a dependency', () => { const manifest: unknown = JSON.parse(readFileSync(join(installedDir, 'package.json'), 'utf8')); const { dependencies, peerDependencies } = Object(manifest) as { From 351be5359161677baafc27b0314e5db2fa407076 Mon Sep 17 00:00:00 2001 From: Ryan Garber Date: Tue, 8 Sep 2026 17:08:37 -0400 Subject: [PATCH 2/2] docs(extension-pgvector): clarify vector(N?) factory description in README Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/3-extensions/pgvector/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/3-extensions/pgvector/README.md b/packages/3-extensions/pgvector/README.md index 571546bffd9c..b8ec78bc9f91 100644 --- a/packages/3-extensions/pgvector/README.md +++ b/packages/3-extensions/pgvector/README.md @@ -83,7 +83,7 @@ export const contract = defineContract({ }); ``` -The `vector(N?)` factory is registered through the unified `CodecDescriptor<{ length?: number }>` shape — `paramsSchema` validates the dimension at the contract boundary when specified, `renderOutputType: ({ length? }) => 'Vector' | 'Vector<' + length + '>'` produces the column's TS type for `contract.d.ts`, and the curried `factory` materializes the runtime codec at context construction. See [ADR 208 — Higher-order codecs for parameterized types](../../../docs/architecture%20docs/adrs/ADR%20208%20-%20Higher-order%20codecs%20for%20parameterized%20types.md) for the descriptor model. Any pgvector column can declare an explicit dimension via `vector(N)` for runtime validation and `Vector` typing, or use a standard `vector` to support variable dimensions. In either case, Postgres still enforces its own vector limits and requires compatible dimensions for distance operations. +The `vector(N?)` factory is registered through the unified `CodecDescriptor<{ length?: number }>` shape — `paramsSchema` validates the dimension at the contract boundary when specified, `renderOutputType: ({ length }) => (length === undefined ? 'Vector' : 'Vector<' + length + '>')` produces the column's TS type for `contract.d.ts`, and the curried `factory` materializes the runtime codec at context construction. See [ADR 208 — Higher-order codecs for parameterized types](../../../docs/architecture%20docs/adrs/ADR%20208%20-%20Higher-order%20codecs%20for%20parameterized%20types.md) for the descriptor model. Any pgvector column can declare an explicit dimension via `vector(N)` for runtime validation and `Vector` typing, or use a standard `vector` to support variable dimensions. In either case, Postgres still enforces its own vector limits and requires compatible dimensions for distance operations. ### Runtime Setup