diff --git a/README.md b/README.md index 4795ab2..5048e6d 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ Schema hierarchy levels support DAG (multi-parent) relationships via configurabl **Customizing Schemas:** - **Partial schemas**: Files starting with an underscore (like `_ost_tools_base.json`) are loaded and used to resolve references (using `$ref`). - **Loading priority**: Partial schemas are loaded from both the default schema directory and the directory of your specified target schema. +- **Transitive resolution**: `$ref` chains are resolved recursively across files/schemas (including nested `allOf` usage in partials). - **Unique IDs**: To encourage clean namespacing, local partial schemas **must** have unique `$id`s that do not collide with the default schemas. If a collision is detected, validation will fail with an error. Schema resolution order: CLI `--schema` > space config `schema` > global config `schema` > bundled `schemas/general.json` diff --git a/docs/schemas.md b/docs/schemas.md index 8d1af42..b3b24a3 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -163,7 +163,7 @@ The validator checks every node type and its parent type(s) against the hierarch Schemas are designed to be composable. You can create custom schemas by: 1. Creating a new `.json` file in the `schemas/` directory -2. Using `$ref` to reference shared definitions from `_shared.json` or other schemas +2. Using `$ref` to reference shared definitions from `_shared.json` or other schemas. This works transitively, including nested `allOf` compositions. 3. Defining your own node types and constraints Referencing another schema file merges its `$defs` into the compiled schema, including any `_metadata` block. If multiple referenced files each define `_metadata`, only the last one merged is used — `rules` arrays are not combined across sources. @@ -199,4 +199,4 @@ Schema files support JSON5 format, allowing inline documentation via `//` commen - [Teresa Torres' work on Opportunity Solution Trees](https://producttalk.org/2021/02/using-opportunity-solution-trees/) - "Continuous Discovery Habits" (2021) by Teresa Torres -- [JSON Schema specification](https://json-schema.org/) \ No newline at end of file +- [JSON Schema specification](https://json-schema.org/) diff --git a/src/commands/schemas.ts b/src/commands/schemas.ts index e1d243d..171ade6 100644 --- a/src/commands/schemas.ts +++ b/src/commands/schemas.ts @@ -3,7 +3,8 @@ import { dirname, join } from 'node:path'; import type { AnySchemaObject, SchemaObject } from 'ajv'; import JSON5 from 'json5'; import { loadConfig, resolveSchema } from '../config'; -import { buildFullRegistry, bundledSchemasDir, loadMetadata, mergeVariantProperties, readRawSchema } from '../schema'; +import { buildFullRegistry, bundledSchemasDir, loadMetadata, readRawSchema } from '../schema'; +import { mergeVariantProperties } from '../schema-refs'; import type { SchemaMetadata } from '../types'; function isBundledPath(schemaPath: string): boolean { diff --git a/src/commands/template-sync.ts b/src/commands/template-sync.ts index ca5a98e..a566cf9 100644 --- a/src/commands/template-sync.ts +++ b/src/commands/template-sync.ts @@ -5,7 +5,8 @@ import { glob } from 'glob'; import matter from 'gray-matter'; import yaml from 'js-yaml'; import { invertFieldMap } from '../config'; -import { buildFullRegistry, mergeVariantProperties, readRawSchema, resolveRef } from '../schema'; +import { buildFullRegistry, readRawSchema } from '../schema'; +import { mergeVariantProperties, resolveRef } from '../schema-refs'; interface TypeVariant { required: string[]; diff --git a/src/schema-refs.ts b/src/schema-refs.ts new file mode 100644 index 0000000..093e0cb --- /dev/null +++ b/src/schema-refs.ts @@ -0,0 +1,227 @@ +import type { AnySchemaObject } from 'ajv'; + +interface ResolvedSchema { + schema: AnySchemaObject; + rootSchema: AnySchemaObject; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function asArray(value: unknown): T[] { + return Array.isArray(value) ? (value as T[]) : []; +} + +function decodeJsonPointerToken(token: string): string { + return token.replace(/~1/g, '/').replace(/~0/g, '~'); +} + +function resolveJsonPointer(root: AnySchemaObject, pointer: string, fullRef: string): AnySchemaObject { + if (pointer === '') return root; + if (!pointer.startsWith('/')) { + throw new Error(`Unsupported $ref pointer "${fullRef}". Expected a JSON pointer (e.g. "#/$defs/name").`); + } + + let current: unknown = root; + for (const rawToken of pointer.slice(1).split('/')) { + const token = decodeJsonPointerToken(rawToken); + + if (Array.isArray(current)) { + const index = Number.parseInt(token, 10); + if (Number.isNaN(index) || index < 0 || index >= current.length) { + throw new Error(`Cannot resolve $ref "${fullRef}": array index "${token}" is out of bounds.`); + } + current = current[index]; + continue; + } + + if (!isObject(current) || !(token in current)) { + throw new Error(`Cannot resolve $ref "${fullRef}": token "${token}" does not exist.`); + } + + current = current[token]; + } + + if (!isObject(current)) { + throw new Error(`Cannot resolve $ref "${fullRef}": target is not an object schema.`); + } + + return current as AnySchemaObject; +} + +function mergeSchemaObjects(base: AnySchemaObject, overlay: AnySchemaObject): AnySchemaObject { + const merged = { ...base, ...overlay } as Record; + + const baseProps = isObject(base.properties) ? (base.properties as Record) : undefined; + const overlayProps = isObject(overlay.properties) + ? (overlay.properties as Record) + : undefined; + if (baseProps || overlayProps) { + merged.properties = { + ...(baseProps ?? {}), + ...(overlayProps ?? {}), + }; + } + + const baseRequired = asArray(base.required); + const overlayRequired = asArray(overlay.required); + if (baseRequired.length > 0 || overlayRequired.length > 0) { + merged.required = [...new Set([...baseRequired, ...overlayRequired])]; + } + + const baseAllOf = asArray(base.allOf); + const overlayAllOf = asArray(overlay.allOf); + if (baseAllOf.length > 0 || overlayAllOf.length > 0) { + merged.allOf = [...baseAllOf, ...overlayAllOf]; + } + + return merged as AnySchemaObject; +} + +function resolveRefTarget( + ref: string, + currentRootSchema: AnySchemaObject, + registry: Map, +): { schema: AnySchemaObject; rootSchema: AnySchemaObject; refKey: string } { + if (ref.startsWith('#')) { + const pointer = ref.slice(1); + const rootId = typeof currentRootSchema.$id === 'string' ? currentRootSchema.$id : '(root)'; + return { + schema: resolveJsonPointer(currentRootSchema, pointer, ref), + rootSchema: currentRootSchema, + refKey: `${rootId}#${pointer}`, + }; + } + + const hashIndex = ref.indexOf('#'); + const baseId = hashIndex >= 0 ? ref.slice(0, hashIndex) : ref; + const pointer = hashIndex >= 0 ? ref.slice(hashIndex + 1) : ''; + + const externalSchema = registry.get(baseId); + if (!externalSchema) { + throw new Error(`Cannot resolve external $ref: ${ref}`); + } + + return { + schema: resolveJsonPointer(externalSchema, pointer, ref), + rootSchema: externalSchema, + refKey: `${baseId}#${pointer}`, + }; +} + +function resolveRefWithContext( + def: AnySchemaObject | undefined, + rootSchema: AnySchemaObject, + registry: Map, + stack: Set, +): ResolvedSchema | undefined { + if (!def) return undefined; + const ref = typeof def.$ref === 'string' ? def.$ref : undefined; + if (!ref) { + return { schema: def, rootSchema }; + } + + const target = resolveRefTarget(ref, rootSchema, registry); + if (stack.has(target.refKey)) { + throw new Error(`Cyclic $ref detected: ${[...stack, target.refKey].join(' -> ')}`); + } + + stack.add(target.refKey); + const resolvedTarget = resolveRefWithContext(target.schema, target.rootSchema, registry, stack); + stack.delete(target.refKey); + + if (!resolvedTarget) return undefined; + + const overlay: Record = {}; + let hasOverlay = false; + for (const [k, v] of Object.entries(def)) { + if (k !== '$ref') { + overlay[k] = v; + hasOverlay = true; + } + } + + if (!hasOverlay) { + return resolvedTarget; + } + + return { + schema: mergeSchemaObjects(resolvedTarget.schema, overlay as AnySchemaObject), + rootSchema: resolvedTarget.rootSchema, + }; +} + +function flattenAllOf( + def: AnySchemaObject | undefined, + rootSchema: AnySchemaObject, + registry: Map, + stack: Set, + visited = new Set(), +): ResolvedSchema[] { + const resolved = resolveRefWithContext(def, rootSchema, registry, stack); + if (!resolved) return []; + + const schemaId = typeof resolved.schema.$id === 'string' ? resolved.schema.$id : undefined; + if (schemaId && visited.has(schemaId)) { + // Cycle detected via allOf: this schema is already being processed + return []; + } + if (schemaId) visited.add(schemaId); + + const parts: ResolvedSchema[] = []; + const allOf = asArray(resolved.schema.allOf); + for (const sub of allOf) { + parts.push(...flattenAllOf(sub, resolved.rootSchema, registry, stack, visited)); + } + + if (schemaId) visited.delete(schemaId); + + const own = { ...resolved.schema } as Record; + delete own.allOf; + parts.push({ schema: own as AnySchemaObject, rootSchema: resolved.rootSchema }); + return parts; +} + +/** + * Resolve a schema definition, following cross-file and internal refs transitively. + */ +export function resolveRef( + propDef: AnySchemaObject | undefined, + schema: AnySchemaObject, + registry: Map, +): AnySchemaObject | undefined { + return resolveRefWithContext(propDef, schema, registry, new Set())?.schema; +} + +/** + * Merge properties and required fields from allOf entries recursively across refs. + * allOf entries are flattened depth-first; direct properties on later fragments override earlier ones. + */ +export function mergeVariantProperties( + variant: AnySchemaObject, + schema: AnySchemaObject, + registry: Map, +): { properties: Record; required: string[] } { + const properties: Record = {}; + const requiredSet = new Set(); + const fragments = flattenAllOf(variant, schema, registry, new Set()); + + for (const fragment of fragments) { + const fragmentProps = isObject(fragment.schema.properties) + ? (fragment.schema.properties as Record) + : undefined; + + if (fragmentProps) { + for (const [key, value] of Object.entries(fragmentProps)) { + properties[key] = resolveRef(value as AnySchemaObject, fragment.rootSchema, registry) ?? value; + } + } + + for (const req of asArray(fragment.schema.required)) { + requiredSet.add(req); + } + } + + return { properties, required: [...requiredSet] }; +} diff --git a/src/schema.ts b/src/schema.ts index 0cf3cc9..aba8eed 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,7 +1,6 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import type { AnySchemaObject, SchemaObject } from 'ajv'; import Ajv, { type ValidateFunction } from 'ajv'; import JSON5 from 'json5'; import type { HierarchyLevel, RulesMetadata, SchemaMetadata } from './types'; @@ -89,67 +88,6 @@ export function createValidator(schemaPath: string): ValidateFunction { return ajv.compile(targetSchema); } -/** - * Resolve a $ref within a schema, handling both external refs (ost-tools://...) and internal refs (#/$defs/...). - * Used by template-sync for traversing schema structures. - */ -export function resolveRef( - propDef: AnySchemaObject | undefined, - schema: SchemaObject, - registry: Map, -): AnySchemaObject | undefined { - if (propDef?.$ref) { - const ref = propDef.$ref as string; - - // Handle external refs (e.g., "ost-tools://_shared#/$defs/baseNodeProps") - if (!ref.startsWith('#/')) { - const [baseId, hashPath] = ref.split('#'); - const externalSchema = registry.get(baseId ?? ''); - if (!externalSchema) { - throw new Error(`Cannot resolve external $ref: ${ref}`); - } - - // Resolve the hash path in the external schema - if (hashPath) { - const path = hashPath.replace(/^\//, '').split('/'); - // biome-ignore lint/suspicious/noExplicitAny: JSON schema traversal - return path.reduce((obj: any, key: string) => obj[key], externalSchema); - } - return externalSchema; - } - - // Handle internal refs (e.g., "#/$defs/baseNodeProps") - const path = ref.replace(/^#\//, '').split('/'); - // biome-ignore lint/suspicious/noExplicitAny: JSON schema traversal - return path.reduce((obj: any, key: string) => obj[key], schema); - } - return propDef; -} - -/** - * Merge properties and required fields from allOf refs and direct variant properties. - * allOf entries are resolved via resolveRef; direct properties take precedence. - */ -export function mergeVariantProperties( - variant: AnySchemaObject, - schema: SchemaObject, - registry: Map, -): { properties: Record; required: string[] } { - const properties: Record = {}; - const required: string[] = []; - - for (const sub of (variant.allOf as AnySchemaObject[] | undefined) ?? []) { - const resolved = resolveRef(sub, schema, registry); - Object.assign(properties, resolved?.properties ?? {}); - required.push(...((resolved?.required ?? []) as string[])); - } - - Object.assign(properties, variant.properties ?? {}); - required.push(...((variant.required ?? []) as string[])); - - return { properties, required: [...new Set(required)] }; -} - export function resolveNodeType(type: string, typeAliases: Record | undefined): string { return typeAliases?.[type] ?? type; } diff --git a/tests/fixtures/schema-refs/_leaf.json b/tests/fixtures/schema-refs/_leaf.json new file mode 100644 index 0000000..74815ac --- /dev/null +++ b/tests/fixtures/schema-refs/_leaf.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "test://leaf", + "$defs": { + "score": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "leafProps": { + "type": "object", + "properties": { + "score": { "$ref": "#/$defs/score" } + }, + "required": ["score"] + } + } +} diff --git a/tests/fixtures/schema-refs/_mid.json b/tests/fixtures/schema-refs/_mid.json new file mode 100644 index 0000000..3b074e8 --- /dev/null +++ b/tests/fixtures/schema-refs/_mid.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "test://mid", + "$defs": { + "mood": { + "type": "string", + "enum": ["happy", "sad"] + }, + "midProps": { + "type": "object", + "allOf": [{ "$ref": "test://leaf#/$defs/leafProps" }], + "properties": { + "mood": { "$ref": "#/$defs/mood" }, + "status": { "$ref": "ost-tools://_ost_tools_base#/$defs/status" } + }, + "required": ["mood", "status"] + } + } +} diff --git a/tests/fixtures/schema-refs/root.json b/tests/fixtures/schema-refs/root.json new file mode 100644 index 0000000..4e3825f --- /dev/null +++ b/tests/fixtures/schema-refs/root.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "test://root", + "oneOf": [ + { + "type": "object", + "allOf": [{ "$ref": "test://mid#/$defs/midProps" }], + "properties": { + "type": { "const": "thing" } + }, + "required": ["type"], + "examples": [ + { + "type": "thing", + "mood": "happy", + "status": "active", + "score": 3 + } + ] + } + ] +} diff --git a/tests/schema-refs.test.ts b/tests/schema-refs.test.ts new file mode 100644 index 0000000..6b6ed48 --- /dev/null +++ b/tests/schema-refs.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; +import type { AnySchemaObject, SchemaObject } from 'ajv'; +import { buildFullRegistry, readRawSchema } from '../src/schema'; +import { mergeVariantProperties, resolveRef } from '../src/schema-refs'; + +const ROOT_SCHEMA_PATH = join(import.meta.dir, 'fixtures/schema-refs/root.json'); + +describe('schema refs', () => { + it('resolves external refs transitively across multiple files', () => { + const schema = readRawSchema(ROOT_SCHEMA_PATH) as SchemaObject; + const registry = buildFullRegistry(ROOT_SCHEMA_PATH) as Map; + const variant = schema.oneOf?.[0] as AnySchemaObject; + + const { properties, required } = mergeVariantProperties(variant, schema, registry); + + expect((properties.mood as { enum?: string[] }).enum).toEqual(['happy', 'sad']); + expect((properties.status as { enum?: string[] }).enum).toContain('active'); + expect((properties.score as { minimum?: number; maximum?: number }).minimum).toBe(1); + expect((properties.score as { minimum?: number; maximum?: number }).maximum).toBe(5); + expect(required).toEqual(expect.arrayContaining(['type', 'mood', 'status', 'score'])); + }); + + it('resolves bundled refs from local schemas', () => { + const schema = readRawSchema(ROOT_SCHEMA_PATH) as SchemaObject; + const registry = buildFullRegistry(ROOT_SCHEMA_PATH) as Map; + + const status = resolveRef({ $ref: 'ost-tools://_ost_tools_base#/$defs/status' }, schema, registry) as { + enum?: string[]; + }; + + expect(status.enum).toContain('exploring'); + }); + + it('detects cycles in mutually recursive allOf schemas', () => { + // Schema A: { "$id": "A", "allOf": [{ "$ref": "B" }] } + // Schema B: { "$id": "B", "allOf": [{ "$ref": "A" }] } + const schemaA: AnySchemaObject = { + $id: 'http://example.com/A', + allOf: [{ $ref: 'http://example.com/B' }], + properties: { fromA: { type: 'string' } }, + }; + const schemaB: AnySchemaObject = { + $id: 'http://example.com/B', + allOf: [{ $ref: 'http://example.com/A' }], + properties: { fromB: { type: 'number' } }, + }; + + const registry = new Map([ + ['http://example.com/A', schemaA], + ['http://example.com/B', schemaB], + ]); + + // This should not throw or infinite loop - it should detect the cycle and return a result + const { properties, required } = mergeVariantProperties(schemaA, schemaA, registry); + + // Both properties should be present despite the cycle + expect(properties.fromA).toBeDefined(); + expect(properties.fromB).toBeDefined(); + expect(required).toEqual([]); + }); +});