From 8db5b1f43bfbde9f1b6ef1eae9417e3d1f53e513 Mon Sep 17 00:00:00 2001 From: Ofri Wolfus Date: Thu, 29 Oct 2026 12:31:21 +0200 Subject: [PATCH] refactor(cfds): remove legacy ref machinery and reject legacy data at ingress --- CHANGELOG.md | 4 + cfds/base/types/core-type.ts | 49 +-- cfds/base/types/index.ts | 80 ++--- cfds/base/types/map-type.ts | 59 +--- cfds/base/types/richtext3-type.ts | 93 ++---- cfds/base/types/set-type.ts | 48 +-- cfds/base/types/string-type.ts | 14 - cfds/base/types/utils.ts | 69 ---- cfds/richtext/composer.ts | 362 --------------------- cfds/richtext/doc-state.ts | 417 ------------------------- cfds/richtext/legacy-ref.ts | 52 +++ cfds/richtext/markdown.ts | 39 --- cfds/richtext/model.ts | 54 +--- cfds/richtext/normalize/index.ts | 12 +- cfds/richtext/normalize/rules.ts | 30 +- cfds/richtext/normalize/setup.ts | 11 +- cfds/richtext/normalize/types.ts | 9 +- cfds/richtext/shortcuts.ts | 336 -------------------- cfds/richtext/tree.ts | 44 +-- cfds/richtext/utils.ts | 30 -- tests/core-value-serialization.test.ts | 149 +++++++++ 21 files changed, 312 insertions(+), 1649 deletions(-) delete mode 100644 cfds/base/types/string-type.ts delete mode 100644 cfds/base/types/utils.ts delete mode 100644 cfds/richtext/composer.ts delete mode 100644 cfds/richtext/doc-state.ts create mode 100644 cfds/richtext/legacy-ref.ts delete mode 100644 cfds/richtext/markdown.ts delete mode 100644 cfds/richtext/shortcuts.ts delete mode 100644 cfds/richtext/utils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 992c58a1..69b82573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Breaking Changes +- Legacy rich-text refs (`RefMarker` leaves or `RefNode` elements) now throw + on read, normalize, and serialize instead of being silently stripped. Back up + the store first. Use a release that supports legacy refs to migrate affected + data, or remove that data and re-sync it from a peer. - `coreValueHash` alias removed. Use `encodableValueHash`. - Clone aliasing: `coreValueClone`/`coreObjectClone` now preserve shared-ref aliasing across array/set siblings. Aliases within object fields were already diff --git a/cfds/base/types/core-type.ts b/cfds/base/types/core-type.ts index 37465a16..37111d3e 100644 --- a/cfds/base/types/core-type.ts +++ b/cfds/base/types/core-type.ts @@ -1,13 +1,17 @@ -import { IValueTypeOperations, ValueType, ValueTypeOptions } from './index.ts'; -import { DecodedValue } from '../../../base/core-types/encoding/index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; +import type { + IValueTypeOperations, + ValueType, + ValueTypeOptions, +} from './index.ts'; +import type { DecodedValue } from '../../../base/core-types/encoding/index.ts'; +import type { Change, EncodedChange } from '../../change/index.ts'; import { FieldChange } from '../../change/field-change.ts'; import { - ConcreteCoreValue, - CoreType, + type ConcreteCoreValue, + type CoreType, coreValueClone, coreValueEquals, - Encoder, + type Encoder, getCoreTypeOrUndef, } from '../../../base/core-types/index.ts'; @@ -30,13 +34,11 @@ export abstract class CoreTypeOperations< return value; } - fillRefs(refs: Set, value: TValue): void {} - - validate(value: any): boolean { + validate(value: unknown): boolean { return getCoreTypeOrUndef(value) === this._coreType; } - equals(val1: TValue, val2: TValue, options?: ValueTypeOptions): boolean { + equals(val1: TValue, val2: TValue, _options?: ValueTypeOptions): boolean { return coreValueEquals(val1, val2); } @@ -48,48 +50,29 @@ export abstract class CoreTypeOperations< key: string, value: TValue, encoder: Encoder, - options?: ValueTypeOptions, + _options?: ValueTypeOptions, ): void { encoder.set(key, value); } - deserialize(value: DecodedValue, options?: ValueTypeOptions) { + deserialize(value: DecodedValue, _options?: ValueTypeOptions) { return value as TValue; } valueAddedDiff( value2: TValue, - options?: ValueTypeOptions, + _options?: ValueTypeOptions, ): undefined | Change | Change[] { return FieldChange.insert(value2, this._valueType); } valueRemovedDiff( value1: TValue, - options?: ValueTypeOptions, + _options?: ValueTypeOptions, ): undefined | Change | Change[] { return FieldChange.delete(value1, this._valueType); } - needGC(value: TValue): boolean { - return false; - } - - gc(value: TValue): TValue | undefined { - return undefined; - } - - rewriteRefs( - keyMapping: Map, - value: TValue, - deleteRefs?: Set, - ): TValue | undefined { - if (deleteRefs && deleteRefs.has(value as string)) { - return undefined; - } - return (keyMapping.get(value as string) as TValue) || value; - } - abstract valueChangedDiff( value1: TValue, value2: TValue, diff --git a/cfds/base/types/index.ts b/cfds/base/types/index.ts index 1230e7c7..f158b656 100644 --- a/cfds/base/types/index.ts +++ b/cfds/base/types/index.ts @@ -1,17 +1,16 @@ -import { DecodedValue } from '../../../base/core-types/encoding/index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; +import type { DecodedValue } from '../../../base/core-types/encoding/index.ts'; +import type { ChecksumEncoderOpts } from '../../../base/core-types/encoding/checksum.ts'; +import type { Change, EncodedChange } from '../../change/index.ts'; import { MapTypeOperations } from './map-type.ts'; import { PrimitiveTypeOperations } from './primitive-type.ts'; import { SetTypeOperations } from './set-type.ts'; -import { StringTypeOperations } from './string-type.ts'; import { RichText3TypeOperations } from './richtext3-type.ts'; import { CoreType, - CoreValue, - Encoder, + type CoreValue, + type Encoder, } from '../../../base/core-types/index.ts'; import { DateTypeOperations } from './date-type.ts'; -import { ChecksumEncoderOpts } from '../../../base/core-types/encoding/checksum.ts'; export type ValueType = | 'string' @@ -84,73 +83,32 @@ export interface IValueTypeOperations { options?: ValueTypeOptions, ): T | undefined; - fillRefs(refs: Set, value: T): void; - normalize(value: T): T; isEmpty(value: T): boolean; - - needGC(value: T): boolean; - gc(value: T): T | undefined; - - rewriteRefs( - keyMapping: Map, - value: T, - deleteRefs?: Set, - ): T | undefined; } -const registeredTypeOperations: Record< - string, - IValueTypeOperations -> = {}; +/** Every `ValueType` has exactly one op set. `satisfies` + `Object.freeze` + * make this exhaustive and immutable at compile time. */ +const kTypeOperations = Object.freeze({ + string: new PrimitiveTypeOperations(CoreType.String, 'string'), + number: new PrimitiveTypeOperations(CoreType.Number, 'number'), + boolean: new PrimitiveTypeOperations(CoreType.Boolean, 'boolean'), + date: new DateTypeOperations(), + set: new SetTypeOperations('set'), + map: new MapTypeOperations('map'), + richtext: new RichText3TypeOperations(), +}) satisfies Record>; export function getTypeOperations( type: ValueType, ): IValueTypeOperations { - const op = registeredTypeOperations[type]; - if (op === undefined) { - throw new Error(`type operations for: ${type} has not been implemented`); - } + // `kTypeOperations` satisfies Record, so the lookup is total. + // Widen to the erased op type before narrowing to `T` (TS needs the step). + const op: IValueTypeOperations = kTypeOperations[type]; return op as IValueTypeOperations; } -export function getTypeOperationsByValue( - value: T, -): IValueTypeOperations { - for (const typeOP of Object.values(registeredTypeOperations)) { - if (typeOP.validate(value)) { - return typeOP as IValueTypeOperations; - } - } - - throw new Error(`getTypeOperationsByValue failed for: ${value}`); -} - -registerTypeOperations(); -function registerType(op: IValueTypeOperations): void { - registeredTypeOperations[op.valueType] = op as IValueTypeOperations< - CoreValue - >; -} - -function registerTypeOperations(): void { - if (Object.entries(registeredTypeOperations).length === 0) { - registerType(new StringTypeOperations(CoreType.String, 'string')); - registerType(new PrimitiveTypeOperations(CoreType.Number, 'number')); - registerType(new PrimitiveTypeOperations(CoreType.Boolean, 'boolean')); - registerType(new DateTypeOperations()); - registerType(new SetTypeOperations(false, 'set')); - // registerType(new SetTypeOperations(false, 'stringset')); - // registerType(new SetTypeOperations(false, 'refset')); - registerType(new MapTypeOperations(false, 'map')); - // registerType(new MapTypeOperations(false, 'refmap')); - registerType(new RichText3TypeOperations()); - - //Add Types Here - } -} - export function valueTypeEquals( type: ValueType, value1: TValue | undefined, diff --git a/cfds/base/types/map-type.ts b/cfds/base/types/map-type.ts index 5f834b29..40955bb1 100644 --- a/cfds/base/types/map-type.ts +++ b/cfds/base/types/map-type.ts @@ -1,16 +1,16 @@ -import { ValueType, ValueTypeOptions } from './index.ts'; +import type { ValueType, ValueTypeOptions } from './index.ts'; import { dictFromPrimitive, - Dictionary, - PrimitiveMap, + type Dictionary, + type PrimitiveMap, } from '../../../base/collections/dict.ts'; -import { DecodedValue } from '../../../base/core-types/encoding/index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; +import type { DecodedValue } from '../../../base/core-types/encoding/index.ts'; +import type { Change, EncodedChange } from '../../change/index.ts'; import { CoreTypeOperations } from './core-type.ts'; import { FieldChange, FieldOperation } from '../../change/field-change.ts'; import { - ConcreteCoreValue, - CoreKey, + type ConcreteCoreValue, + type CoreKey, CoreType, coreValueEquals, } from '../../../base/core-types/index.ts'; @@ -18,7 +18,7 @@ import { export class MapTypeOperations extends CoreTypeOperations< Dictionary > { - constructor(readonly isRef: boolean, valueType: ValueType) { + constructor(valueType: ValueType) { super(CoreType.Dictionary, valueType); } @@ -131,47 +131,4 @@ export class MapTypeOperations extends CoreTypeOperations< private createMap(): Dictionary { return new Map(); } - - override gc( - value: Dictionary, - ): Dictionary | undefined { - return undefined; - } - - override fillRefs( - refs: Set, - dict: Dictionary, - ): void { - if (this.isRef && dict !== undefined) { - for (const [key, value] of dict) { - if (typeof key === 'string') { - refs.add(key); - } - if (typeof value === 'string') { - refs.add(value); - } - } - } - } - - override rewriteRefs( - keyMapping: Map, - dict: Dictionary, - deleteRefs?: Set, - ): Dictionary { - if (!this.isRef) { - return dict; - } - const result = new Map(); - for (const [key, value] of dict) { - if (deleteRefs?.has(key) || deleteRefs?.has(value as string)) { - continue; - } - result.set( - keyMapping.get(key as string) || key, - keyMapping.get(value as string) || value, - ); - } - return result; - } } diff --git a/cfds/base/types/richtext3-type.ts b/cfds/base/types/richtext3-type.ts index 34e5bc1f..182c6b0f 100644 --- a/cfds/base/types/richtext3-type.ts +++ b/cfds/base/types/richtext3-type.ts @@ -1,39 +1,40 @@ -import { isRefMarker } from '../../richtext/model.ts'; +import type { SerializeValueTypeOptions, ValueTypeOptions } from './index.ts'; +import type { Change, EncodedChange } from '../../change/index.ts'; +import type { RichTextChange } from '../../change/richtext-change.ts'; import { - SerializeValueTypeOptions, - ValueType, - ValueTypeOptions, -} from './index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; -import { RichTextChange } from '../../change/richtext-change.ts'; -import { - CoreOptions, + type CoreOptions, CoreType, + type CoreValue, coreValueClone, coreValueEquals, - Encoder, + type Encoder, } from '../../../base/core-types/index.ts'; -import { DecodedValue } from '../../../base/core-types/encoding/index.ts'; +import type { DecodedValue } from '../../../base/core-types/encoding/index.ts'; import { diff, patch } from '../../richtext/diff-patch.ts'; import { flattenRichText } from '../../richtext/flat-rep.ts'; import { normalizeRichText } from '../../richtext/normalize/index.ts'; import { - dfs, initRichText, - isExpiredPointer, isRichText, onlyNoneLocal, - purgeExpiredPointers, - RichText, + type RichText, treeAtomKeyFilterIgnoreText, } from '../../richtext/tree.ts'; +import { assertNoLegacyRefs } from '../../richtext/legacy-ref.ts'; import { CoreTypeOperations } from './core-type.ts'; const kCoreValueOpts: CoreOptions = { objectFilterFields: treeAtomKeyFilterIgnoreText, }; -const expiredPointersBuffer = 1000 * 60 * 10; //10 minutes +/** + * Richtext-layer serialization options. `local`/`byCharacter` are inherited + * from the single source of truth in `ValueTypeOptions`; only checksum-specific + * `flatRep` is added here. + */ +export interface RichTextSerializeOptions extends SerializeValueTypeOptions { + flatRep?: boolean; +} export class RichText3TypeOperations extends CoreTypeOperations { constructor() { @@ -47,8 +48,10 @@ export class RichText3TypeOperations extends CoreTypeOperations { if (curValue === undefined) { curValue = initRichText(); } + assertNoLegacyRefs(curValue); const newValue = patch(curValue, changes as RichTextChange[], true); + assertNoLegacyRefs(newValue); return newValue; } @@ -57,11 +60,7 @@ export class RichText3TypeOperations extends CoreTypeOperations { return normalizeRichText(val); } - getRefOperation() { - return undefined; - } - - override validate(value: any): boolean { + override validate(value: CoreValue): boolean { return isRichText(value); } @@ -69,8 +68,9 @@ export class RichText3TypeOperations extends CoreTypeOperations { key: string, value: RichText, encoder: Encoder, - options?: SerializeValueTypeOptions, + options?: RichTextSerializeOptions, ): void { + assertNoLegacyRefs(value); const local = options?.local === true; //default false if (options?.flatRep === true) { @@ -91,10 +91,13 @@ export class RichText3TypeOperations extends CoreTypeOperations { } override deserialize(value: DecodedValue, options?: ValueTypeOptions) { - return value as RichText; + const decoded = value as RichText; + assertNoLegacyRefs(decoded); + return decoded; } override valueAddedDiff(value2: RichText, options?: ValueTypeOptions) { + assertNoLegacyRefs(value2); const local = options?.local === true; //Default is false const empty = initRichText(); @@ -106,6 +109,7 @@ export class RichText3TypeOperations extends CoreTypeOperations { value1: RichText, options?: ValueTypeOptions, ): Change | Change[] | undefined { + assertNoLegacyRefs(value1); const local = options?.local === true; //Default is false const empty = initRichText(); @@ -118,6 +122,8 @@ export class RichText3TypeOperations extends CoreTypeOperations { value2: RichText, options?: ValueTypeOptions, ) { + assertNoLegacyRefs(value1); + assertNoLegacyRefs(value2); return diff( value1, value2, @@ -146,45 +152,4 @@ export class RichText3TypeOperations extends CoreTypeOperations { override clone(value: RichText) { return coreValueClone(value); } - - override needGC(value: RichText): boolean { - if (value.pointers) { - for (const pointer of value.pointers) { - if (isExpiredPointer(pointer, expiredPointersBuffer)) { - return true; - } - } - } - return false; - } - - override gc(value: RichText): RichText | undefined { - return purgeExpiredPointers(value, expiredPointersBuffer); - } - - override fillRefs(refs: Set, value: RichText): void { - for (const [node] of dfs(value.root)) { - if (isRefMarker(node)) { - refs.add(node.ref); - } - } - } - - override rewriteRefs( - keyMapping: Map, - value: RichText, - deleteRefs?: Set, - ): RichText { - for (const [node, _depth, path] of dfs(value.root, true)) { - if (isRefMarker(node)) { - if (deleteRefs?.has(node.ref)) { - const children = path[path.length - 1].children!; - children.splice(children.indexOf(node), 1); - } else { - node.ref = keyMapping.get(node.ref) || node.ref; - } - } - } - return value; - } } diff --git a/cfds/base/types/set-type.ts b/cfds/base/types/set-type.ts index cd53f1aa..d92abe99 100644 --- a/cfds/base/types/set-type.ts +++ b/cfds/base/types/set-type.ts @@ -1,15 +1,17 @@ -import { ValueType, ValueTypeOptions } from './index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; +import type { ValueType, ValueTypeOptions } from './index.ts'; +import type { Change, EncodedChange } from '../../change/index.ts'; import { CoreTypeOperations } from './core-type.ts'; import { FieldChange, FieldOperation } from '../../change/field-change.ts'; -import { ConcreteCoreValue, CoreType } from '../../../base/core-types/index.ts'; -import { DecodedValue } from '../../../base/core-types/encoding/index.ts'; +import { + type ConcreteCoreValue, + CoreType, +} from '../../../base/core-types/index.ts'; import * as SetUtils from '../../../base/set.ts'; export class SetTypeOperations extends CoreTypeOperations< Set > { - constructor(readonly isRef: boolean, valueType: ValueType) { + constructor(valueType: ValueType) { super(CoreType.Set, valueType); } @@ -48,33 +50,6 @@ export class SetTypeOperations extends CoreTypeOperations< return newValue.size > 0 ? newValue : undefined; } - override fillRefs(refs: Set, value: Set): void { - if (this.isRef && value && value instanceof Set) { - for (const val of value) { - if (typeof val === 'string') { - refs.add(val); - } - } - } - } - - override rewriteRefs( - keyMapping: Map, - value: Set, - deleteRefs?: Set, - ): Set { - if (!this.isRef) { - return value; - } - const result = new Set(); - for (const v of value) { - if (!deleteRefs?.has(v as string)) { - result.add(keyMapping.get(v as string) || v); - } - } - return result; - } - valueChangedDiff( value1: Set, value2: Set, @@ -103,13 +78,4 @@ export class SetTypeOperations extends CoreTypeOperations< isEmpty(value: Set) { return value.size === 0; } - - override deserialize(value: DecodedValue, options?: ValueTypeOptions) { - const desValue = super.deserialize(value, options); - if (desValue instanceof Array) { - //Backward compatibility from old deserialize that save as array of json - return SetUtils.from(desValue, JSON.parse); - } - return desValue; - } } diff --git a/cfds/base/types/string-type.ts b/cfds/base/types/string-type.ts deleted file mode 100644 index eee2b962..00000000 --- a/cfds/base/types/string-type.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { ValueType } from './index.ts'; -import { CoreType } from '../../../base/core-types/index.ts'; -import { PrimitiveTypeOperations } from './primitive-type.ts'; - -export class StringTypeOperations extends PrimitiveTypeOperations { - // constructor(readonly isRef: boolean) { - // super(CoreType.String, isRef ? ValueType.REF : ValueType.STRING); - // } - // fillRefs(refs: Set, value: string | undefined): void { - // if (this.isRef && typeof value === 'string' && value.length > 0) { - // refs.add(value); - // } - // } -} diff --git a/cfds/base/types/utils.ts b/cfds/base/types/utils.ts deleted file mode 100644 index 4c6fb7c4..00000000 --- a/cfds/base/types/utils.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { - getTypeOperationsByValue, - IValueTypeOperations, - ValueTypeOptions, -} from './index.ts'; -import { Change, EncodedChange } from '../../change/index.ts'; -import { CoreValue } from '../../../base/core-types/index.ts'; - -export function diff( - value1: CoreValue, - value2: CoreValue, - typeOP?: IValueTypeOperations, - options: ValueTypeOptions = {}, -): undefined | Change | Change[] { - if (value1 === undefined && value2 === undefined) return; - - const lTypeOP = typeOP || getTypeOperationsByValue(value1); - - if (value1 === undefined && value2 !== undefined) { - //New Value - return lTypeOP.valueAddedDiff(value2, options); - } else if (value1 !== undefined && value2 === undefined) { - //Value Removed - return lTypeOP.valueRemovedDiff(value1, options); - } else { - //Value Changed - return lTypeOP.valueChangedDiff(value1, value2, options); - } -} - -export function patch( - value: CoreValue, - changes: Change[], - typeOP?: IValueTypeOperations, - options: ValueTypeOptions = {}, -): CoreValue { - const lTypeOP = typeOP || getTypeOperationsByValue(value); - const newValue = lTypeOP.patch(value, changes, options); - - return newValue; -} - -export function concatChanges( - changes1: undefined | Change | Change[], - changes2: undefined | Change | Change[], -): Change[] { - const result: Change[] = []; - if (changes1) { - if (Array.isArray(changes1)) { - for (const c of changes1) { - result.push(c); - } - } else { - result.push(changes1); - } - } - - if (changes2) { - if (Array.isArray(changes2)) { - for (const c of changes2) { - result.push(c); - } - } else { - result.push(changes2); - } - } - - return result; -} diff --git a/cfds/richtext/composer.ts b/cfds/richtext/composer.ts deleted file mode 100644 index 30b7d224..00000000 --- a/cfds/richtext/composer.ts +++ /dev/null @@ -1,362 +0,0 @@ -import { assert } from '../../base/error.ts'; -import { - DepthMarker, - FlatRepAtom, - flattenRichText, - isDepthMarker, - reconstructRichText, -} from './flat-rep.ts'; -import { - isRefMarker, - isRefNode, - RefMarker, - RefNode, - RefType, -} from './model.ts'; -import { - dfs, - ElementNode, - isElementNode, - isRichText, - RichText, - TextNode, -} from './tree.ts'; - -export enum RefPlaceholder { - Deleted, - Loading, -} - -/** - * A function that resolves a given ref value to RichText. - */ -export interface RefResolver { - (ref: string): RichText | RefPlaceholder; -} - -/** - * A function that writes an updated RichText value to a given reference. - */ -export interface RefUpdater { - (ref: string, rt: RichText): void; -} - -/** - * Converts all RefNode instances in the text to RefMarker elements. - * - * @param resolver A resolver function. - * @param rt The text to convert. - * @param local Include/exclude local tree elements. - * - * @returns A full RichText representation - */ -export function composeRichText( - resolver: RefResolver, - rt: RichText, - local = true, -): RichText { - // First, do a full tree scan and check if the composition will have pointers - // in it. - // if (compositeRichTextHasPointers(resolver, rt)) { - // If the result has pointers, we have no choice but to do a slow flat - // composition. - return reconstructRichText(composeFlatRichText(resolver, rt, local)); - // } - // // If no pointers are found, we simply clone the tree and replace reference - // // nodes inline which is significantly faster than flat composition. - // const result: RichText = coreValueClone(rt); - // fastComposeRichTextTreeNoPointers(resolver, result.root); - // return result; -} - -export function decomposeRichText( - updater: RefUpdater, - rt: RichText, - local = true, -): RichText { - // const atoms = Array.from(decomposeFlatRichText(updater, rt, local)); - // debugger; - // return reconstructRichText(atoms); - return reconstructRichText(decomposeFlatRichText(updater, rt, local)); -} - -function* composeFlatRichText( - resolver: RefResolver, - rt: RichText, - local: boolean, -): Generator { - let depth = 0; - for (const v of flattenRichText(rt, local, false)) { - if (isRefMarker(v)) { - const replacementRT = resolver(v.ref); - // Target is deleted. Skip this ref so it's also deleted in the composed - // result - if (replacementRT === RefPlaceholder.Deleted) { - continue; - } - // Target is loading. Spit a placeholder that works for Slate. - if (replacementRT === RefPlaceholder.Loading) { - yield { - ...v, - loading: true, - children: [], - } as RefMarker; - yield { - depthMarker: depth + 1, - } as DepthMarker; - yield { - text: '', - } as TextNode; - yield { - depthMarker: depth, - } as DepthMarker; - continue; - } - // Dynamically replace the ref marker with a RefNode element followed by - // the inner rich text contents - yield { - tagName: 'ref', - ref: v.ref, - type: v.type, - isLocal: v.isLocal === true, - children: [], - } as RefNode; - // Open a new level in the tree - yield { - depthMarker: depth + 1, - } as DepthMarker; - // Pass through all flat atoms of the resolved ref. - // Note: Replace flattenRichText() with a recursive call to - // composeRichText() if you wish to recursively resolve inner refs as well - for (const innerAtom of flattenRichText(replacementRT, local, false)) { - // Fixup inner depth markers' count - if (isDepthMarker(innerAtom)) { - yield { - depthMarker: depth + 1 + innerAtom.depthMarker, - isLocal: innerAtom.isLocal === true, - } as DepthMarker; - } else { - yield innerAtom; - } - } - // Close the tree level we opened - yield { - depthMarker: depth, - } as DepthMarker; - } else { - if (isDepthMarker(v)) { - depth = v.depthMarker; - } - yield v; - } - } -} - -function fastComposeRichTextTreeNoPointers( - resolver: RefResolver, - parent: ElementNode, -): void { - const children = parent.children; - for (let idx = 0; idx < children.length; ++idx) { - const child = children[idx]; - if (isElementNode(child)) { - fastComposeRichTextTreeNoPointers(resolver, child); - } else if (isRefMarker(child)) { - const replacementRT = resolver(child.ref); - if (replacementRT === RefPlaceholder.Deleted) { - // Target is deleted. Skip this ref so it's also deleted in the composed - // result - children.splice(idx, 1); - --idx; - continue; - } else if (replacementRT === RefPlaceholder.Loading) { - // Target is loading. Spit a placeholder that works for Slate. - children[idx] = { - tagName: 'ref', - type: child.type, - ref: child.ref, - loading: true, - isLocal: child.isLocal === true, - children: [ - { - text: '', - }, - ], - } as RefNode; - } else { - assert( - replacementRT.pointers === undefined || - replacementRT.pointers.size === 0, - ); - // Dynamically replace the ref marker with a RefNode element followed by - // the inner rich text contents - children[idx] = { - tagName: 'ref', - ref: child.ref, - type: child.type, - isLocal: child.isLocal === true, - children: replacementRT.root.children, - } as RefNode; - } - } - } -} - -/** - * This function does a full tree scan and checks if the resulting composition - * has pointers in it or not. - * @param resolver A resolver function. - * @param rt The base text. - * @returns Whether the resulting composition has pointers or not. - */ -function compositeRichTextHasPointers( - resolver: RefResolver, - rt: RichText, -): boolean { - if (rt.pointers !== undefined && rt.pointers.size > 0) { - return true; - } - for (const [node] of dfs(rt.root)) { - if (isRefMarker(node)) { - const replacementRT = resolver(node.ref); - if ( - isRichText(replacementRT) && - replacementRT.pointers !== undefined && - replacementRT.pointers.size > 0 - ) { - return true; - } - } - } - return false; -} - -function* decomposeFlatRichText( - updater: RefUpdater, - rt: RichText, - local: boolean, -): Generator { - let depth = 0; - const flatRepGenerator = flattenRichText(rt, local, false); - for (let atom of flatRepGenerator) { - // Dynamically replace RefNode elements with a RefMarker leaf, while - // updating the subtree at the relevant target - if (isRefNode(atom)) { - yield _updateLinkedDoc( - updater, - atom.ref, - flatRepGenerator, - depth, - atom.isLocal === true, - ); - } else if (isRefMarker(atom)) { - // Clean the loading flag from ref markers - const copy: RefMarker = { - ref: atom.ref, - type: atom.type, - }; - if (atom.isLocal) { - copy.isLocal = true; - } - yield copy; - } else { - // Keep track of current depth - if (isDepthMarker(atom)) { - depth = atom.depthMarker; - } - yield atom; - } - } -} - -function _updateLinkedDoc( - updater: RefUpdater, - key: string, - flatRepGenerator: Iterator, - markerDepth: number, - local: boolean, -): RefMarker { - const innerAtoms: FlatRepAtom[] = []; - // Fast forward our flat rep until we reach the end of this ref element - let { value: innerAtm, done } = flatRepGenerator.next(); - while (!done) { - if (isDepthMarker(innerAtm) && innerAtm.depthMarker <= markerDepth) { - // Reached a depth marker outside our parent. - // Apply updates - break; - } else { - // All inner atoms (including depth markers) pass unmodified. - // Inner nodes will naturally collapse into the root of the target. - innerAtoms.push(innerAtm); - } - const next = flatRepGenerator.next(); - innerAtm = next.value; - done = next.done; - } - - updater(key, reconstructRichText(innerAtoms)); - // Generate a RefMarker - const refMarker: RefMarker = { - ref: key, - type: RefType.InternalDoc, - }; - if (local) { - refMarker.isLocal = true; - } - return refMarker; -} - -export function extractRefs( - root: ElementNode, - local: boolean, - outSet?: Set, -): Set { - if (outSet === undefined) { - outSet = new Set(); - } - for (const [node] of dfs(root)) { - if (node.isLocal === true && !local) { - continue; - } - if (isRefMarker(node) || isRefNode(node)) { - outSet.add(node.ref); - } - } - return outSet; -} - -export function extractOrderedRefs( - root: ElementNode, - local: boolean, - outKeys?: string[], -): string[] { - if (outKeys === undefined) { - outKeys = []; - } - for (const [node] of dfs(root)) { - if (node.isLocal === true && !local) { - continue; - } - if (isRefMarker(node) || isRefNode(node)) { - if (!outKeys.includes(node.ref)) { - outKeys.push(node.ref); - } - } - } - return outKeys; -} - -/** - *

- * - *

- *

- *
    - *
      - *
    1. - * - * - * - * - *
      - */ diff --git a/cfds/richtext/doc-state.ts b/cfds/richtext/doc-state.ts deleted file mode 100644 index 5decf888..00000000 --- a/cfds/richtext/doc-state.ts +++ /dev/null @@ -1,417 +0,0 @@ -import { CoreObject } from '../../base/core-types/index.ts'; -import { - dfs, - ElementNode, - findFirstTextNode, - findLastTextNode, - isElementNode, - isPointer, - isTextNode, - pathToNode, - Point, - Pointer, - PointerDirection, - PointerType, - RichText, - TextNode, - TreeNode, -} from './tree.ts'; -import { assert, notReached } from '../../base/error.ts'; -import { TreeKeys } from './tree-keys.ts'; -import { - flattenRichText, - PointerValue, - projectPointers, - reconstructRichText, -} from './flat-rep.ts'; -import { Dictionary } from '../../base/collections/dict.ts'; -import { - resolveWritingDirection, - WritingDirection, -} from '../../base/string.ts'; -import { isRefNode, MarkupElement, MarkupNode } from './model.ts'; - -export interface Range extends CoreObject { - anchor: Point; - focus: Point; - dir: PointerDirection; - isLocal?: boolean; - expiration?: Date; -} - -export interface DocumentRanges extends CoreObject { - [key: string]: Range; -} - -export interface UnkeyedDocument extends CoreObject { - root: ElementNode; - ranges?: DocumentRanges; -} - -export interface Document extends UnkeyedDocument { - nodeKeys: TreeKeys; -} - -export function docFromRT(rt: RichText | UnkeyedDocument): Document { - const result: Document = { - root: rt.root, - nodeKeys: new TreeKeys(rt.root), - }; - - // if (rt.ranges !== undefined) { - // result.ranges = rt.ranges as DocumentRanges; - // } - - if (rt.pointers !== undefined) { - result.ranges = composeRanges(rt); - } - - return result; -} - -export function docToRT(doc: RichText | UnkeyedDocument): RichText { - const result: RichText = { - root: doc.root, - }; - // if (doc.pointers !== undefined) { - // result.pointers = doc.pointers as Set; - // } - if (doc.ranges !== undefined) { - const pointers = decomposeRanges(doc); - if (pointers.size > 0) { - result.pointers = pointers; - } - } - return result; -} - -export function docClone(doc: Document): Document { - return docFromRT( - reconstructRichText(flattenRichText(docToRT(doc), true, false)), - ); -} - -export function isDocument(doc: UnkeyedDocument): doc is Document { - return doc.nodeKeys instanceof TreeKeys; -} - -export function unkeyedDocToDoc(doc: UnkeyedDocument): Document { - if (isDocument(doc)) { - return doc; - } - return { - ...doc, - nodeKeys: new TreeKeys(doc.root), - }; -} - -export function composeRanges(rt: RichText): DocumentRanges { - const result: DocumentRanges = {}; - if (rt.pointers === undefined) { - return result; - } - let lastTextNode: TextNode | undefined; - let firstTextNode: TextNode | undefined; - for (const ptr of rt.pointers) { - if (isPointer(ptr)) { - // Skip expired pointers - // if (ptr.expiration && ptr.expiration.getTime() < now) { - // continue; - // } - let range = result[ptr.key]; - if (range === undefined) { - if (ptr.type === 'anchor') { - if (ptr.dir === PointerDirection.Forward) { - if (lastTextNode === undefined) { - lastTextNode = findLastTextNode(rt.root)!; - } - range = { - anchor: { - node: ptr.node, - offset: ptr.offset, - }, - focus: { - node: lastTextNode, - offset: lastTextNode.text.length, - }, - dir: ptr.dir, - }; - } else if (ptr.dir === PointerDirection.Backward) { - if (firstTextNode === undefined) { - firstTextNode = findFirstTextNode(rt.root)!; - } - range = { - anchor: { - node: ptr.node, - offset: ptr.offset, - }, - focus: { - node: firstTextNode, - offset: 0, - }, - dir: ptr.dir, - }; - } - } else { - assert(ptr.type === 'focus'); - if (ptr.dir === PointerDirection.Backward) { - if (lastTextNode === undefined) { - lastTextNode = findLastTextNode(rt.root)!; - } - range = { - anchor: { - node: lastTextNode, - offset: lastTextNode.text.length, - }, - focus: { - node: ptr.node, - offset: ptr.offset, - }, - dir: ptr.dir, - }; - } else if (ptr.dir === PointerDirection.Forward) { - if (firstTextNode === undefined) { - firstTextNode = findFirstTextNode(rt.root)!; - } - range = { - anchor: { - node: firstTextNode, - offset: 0, - }, - focus: { - node: ptr.node, - offset: ptr.offset, - }, - dir: ptr.dir, - }; - } - } - if (range === undefined) { - assert(ptr.dir === PointerDirection.None); - range = { - anchor: { - node: ptr.node, - offset: ptr.offset, - }, - focus: { - node: ptr.node, - offset: ptr.offset, - }, - dir: ptr.dir, - }; - } - // Update in resulting map - result[ptr.key] = range; - } else { - // Update an existing range entry - // TODO(ofri): Only allow expansion of the range - range[ptr.type] = { - node: ptr.node, - offset: ptr.offset, - }; - } - - // Bump expiration up if needed - if ( - ptr.expiration && - (range.expiration === undefined || - range.expiration.getTime() < ptr.expiration.getTime()) - ) { - range.expiration = ptr.expiration; - } - // Copy local flag - if (ptr.isLocal) { - range.isLocal = true; - } - } - } - return result; -} - -function nodesByFlatIndexes(rt: RichText): Dictionary { - const result = new Map(); - let idx = 0; - for (const [node] of dfs(rt.root)) { - result.set(node, idx); - if (isTextNode(node)) { - idx += node.text.length; - } - idx += 1; - } - return result; -} - -export function decomposeRanges(doc: UnkeyedDocument): Set { - if (doc.ranges === undefined) { - return new Set(); - } - const result = new Set(); - // const now = Date.now(); - const nodesToIndexes = nodesByFlatIndexes({ root: doc.root }); - for (const [key, range] of Object.entries(doc.ranges)) { - // if (range.expiration !== undefined && range.expiration.getTime() < now) { - // continue; - // } - const anchorNodeOffset = nodesToIndexes.get(range.anchor.node)!; - const focusNodeOffset = nodesToIndexes.get(range.focus.node)!; - const dir = anchorNodeOffset < focusNodeOffset - ? PointerDirection.Forward - : anchorNodeOffset > focusNodeOffset - ? PointerDirection.Backward - : range.anchor.offset < range.focus.offset - ? PointerDirection.Forward - : range.anchor.offset > range.focus.offset - ? PointerDirection.Backward - : PointerDirection.None; - result.add(buildRangePointer(key, range, 'anchor', dir)); - result.add(buildRangePointer(key, range, 'focus', dir)); - } - return result; -} - -function buildRangePointer( - key: string, - range: Range, - type: PointerType, - dir: PointerDirection, -): Pointer { - const result: Pointer = { - key, - type, - dir, - node: range[type].node, - offset: range[type].offset, - }; - if (range.expiration !== undefined) { - result.expiration = range.expiration; - } - if (range.isLocal) { - result.isLocal = true; - } - return result; -} - -export interface RangeFilter { - (key: string): boolean; -} - -/** - * Given two documents, this function projects pointers from the source to - * destination doc, adjusting for any edits that happened between the two docs. - * - * Use this function before updating a note's body to project ranges other than - * the ones you've changed. - * - * @param src Source document - * @param dst Destination document. This function updates the destination in - * place. - * @returns The updated destination document. - */ -export function projectRanges( - src: UnkeyedDocument, - dst: UnkeyedDocument, - filter: (ptr: PointerValue) => boolean, -): Document { - if (src.ranges === undefined) { - return unkeyedDocToDoc(dst); - } - return unkeyedDocToDoc( - docFromRT(projectPointers(docToRT(src), docToRT(dst), filter)), - ); -} - -function nodePathToIndexPath( - root: ElementNode, - nodePath: readonly ElementNode[], -): number[] { - const result: number[] = []; - let parent: ElementNode = root; - for (const element of nodePath) { - const idx = parent.children.indexOf(element); - assert(idx >= 0); - result.push(idx); - parent = element; - } - return result; -} - -export function pointToPath( - document: UnkeyedDocument, - point: Point, -): { path: number[]; offset: number } { - for (const [node, _depth, path] of dfs(document.root)) { - if (node === point.node) { - const indexPath = nodePathToIndexPath(document.root, path); - indexPath.push(path[path.length - 1].children.indexOf(node)); - return { - path: indexPath, - offset: point.offset, - }; - } - } - notReached(); -} - -export function findEndOfDocument(document: Document): TextNode | ElementNode { - let lastTextNode: TextNode | undefined; - let lastParent: ElementNode = document.root; - for (const [node] of dfs(document.root)) { - if (isTextNode(node)) { - lastTextNode = node; - } else if (isElementNode(node)) { - lastParent = node; - } - } - return lastTextNode || lastParent; -} - -export function writingDirectionAtTextNode( - doc: Document, - node: TextNode, - baseDirection: WritingDirection = 'auto', -): WritingDirection { - const focusPath = pathToNode(doc.root, node); - if (!focusPath) { - return baseDirection; - } - for (let i = focusPath.length - 1; i >= 0; --i) { - const node = focusPath[i]; - if (node.dir && node.dir !== 'auto') { - return node.dir; - } - } - const dir = resolveWritingDirection(node.text); - return dir === 'auto' ? baseDirection : dir; -} - -export function writingDirectionAtNode( - doc: Document, - node: MarkupNode, - baseDirection: WritingDirection = 'auto', -): WritingDirection { - if (!isTextNode(node)) { - const path = pathToNode(doc.root, node)!; - // For tasks, look at the previous task and copy its direction, meaning - // first tasks dictates the direction. - // if (!path.length && isRefNode(node)) { - // const root = doc.root; - // const idx = root.children.indexOf(node); - // if (idx > 0) { - // const maybePrevRef = root.children[idx - 1]; - // if (isRefNode(maybePrevRef)) { - // return writingDirectionAtNode(doc, maybePrevRef, baseDirection); - // } - // } - // } - for (const parent of path) { - const dir = writingDirectionAtNode(doc, parent, baseDirection); - if (dir !== 'auto') { - return dir; - } - } - } - const textNode = isTextNode(node) ? node : findFirstTextNode(node, true); - return textNode - ? writingDirectionAtTextNode(doc, textNode, baseDirection) - : baseDirection; -} diff --git a/cfds/richtext/legacy-ref.ts b/cfds/richtext/legacy-ref.ts new file mode 100644 index 00000000..aa4aba51 --- /dev/null +++ b/cfds/richtext/legacy-ref.ts @@ -0,0 +1,52 @@ +import { + type CoreValue, + isReadonlyCoreObject, +} from '../../base/core-types/index.ts'; +import { dfs, type ElementNode, isElementNode, type RichText } from './tree.ts'; + +/** + * Reject-never-strip guard for legacy rich-text refs. + * + * WHY: pre-`isRef` payloads (`RefMarker` leaves / `RefNode` elements) must fail + * loudly at every ingress instead of being silently stripped by normalization + * or serialization, so a legacy document is never silently truncated. This + * module intentionally avoids `model.ts` (whose ref types are deleted) and + * recognizes the shapes structurally. + */ +export function assertNoLegacyRefs(value: RichText): void { + if (value === undefined || value === null) return; + assertNodeIsNotLegacyRef(value); + const root = (value as { root?: CoreValue }).root; + if (!isElementNode(root)) return; + // WHY: dfs yields only descendants, so root itself needs an explicit check. + assertNodeIsNotLegacyRef(root); + for (const [node] of dfs(root as ElementNode)) { + assertNodeIsNotLegacyRef(node); + } + // Pointers hold TextNodes outside the tree; a legacy node smuggled here + // bypasses dfs. Accept Set (live) and Array (decoded) pointer collections. + const pointers = (value as { pointers?: unknown }).pointers; + if (pointers instanceof Set || Array.isArray(pointers)) { + for (const ptr of pointers as Iterable) { + assertNodeIsNotLegacyRef(ptr); + const target = (ptr as { node?: unknown }).node; + if (target !== undefined) assertNodeIsNotLegacyRef(target); + } + } +} + +function assertNodeIsNotLegacyRef(node: unknown): void { + if (!isLegacyRefNode(node)) return; + throw new Error( + 'Legacy rich text ref is not supported (RefMarker/RefNode): ' + + `${JSON.stringify(node)}. Delete the store and re-sync it.`, + ); +} + +/** A legacy ref leaf or element, matched structurally (no model.ts types). */ +function isLegacyRefNode(node: unknown): boolean { + if (!isReadonlyCoreObject(node)) return false; + const obj = node as { ref?: unknown; type?: unknown; tagName?: unknown }; + if (obj.tagName === 'ref') return true; + return typeof obj.ref === 'string' && typeof obj.type === 'string'; +} diff --git a/cfds/richtext/markdown.ts b/cfds/richtext/markdown.ts deleted file mode 100644 index ca4366b1..00000000 --- a/cfds/richtext/markdown.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { dfs, ElementNode, isTextNode } from './tree.ts'; - -export function treeToMarkdown(root: ElementNode | undefined): string { - if (!root) { - return ''; - } - let result = ''; - let prevDepth = 1; - for (const [node, depth] of dfs(root)) { - if (depth !== prevDepth) { - result += '\n'; - prevDepth = depth; - } - switch (node.tagName) { - case 'h1': - result += '# '; - break; - - case 'h2': - result += '## '; - break; - - case 'li': - result += '* '; - break; - - case 'ref': - result += '- '; - break; - - default: - break; - } - if (isTextNode(node)) { - result += node.text; - } - } - return result; -} diff --git a/cfds/richtext/model.ts b/cfds/richtext/model.ts index 0b0aa215..b3bb00bd 100644 --- a/cfds/richtext/model.ts +++ b/cfds/richtext/model.ts @@ -1,7 +1,3 @@ -import { - type CoreValue, - isReadonlyCoreObject, -} from '../../base/core-types/index.ts'; import { type ElementNode, isTextNode, @@ -20,8 +16,7 @@ export type MarkupElement = | Header6Node | UnorderedListNode | OrderedListNode - | ListItemNode - | RefNode; + | ListItemNode; export type MarkupNode = MarkupElement | TextNode; // | TableNode // | TableRowNode @@ -91,53 +86,6 @@ export interface TableCellNode extends ElementNode { readonly tagName: 'td'; } -export enum RefType { - Link = 'link', - InternalDoc = 'inter-doc', -} - -/** - * A leaf node marker for a reference. It is used in the following cases: - * - * - In persistent, decomposed, rich text. This node is what's actually stored - * in the body field of a note. - * - * - During composition, to represent a reference that's still loading. - */ -export interface RefMarker extends RichTextValue, NodeWithDirection { - ref: string; - readonly type: RefType; - readonly loading?: true; -} - -/** - * A composited reference node as an element. Its children hold a - * representation of the reference's target. - */ -export interface RefNode extends ElementNode, RefMarker, NodeWithDirection { - readonly tagName: 'ref'; - ref: string; - type: RefType; -} - -export function isRefNode(atom: CoreValue): atom is RefNode { - return ( - isReadonlyCoreObject(atom) && - atom.tagName === 'ref' && - typeof atom.ref === 'string' && - typeof atom.type === 'string' - ); -} - -export function isRefMarker(atom: CoreValue): atom is RefMarker { - return ( - isReadonlyCoreObject(atom) && - typeof atom.ref === 'string' && - typeof atom.type === 'string' && - (atom.local === undefined || atom.local === true) - ); -} - export interface SpanNode extends TextNode { readonly tagName: 'span'; bold?: boolean; diff --git a/cfds/richtext/normalize/index.ts b/cfds/richtext/normalize/index.ts index a7de517c..64f2cc3e 100644 --- a/cfds/richtext/normalize/index.ts +++ b/cfds/richtext/normalize/index.ts @@ -3,22 +3,24 @@ import { coreValueClone } from '../../../base/core-types/index.ts'; import { flattenRichText, reconstructRichText } from '../flat-rep.ts'; import { dfs, - ElementNode, + type ElementNode, initRichTextRoot, isElementNode, isNodeSameLocality, isTextNode, isTrivialTextNode, - Pointer, + type Pointer, pointersForNode, - RichText, - TextNode, - TreeNode, + type RichText, + type TextNode, + type TreeNode, } from '../tree.ts'; import { getRuleActions } from './rules.ts'; import { inlineElementTags } from './setup.ts'; +import { assertNoLegacyRefs } from '../legacy-ref.ts'; export function normalizeRichText(richText: RichText): RichText { + assertNoLegacyRefs(richText); // normalizeNodeText(richText, richText.root); richText = coreValueClone(richText); stripTrivialTextNodes(richText); diff --git a/cfds/richtext/normalize/rules.ts b/cfds/richtext/normalize/rules.ts index 5a295583..21d6f35d 100644 --- a/cfds/richtext/normalize/rules.ts +++ b/cfds/richtext/normalize/rules.ts @@ -1,7 +1,11 @@ -import { isRefMarker } from '../model.ts'; -import { ElementNode, isElementNode, isTextNode, TreeNode } from '../tree.ts'; +import { + type ElementNode, + isElementNode, + isTextNode, + type TreeNode, +} from '../tree.ts'; import { userDefinedRules } from './setup.ts'; -import { ConditionSource, RuleAction } from './types.ts'; +import type { ConditionSource, RuleAction } from './types.ts'; import * as SetUtils from '../../../base/set.ts'; interface SingleRuleCondition { @@ -25,13 +29,10 @@ const ELEMENT_KEY = 'elem'; const NOT_ELEMENT_KEY = 'n_elem'; const TEXT_KEY = 'txt'; const NOT_TEXT_KEY = 'n_txt'; -const REF_MARKER_KEY = 'rm'; -const NOT_REF_MARKER_KEY = 'n_rm'; -const ELEMENT_KEYS = [ELEMENT_KEY, NOT_TEXT_KEY, NOT_REF_MARKER_KEY]; -const TEXT_KEYS = [TEXT_KEY, NOT_ELEMENT_KEY, NOT_REF_MARKER_KEY]; -const REF_MARKER_KEYS = [REF_MARKER_KEY, NOT_ELEMENT_KEY, NOT_TEXT_KEY]; -const NON_KEYS = [NOT_TEXT_KEY, NOT_ELEMENT_KEY, NOT_REF_MARKER_KEY]; +const ELEMENT_KEYS = [ELEMENT_KEY, NOT_TEXT_KEY]; +const TEXT_KEYS = [TEXT_KEY, NOT_ELEMENT_KEY]; +const NON_KEYS = [NOT_TEXT_KEY, NOT_ELEMENT_KEY]; function getKeysForNode(node: TreeNode) { if (typeof node.tagName === 'string') { @@ -42,8 +43,6 @@ function getKeysForNode(node: TreeNode) { tags.push(...ELEMENT_KEYS); } else if (isTextNode(node)) { tags.push(...TEXT_KEYS); - } else if (isRefMarker(node)) { - tags.push(...REF_MARKER_KEYS); } else { tags.push(...NON_KEYS); } @@ -58,8 +57,6 @@ function getKeysForNode(node: TreeNode) { return ELEMENT_KEYS; } else if (isTextNode(node)) { return TEXT_KEYS; - } else if (isRefMarker(node)) { - return REF_MARKER_KEYS; } return NON_KEYS; @@ -179,13 +176,6 @@ function convertNodeCondition(node: any): string[] | undefined { } } - if (typeof node.isRefMarker === 'boolean') { - if (node.isRefMarker) { - keys.push(REF_MARKER_KEY); - } else { - keys.push(NOT_REF_MARKER_KEY); - } - } return keys; } diff --git a/cfds/richtext/normalize/setup.ts b/cfds/richtext/normalize/setup.ts index e47ec30d..5e064640 100644 --- a/cfds/richtext/normalize/setup.ts +++ b/cfds/richtext/normalize/setup.ts @@ -1,5 +1,5 @@ -import { ElementNode, isElementNode } from '../tree.ts'; -import { RuleSetup } from './types.ts'; +import { type ElementNode, isElementNode } from '../tree.ts'; +import type { RuleSetup } from './types.ts'; export const userDefinedRules: RuleSetup[] = [ { @@ -83,7 +83,10 @@ export const userDefinedRules: RuleSetup[] = [ }, }, { - node: { isRefMarker: true, tag: 'inline-task' }, + // WHY broadened: legacy `isRefMarker` refs now throw at ingress + // (legacy-ref.ts), so all surviving `inline-task` nodes are non-ref and + // take the same `to-sibling-below` path the ref ones once took. + node: { tag: 'inline-task' }, condition: { source: 'grandparent', isDefined: true }, action: { name: 'to-sibling-below', @@ -126,7 +129,7 @@ export const userDefinedRules: RuleSetup[] = [ }, }, { - node: { tags: ['ref', 'inline-task'] }, + node: { tags: ['inline-task'] }, condition: { source: 'node', func: (n) => { diff --git a/cfds/richtext/normalize/types.ts b/cfds/richtext/normalize/types.ts index 4f3966f7..feeb63a4 100644 --- a/cfds/richtext/normalize/types.ts +++ b/cfds/richtext/normalize/types.ts @@ -1,4 +1,4 @@ -import { TreeNode } from '../tree.ts'; +import type { TreeNode } from '../tree.ts'; interface SingleTagCondition { tag: string; @@ -20,10 +20,6 @@ interface IsTextCondition { isText: boolean; } -interface IsRefMarkerCondition { - isRefMarker: boolean; -} - interface HasTagMarkerCondition { hasTag: boolean; } @@ -42,8 +38,7 @@ export type NodeCondition = | SingleTagCondition | MultipleTagsCondition | IsElementCondition - | IsTextCondition - | IsRefMarkerCondition; + | IsTextCondition; type RuleCondition = & { source: ConditionSource } diff --git a/cfds/richtext/shortcuts.ts b/cfds/richtext/shortcuts.ts deleted file mode 100644 index 248c571c..00000000 --- a/cfds/richtext/shortcuts.ts +++ /dev/null @@ -1,336 +0,0 @@ -import { uniqueId } from '../../base/common.ts'; -import { - FlatRepAtom, - isDepthMarker, - isElementSpacer, - kElementSpacer, -} from './flat-rep.ts'; -import { OrderedListNode } from './model.ts'; -import { isElementNode, isTextNode } from './tree.ts'; - -type ReplacerFunction = ( - buffer: readonly FlatRepAtom[], -) => FlatRepAtom[] | undefined; - -const ShortcutsByLength: Record = { - 5: [replaceH1, replaceUL, replaceTask], - 6: [replaceH2, replaceOL], - // 7: [replaceH3], - // 8: [replaceH4], - // 9: [replaceH5], - // 10: [replaceH6], -}; - -function replaceH1(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h1', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceH2(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === '#' && - isTextNode(buffer[5]) && - buffer[5].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h2', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceH3(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === '#' && - isTextNode(buffer[5]) && - buffer[5].text === '#' && - isTextNode(buffer[6]) && - buffer[6].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h3', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceH4(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === '#' && - isTextNode(buffer[5]) && - buffer[5].text === '#' && - isTextNode(buffer[6]) && - buffer[6].text === '#' && - isTextNode(buffer[7]) && - buffer[7].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h4', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceH5(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === '#' && - isTextNode(buffer[5]) && - buffer[5].text === '#' && - isTextNode(buffer[6]) && - buffer[6].text === '#' && - isTextNode(buffer[7]) && - buffer[7].text === '#' && - isTextNode(buffer[8]) && - buffer[8].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h5', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceH6(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '#' && - isTextNode(buffer[4]) && - buffer[4].text === '#' && - isTextNode(buffer[5]) && - buffer[5].text === '#' && - isTextNode(buffer[6]) && - buffer[6].text === '#' && - isTextNode(buffer[7]) && - buffer[7].text === '#' && - isTextNode(buffer[8]) && - buffer[8].text === '#' && - isTextNode(buffer[9]) && - buffer[9].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'h6', - }, - buffer[2], - { - text: '', - }, - ]; - } -} - -function replaceUL(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '*' && - isTextNode(buffer[4]) && - buffer[4].text === ' ' - ) { - return [ - buffer[0], - { - children: [], - tagName: 'ul', - }, - buffer[2], - { - tagName: 'li', - children: [], - }, - { depthMarker: buffer[2].depthMarker + 1 }, - { - text: '', - }, - ]; - } -} - -function replaceOL(buffer: readonly FlatRepAtom[]): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text.charCodeAt(0) >= 48 && - buffer[3].text.charCodeAt(0) <= 57 && - isTextNode(buffer[4]) && - buffer[4].text === '.' && - isTextNode(buffer[5]) && - buffer[5].text === ' ' - ) { - const ol: OrderedListNode = { - tagName: 'ol', - children: [], - }; - if (buffer[3].text !== '1') { - ol.start = parseInt(buffer[3].text); - } - return [ - buffer[0], - ol, - buffer[2], - { - tagName: 'li', - children: [], - }, - { depthMarker: buffer[2].depthMarker + 1 }, - { - text: '', - }, - ]; - } -} - -function replaceTask( - buffer: readonly FlatRepAtom[], -): FlatRepAtom[] | undefined { - if ( - isElementSpacer(buffer[0]) && - isElementNode(buffer[1]) && - isDepthMarker(buffer[2]) && - isTextNode(buffer[3]) && - buffer[3].text === '-' && - isTextNode(buffer[4]) && - buffer[4].text === ' ' - ) { - return [ - buffer[0], - { - tagName: 'ref', - type: 'inter-doc', - ref: uniqueId(), - children: [], - }, - buffer[2], - kElementSpacer, - { - tagName: 'p', - children: [], - }, - { depthMarker: buffer[2].depthMarker + 1 }, - ]; - } -} - -export function* applyShortcuts( - flatRep: Iterable, -): Generator { - const buffer: FlatRepAtom[] = []; - const detectionSizes = Object.keys(ShortcutsByLength).map((k) => parseInt(k)) - .sort((a, b) => a - b); - const minDetectionSize = detectionSizes[0]; - const maxDetectionSize = detectionSizes[detectionSizes.length - 1]; - for (const atom of flatRep) { - buffer.push(atom); - const bufferLen = buffer.length; - if (bufferLen >= minDetectionSize) { - for ( - let size = Math.min(maxDetectionSize, bufferLen); - size >= minDetectionSize; - --size - ) { - const offset = buffer.length - size; - const detectors = ShortcutsByLength[size] || []; - for (const f of detectors) { - const replacement = f(buffer.slice(offset)); - if (replacement) { - for (let i = 0; i < offset; ++i) { - yield buffer[i]; - } - for (const atom of replacement) { - yield atom; - } - buffer.splice(0, size + offset); - break; - } - } - } - } - if (buffer.length === maxDetectionSize) { - yield buffer[0]; - buffer.shift(); - } - } - for (const a of buffer) { - yield a; - } -} diff --git a/cfds/richtext/tree.ts b/cfds/richtext/tree.ts index aeda64e8..8c6113eb 100644 --- a/cfds/richtext/tree.ts +++ b/cfds/richtext/tree.ts @@ -214,8 +214,7 @@ export function isTrivialTextNode(node: TreeNode): node is TextNode { return false; } for (const k in node) { - // deno-lint-ignore no-prototype-builtins - if (node.hasOwnProperty(k) && k !== 'text') { + if (Object.hasOwn(node, k) && k !== 'text') { return false; } } @@ -398,47 +397,6 @@ export function isExpiredPointer(v: CoreValue, buffer = 0): boolean { ); } -/** - * @param rt - * @param buffer in milliseconds - * @returns - */ -export function purgeExpiredPointers( - rt: RichText, - buffer = 0, -): RichText | undefined { - if (rt.pointers === undefined || rt.pointers.size === 0) { - return undefined; - } - const now = Date.now(); - let foundExpired = false; - for (const ptr of rt.pointers) { - if ( - ptr.expiration !== undefined && - ptr.expiration.getTime() + buffer < now - ) { - foundExpired = true; - break; - } - } - if (foundExpired) { - const newPtrs = new Set(); - for (const ptr of rt.pointers) { - if (ptr.expiration === undefined || ptr.expiration.getTime() >= now) { - newPtrs.add(ptr); - } - } - const result: RichText = { - root: rt.root, - }; - if (newPtrs.size > 0) { - result.pointers = newPtrs; - } - return result; - } - return undefined; -} - export function* pointersForNode( ptrSet: Iterable | undefined, node: TreeNode, diff --git a/cfds/richtext/utils.ts b/cfds/richtext/utils.ts deleted file mode 100644 index 0c9e49c3..00000000 --- a/cfds/richtext/utils.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { JSONObject, ReadonlyJSONObject } from '../../base/interfaces.ts'; -import { - JSONCyclicalDecoder, - JSONCyclicalEncoder, -} from '../../base/core-types/encoding/json.ts'; -import { dfs, isTextNode, RichText, TextNode } from './tree.ts'; - -export function findFirstTextNode(rt: RichText): TextNode | undefined { - for (const [node] of dfs(rt.root)) { - if (isTextNode(node)) { - return node; - } - } -} - -export function richTextToJS(rt: RichText): JSONObject { - const jsEncoder = new JSONCyclicalEncoder(); - jsEncoder.set('rt', rt); - - const out = jsEncoder.getOutput() as ReadonlyJSONObject; - const rtJS = out['rt'] as JSONObject; - return rtJS; -} - -export function jsToRichtext(rtJS: JSONObject): RichText { - const jsDecoder = JSONCyclicalDecoder.get({ rt: rtJS }); - const out = jsDecoder.get('rt') as RichText; - jsDecoder.finalize(); - return out; -} diff --git a/tests/core-value-serialization.test.ts b/tests/core-value-serialization.test.ts index 3c6bd949..3fc245ac 100644 --- a/tests/core-value-serialization.test.ts +++ b/tests/core-value-serialization.test.ts @@ -1,3 +1,5 @@ +import { getTypeOperations } from '../cfds/base/types/index.ts'; +import { assertNoLegacyRefs } from '../cfds/richtext/legacy-ref.ts'; import { kTypeOps } from '../base/core-types/type-ops.ts'; import { CoreType } from '../base/core-types/base.ts'; import { assertExists } from './asserts.ts'; @@ -69,7 +71,42 @@ const kDateSchema: Schema = { fields: { when: { type: 'date' } }, }; +function assertLegacyIngress(doc: RichText): void { + const op = getTypeOperations('richtext'); + const before = JSON.stringify(doc); + for ( + const ingress of [ + () => op.patch(doc, []), + () => op.deserialize(doc as never), + () => op.normalize(doc), + () => normalizeRichText(doc), + () => op.serialize('body', doc, new JSONEncoder()), + () => op.valueAddedDiff(doc), + () => op.valueRemovedDiff(doc), + () => op.valueChangedDiff(doc, kRichTextDoc), + () => op.valueChangedDiff(kRichTextDoc, doc), + ] + ) { + assertThrows(ingress, 'Legacy rich text ref'); + assertEquals(JSON.stringify(doc), before); + } +} + export default function setup(): void { + TEST( + 'CoreValueSerialization', + 'rejects legacy refs in pointers without stripping', + () => { + const marker = { ref: 'r1', type: 'link' }; + for (const pointer of [marker, { node: marker }]) { + for (const pointers of [[pointer], new Set([pointer])]) { + assertLegacyIngress( + { ...kRichTextDoc, pointers } as unknown as RichText, + ); + } + } + }, + ); TEST( 'CoreValueSerialization', 'Date checksum and wire literal goldens', @@ -1091,4 +1128,116 @@ export default function setup(): void { assertExists(ops.childrenOf); } }); + + // ---------------- legacy richtext ref rejection ---------------- + + // Legacy RefMarker: a leaf carrying ref/type. + const kLegacyRefMarkerDoc = { + root: { + children: [ + { tagName: 'p', children: [{ ref: 'r1', type: 'link' }] }, + ], + }, + } as unknown as RichText; + + // Legacy RefNode: an element with tagName 'ref' wrapping its target. + const kLegacyRefNodeDoc = { + root: { + children: [ + { + tagName: 'p', + children: [ + { + tagName: 'ref', + ref: 'r1', + type: 'link', + children: [{ text: 'x' }], + }, + ], + }, + ], + }, + } as unknown as RichText; + + TEST( + 'CoreValueSerialization', + 'rejects a legacy RefMarker leaf at every richtext ingress', + () => { + assertLegacyIngress(kLegacyRefMarkerDoc); + }, + ); + + TEST( + 'CoreValueSerialization', + 'rejects a legacy RefNode element at every richtext ingress', + () => { + assertLegacyIngress(kLegacyRefNodeDoc); + }, + ); + + TEST( + 'CoreValueSerialization', + 'rejects a root-level legacy ref node', + () => { + // WHY: dfs yields only descendants, so a legacy ref placed at `root` + // itself needs the explicit check added to assertNoLegacyRefs. + const legacyRoot = { + root: { tagName: 'ref', ref: 'r1', type: 'link', children: [] }, + } as unknown as RichText; + assertThrows( + () => assertNoLegacyRefs(legacyRoot), + 'Legacy rich text ref', + ); + }, + ); + + TEST( + 'CoreValueSerialization', + 'legacy inline-task ref throws instead of moving to sibling below', + () => { + // WHY: assertNoLegacyRefs runs first in normalizeRichText, so an + // inline-task carrying legacy ref/type strings fails loudly instead + // of being relocated by the widened {tag:'inline-task'} rule. + const doc = { + root: { + children: [ + { + tagName: 'p', + children: [ + { + tagName: 'inline-task', + ref: 'r1', + type: 'link', + children: [{ text: 'task' }], + }, + ], + }, + ], + }, + } as unknown as RichText; + assertThrows(() => normalizeRichText(doc)); + }, + ); + + TEST( + 'CoreValueSerialization', + 'valid nodes carrying only ref or only type pass the legacy guard', + () => { + // WHY: the detector requires both string fields (or tagName 'ref'), + // so a TextNode with a lone `ref` and an element with a lone `type` + // are ordinary content and must not trip assertNoLegacyRefs. + const doc = { + root: { + children: [ + { + tagName: 'p', + type: 'link', + children: [{ text: 'x', ref: 'r1' }], + }, + ], + }, + } as unknown as RichText; + assertNoLegacyRefs(doc); + }, + ); }