Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag

### Added

### D5-5 — 渲染源窗口化 + 判官 tailStart 视图域修复

- **渲染分窗(storage-review D5-5)**:`buildView` 不再全表加载正史行(数万行 × 每行媒体水合 = 每请求读放大)。新两层查询:`messageIdRows`(seq+msg_id 列,无字节,全行,驱动块锡定+接缝链)+ `uncoveredRows`(仅覆盖区间外的行取字节+媒体;covered 字节永不读——covered 行腐败不再导致渲染降级)。块锡定从字节匹配改 `msg_id` 列匹配零语义变化;两查询间隙新行的竞态→一次全量回退兼音。`unifiedTailViewRows` 保留为回退路径。
- **判官 tailStart 视图域(#4241 回归修复)**:`snapshotRenderTailStart` 原记客户端域下标(clientJoin),但判官用它索引 `judgeBaseline`(渲染视图)——视图前缀压缩后 clientJoin 可指向视图末尾之外→判官空转(total=0 空转 clean,T1 合车后把 #4241 牙齿测试打红)。改记视图域(视图行数 − 尾巴数)。牙齿测试同步改注尾巴行(引擎尾随 nudge 占末位,最后一条尾巴在 exitCores.length − 2):渲染前缀豁免是设计内(#4241:跨轮前缀稳定归 seamLcp/shadow),尾巴行丢失必须开火。

### #4316 层 3 PR3/3 — 存量编号一次性对齐正史 + fork 克隆保号

- **kernel `restampRefsToLedger`**:把既有引用空间一次性重排到正史账本(ref = store 行 seq + 1)。计数器路径按到达顺序发号但受保护消息不占号,而行 seq 照发 → 两者天然漂移;编辑与 churn 拉大缺口。重排后引用空间与存储真值一致,churn 不再改写编号。诚实边界:无映射 id 留给 assignRefs 的章路径;BLOCKED 仍不占号;死 ref 保留;被夺号的无行引用重安居新号(id 永不复用契约不破);多代(同 id 多行编辑)id 由 `seqByRaw` 排除不误绑。`CompressionState.refLedgerAligned` 标记一次性(`state-hydration` 字段单一事实源自动持久化)。
Expand Down
23 changes: 23 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1823,6 +1823,29 @@ export class SessionStore {
}
}

/** #D5-5 render windowing: identity-only rows (seq + msg_id). Null when
* the unified store is unavailable — the render lane degrades, never
* guesses. */
unifiedMessageIdRows(sessionId: string): { seq: number; msgId: string }[] | null {
if (!this.shadow || this.shadow.stats.disabled) return null;
try {
return this.shadow.messageIdRows(sessionId);
} catch {
return null;
}
}

/** #D5-5 render windowing: content rows outside the covered intervals.
* Null when the unified store is unavailable. */
unifiedUncoveredRows(sessionId: string, intervals: readonly [number, number][]): import("./storage/store.js").MessageRow[] | null {
if (!this.shadow || this.shadow.stats.disabled) return null;
try {
return this.shadow.uncoveredRows(sessionId, intervals);
} catch {
return null;
}
}

/** #2960 R2.1: summary rows for list surfaces (CLI export, web exposure
* tick). Full mode only — in shadow the JSON tree is the truth base and
* rows may lag files never yet recorded, so list surfaces must walk.
Expand Down
103 changes: 87 additions & 16 deletions src/server/snapshot-render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,11 @@ export function identitySigForTest(m: BiliMessage): string {

interface ViewRow {
seq: number;
msg: BiliMessage;
msgId: string;
/** Present only for UNCOVERED rows — covered rows' bytes never ride the
* outbound (suppressed behind anchors), so the windowed loader (#D5-5)
* never reads them. */
msg?: BiliMessage;
}

interface SnapshotRenderDeps {
Expand Down Expand Up @@ -152,17 +156,20 @@ function buildView(session: Session, clientMsgs: BiliMessage[]): ViewResult {
if (activeBlocks.length === 0) return { ok: false, reason: "no-active-blocks" };
const store = getStore();
if (!store) return { ok: false, reason: "store-unavailable" };
// #D5-5 windowed load — two tiers:
// (a) identity rows (seq + msg_id, no bytes) for EVERY live row —
// positions and ids drive block anchoring and the byId seam chain;
// (b) content rows (bytes + media) ONLY for seqs OUTSIDE the covered
// intervals — covered rows are suppressed behind anchors, so their
// bytes never ride the outbound and are never read. A covered row
// with corrupt bytes therefore cannot degrade the render.
let rows: ViewRow[] = [];
try {
const rawRows = store.unifiedTailViewRows(session.id);
const light = store.unifiedMessageIdRows(session.id);
const seqs = store.unifiedTailSeqs(session.id);
if (!rawRows || rawRows.length === 0 || !seqs || seqs.length === 0) return { ok: false, reason: "transcript-empty" };
if (!light || light.length === 0 || !seqs || seqs.length === 0) return { ok: false, reason: "transcript-empty" };
const bySeq = new Map<number, ViewRow>();
for (const row of rawRows) {
const msg = parseRow(row.content, row.seq);
if (!msg) return { ok: false, reason: "row-unparsable" }; // untrustworthy transcript: degrade, never guess
bySeq.set(row.seq, { seq: row.seq, msg });
}
for (const row of light) bySeq.set(row.seq, { seq: row.seq, msgId: row.msgId });
for (const seq of seqs) {
const v = bySeq.get(seq);
if (!v) return { ok: false, reason: "view-pointer-missing-row" }; // view pointer to a missing row: degrade
Expand All @@ -184,10 +191,9 @@ function buildView(session: Session, clientMsgs: BiliMessage[]): ViewResult {
const coveredSeqUnion = new Set<number>();
for (const block of activeBlocks) {
const blockSeqs = new Set<number>();
for (const id of block.effectiveMessageIds) {
for (let i = 0; i < rows.length; i++) {
if (rows[i].msg.id === id) blockSeqs.add(rows[i].seq);
}
const wanted = new Set(block.effectiveMessageIds);
for (let i = 0; i < rows.length; i++) {
if (wanted.has(rows[i].msgId)) blockSeqs.add(rows[i].seq);
}
// #3812: no fold-time id resolves against the view — the block cannot
// be positioned (and its covered region cannot be bounded). Anchoring
Expand All @@ -199,14 +205,67 @@ function buildView(session: Session, clientMsgs: BiliMessage[]): ViewResult {
blockSeeds.push({ blockId: block.blockId, seqs: blockSeqs });
}

// #D5-5: merge the covered seqs into disjoint intervals and load bytes
// only for the uncovered window. Row-unparsable now guards UNCOVERED
// bytes only — covered corruption cannot degrade the render.
try {
const intervals: [number, number][] = [];
{
const sorted = [...coveredSeqUnion].sort((a, b) => a - b);
for (const seq of sorted) {
const last = intervals[intervals.length - 1];
if (last && seq <= last[1] + 1) last[1] = Math.max(last[1], seq);
else intervals.push([seq, seq]);
}
}
const contentRows = store.unifiedUncoveredRows(session.id, intervals);
if (contentRows !== null) {
const bySeq = new Map<number, ViewRow>();
for (const row of contentRows) {
const msg = parseRow(row.content, row.seq);
if (!msg) return { ok: false, reason: "row-unparsable" }; // uncovered bytes are untrustworthy: degrade, never guess
bySeq.set(row.seq, { seq: row.seq, msgId: row.msgId, msg });
}
let coveredSeen = 0;
for (const r of rows) {
const v = bySeq.get(r.seq);
if (v) r.msg = v.msg;
else if (coveredSeqUnion.has(r.seq)) coveredSeen++;
}
// Every row must be accounted for: uncovered-with-bytes or
// covered. A gap (a row neither loaded nor covered — e.g. a
// concurrent ingest raced the two queries) falls back to one full
// load rather than rendering a hole.
if (coveredSeen !== [...coveredSeqUnion].filter((seq) => rows.some((r) => r.seq === seq)).length || rows.some((r) => !r.msg && !coveredSeqUnion.has(r.seq))) {
const full = store.unifiedTailViewRows(session.id);
if (full !== null) {
const fullBySeq = new Map<number, BiliMessage>();
for (const row of full) {
const msg = parseRow(row.content, row.seq);
if (msg) fullBySeq.set(row.seq, msg);
}
for (const r of rows) {
if (!r.msg) {
const m = fullBySeq.get(r.seq);
if (!m && !coveredSeqUnion.has(r.seq)) return { ok: false, reason: "row-unparsable" };
if (m) r.msg = m;
}
}
}
}
}
} catch {
return { ok: false, reason: "transcript-error" };
}

// Backward chain: from the client's end, match view rows (id fast path,
// then identity signature). The chain must be contiguous; the first
// unmatched client message bounds the client tail.
let joinRow = -1; // view index of the TOPMOST matched row — the join point
let clientJoin = clientMsgs.length; // first client index of the tail
{
const byId = new Map<string, number>(); // msgId -> view index (last occurrence)
for (let i = 0; i < rows.length; i++) byId.set(rows[i].msg.id, i);
for (let i = 0; i < rows.length; i++) byId.set(rows[i].msgId, i);
const bySig = new Map<string, number[]>(); // sig -> descending view indices
for (let i = rows.length - 1; i >= 0; i--) {
// #4052: covered rows take no signature binds. They never render
Expand All @@ -222,7 +281,9 @@ function buildView(session: Session, clientMsgs: BiliMessage[]): ViewResult {
// an exact id is unambiguous (kernel contract: ids never reused)
// and steady-state seams need it.
if (coveredSeqUnion.has(rows[i].seq)) continue;
const sig = identitySig(rows[i].msg);
const m = rows[i].msg;
if (!m) continue;
const sig = identitySig(m);
const list = bySig.get(sig);
if (list) list.push(i);
else bySig.set(sig, [i]);
Expand Down Expand Up @@ -319,12 +380,22 @@ function buildView(session: Session, clientMsgs: BiliMessage[]): ViewResult {
for (let i = 0; i < d; i++) {
while (ai < anchors.length && anchors[ai].at === i) merged.push(anchors[ai++].msg);
if (coveredSeqUnion.has(rows[i].seq)) continue;
merged.push(rows[i].msg);
const m = rows[i].msg;
if (!m) return { ok: false, reason: "row-unparsable" }; // uncovered row without bytes: never render a hole
merged.push(m);
}
while (ai < anchors.length) merged.push(anchors[ai++].msg);
for (let k = clientJoin; k < clientMsgs.length; k++) merged.push(clientMsgs[k]);

return { ok: true, view: merged, tailStart: clientJoin };
// #4241 tailStart is recorded in VIEW domain: the judge indexes its
// baseline (judgeBaseline === this rendered view) with it, so it must
// count the view rows that precede the first tail row. The client-domain
// join (clientJoin) is NOT a valid index into the view — the view prefix
// suppressed covered rows, so clientJoin can point past the view's end and
// the judge would validate nothing (vacuous clean — the regression that
// blinded the #4241 teeth on the T1-consolidation train).
const tailCount = clientMsgs.length - clientJoin;
return { ok: true, view: merged, tailStart: merged.length - tailCount };
}

export function maybeSnapshotView(session: Session, clientMsgs: BiliMessage[], deps: SnapshotRenderDeps): BiliMessage[] {
Expand Down
14 changes: 14 additions & 0 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,20 @@ export class UnifiedShadow {
* coverage matcher needs (msgId for fast path, ref for the
* upgrade-tolerant interval path, contentHash for the exact-content
* dedup path). */
/** #D5-5: identity-only rows (no content bytes) for the render lane. */
messageIdRows(sessionId: string): { seq: number; msgId: string }[] {
if (this.sync === null) return [];
this.assertLive();
return this.sync.messageIdRows(sessionId);
}

/** #D5-5: content rows outside the covered intervals only. */
uncoveredRows(sessionId: string, intervals: readonly [number, number][]): (MessageRow & { contentHash: string })[] {
if (this.sync === null) return [];
this.assertLive();
return this.sync.uncoveredRows(sessionId, intervals);
}

tailViewRows(sessionId: string): MessageRow[] {
if (this.sync === null) return [];
this.assertLive();
Expand Down
37 changes: 37 additions & 0 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2219,6 +2219,43 @@ export class UnifiedStore {
.run(hash);
}

/** #D5-5 render windowing: identity-only rows (seq + msg_id) — no
* content bytes, no media hydration. The render lane needs every row's
* POSITION and id (block anchoring, seam chain byId), but covered rows'
* BYTES never ride the outbound — loading them per request is pure read
* amplification on big folded sessions. */
messageIdRows(sessionId: string): { seq: number; msgId: string }[] {
const rows = this.db
.prepare("SELECT seq, msg_id FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq")
.all(sessionId) as Record<string, unknown>[];
return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string }));
}

/** #D5-5 render windowing: full rows (content + media) ONLY for seqs
* outside the covered intervals. Inclusive [start, end] pairs; overlaps
* are harmless (each pair contributes an outside-of-pair conjunct).
* Empty interval list = every live row. */
uncoveredRows(sessionId: string, intervals: readonly [number, number][]): (MessageRow & { contentHash: string })[] {
let sql = "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? AND retired_at IS NULL";
const params: unknown[] = [sessionId];
for (const [a, b] of intervals) {
sql += " AND (seq < ? OR seq > ?)";
params.push(a, b);
}
sql += " ORDER BY seq";
const rows = this.db.prepare(sql).all(...params) as Record<string, unknown>[];
return rows.map((r) => ({
seq: r.seq as number,
msgId: r.msg_id as string,
ref: (r.ref as string | null) ?? null,
refNum: (r.ref_num as number | null) ?? null,
role: r.role as string,
kind: r.kind as string,
contentHash: r.content_hash as string,
content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""),
}));
}

getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] {
// #4390: retired rows (rewound/undo history) stay out of the unified
// replay by default — they are history, not view. Callers that need
Expand Down
Loading
Loading