From 7a3ac2d7a9253c4e816693214008ed3f0c4b72b6 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:06:36 +0800 Subject: [PATCH 1/2] fix: clarify token and video usage in model call logs --- .../internal/application/billing/service.go | 10 ++ .../billing/service_model_identity_test.go | 9 ++ .../conversation/service_billing.go | 11 ++- .../components/sections/logs/admin-logs.tsx | 95 ++++++++++++++++--- .../chat/components/message/message-bot.tsx | 21 +--- frontend/i18n/messages/en-US/admin-logs.json | 9 +- frontend/i18n/messages/en-US/chat.json | 1 - frontend/i18n/messages/zh-CN/admin-logs.json | 9 +- frontend/i18n/messages/zh-CN/chat.json | 1 - .../components/file-preview/preview-media.tsx | 5 +- 10 files changed, 127 insertions(+), 44 deletions(-) diff --git a/backend/internal/application/billing/service.go b/backend/internal/application/billing/service.go index 2371daa2..0f2e2b19 100644 --- a/backend/internal/application/billing/service.go +++ b/backend/internal/application/billing/service.go @@ -151,6 +151,8 @@ type UsagePricingInput struct { CallCount int64 DurationBillable bool DurationSeconds int64 + MediaType string + InputImageCount int64 LatencyMS int64 ServerSideToolUsage map[string]int64 ServiceItems []ServiceUsageInput @@ -1768,6 +1770,14 @@ func (s *Service) BuildUsageLedger(ctx context.Context, input UsagePricingInput) "base_service_billed_nanousd": serviceBilledNanousd, "service_items": usageServiceItemSnapshots(serviceItems), } + if strings.EqualFold(strings.TrimSpace(input.MediaType), "video") { + inputImageCount := input.InputImageCount + if inputImageCount < 0 { + inputImageCount = 0 + } + snapshot["media_type"] = "video" + snapshot["input_image_count"] = inputImageCount + } if usageSource := strings.TrimSpace(input.UsageSource); usageSource != "" { snapshot["usage_source"] = usageSource } diff --git a/backend/internal/application/billing/service_model_identity_test.go b/backend/internal/application/billing/service_model_identity_test.go index 3d221627..ac23f70c 100644 --- a/backend/internal/application/billing/service_model_identity_test.go +++ b/backend/internal/application/billing/service_model_identity_test.go @@ -101,6 +101,8 @@ func TestBuildUsageLedgerBillsDurationOnlyWhenExplicitlyBillable(t *testing.T) { PlatformModelName: "video-model", DurationBillable: true, DurationSeconds: 6, + MediaType: "video", + InputImageCount: 1, }) if err != nil { t.Fatalf("build video duration ledger: %v", err) @@ -108,6 +110,13 @@ func TestBuildUsageLedgerBillsDurationOnlyWhenExplicitlyBillable(t *testing.T) { if video.DurationSeconds != 6 || video.BilledNanousd != 18 { t.Fatalf("unexpected video duration billing: %#v", video) } + var snapshot map[string]interface{} + if err := json.Unmarshal([]byte(video.PricingSnapshotJSON), &snapshot); err != nil { + t.Fatalf("unmarshal video pricing snapshot: %v", err) + } + if snapshot["media_type"] != "video" || snapshot["input_image_count"] != float64(1) { + t.Fatalf("unexpected video media snapshot: %#v", snapshot) + } } func TestAuthorizeUsageRejectsLegacyDurationPricingForNonVideoModel(t *testing.T) { diff --git a/backend/internal/application/conversation/service_billing.go b/backend/internal/application/conversation/service_billing.go index 88cfd6cb..dfa90e4c 100644 --- a/backend/internal/application/conversation/service_billing.go +++ b/backend/internal/application/conversation/service_billing.go @@ -278,6 +278,13 @@ func (s *Service) buildSendMessageUsageLedger(ctx context.Context, input SendMes if result == nil { return nil, nil } + isVideoGeneration := sendMessageResultIsVideoGeneration(result) + mediaType := "" + inputImageCount := int64(0) + if isVideoGeneration { + mediaType = "video" + inputImageCount, _ = countAttachmentKinds(result.UserMessage.Attachments) + } latencyMS := result.LatencyMS if latencyMS <= 0 { latencyMS = result.AssistantMessage.LatencyMS @@ -305,8 +312,10 @@ func (s *Service) buildSendMessageUsageLedger(ctx context.Context, input SendMes OutputTokens: result.AssistantMessage.OutputTokens, ReasoningTokens: result.AssistantMessage.ReasoningTokens, CallCount: 1, - DurationBillable: sendMessageResultIsVideoGeneration(result), + DurationBillable: isVideoGeneration, DurationSeconds: sendMessageBillingDurationSeconds(result), + MediaType: mediaType, + InputImageCount: inputImageCount, LatencyMS: latencyMS, ServerSideToolUsage: result.ServerSideToolUsage, RawUsageJSON: result.RawUsageJSON, diff --git a/frontend/features/admin/components/sections/logs/admin-logs.tsx b/frontend/features/admin/components/sections/logs/admin-logs.tsx index 9039f630..ccf39e7f 100644 --- a/frontend/features/admin/components/sections/logs/admin-logs.tsx +++ b/frontend/features/admin/components/sections/logs/admin-logs.tsx @@ -167,13 +167,20 @@ function formatUsageBalance(value: number | null | undefined, billingDisplay: Bi return value === null || value === undefined ? "-" : formatBillingBalance(value, billingDisplay); } +function usageBillableOutputTokens(item: AdminUsageLogDTO): number { + return item.outputTokens + item.reasoningTokens; +} + function usageTotalTokens(item: AdminUsageLogDTO): number { - return item.inputTokens + item.cacheReadTokens + item.cacheWriteTokens + item.outputTokens + item.reasoningTokens; + return item.inputTokens + item.cacheReadTokens + item.cacheWriteTokens + usageBillableOutputTokens(item); } type UsagePricingSnapshot = { pricing_mode?: "token" | "call" | "duration" | "tiered" | string; provider_protocol?: string; + duration_billable?: boolean; + media_type?: string; + input_image_count?: number; cache_timeout?: string; fast_mode?: boolean; billing_speed?: string; @@ -494,7 +501,7 @@ function buildUsageBillingTooltipLines( const outputRate = readUsageSnapshotNumber(snapshot, "output_nanousd_per_m_tokens"); const cacheReadRate = readUsageSnapshotNumber(snapshot, "cache_read_nanousd_per_m_tokens"); const cacheWriteRate = readUsageSnapshotNumber(snapshot, "cache_write_nanousd_per_m_tokens"); - const billedOutputTokens = item.outputTokens + item.reasoningTokens; + const billedOutputTokens = usageBillableOutputTokens(item); const totalLine = usageTotalLine(item, labels, billingDisplay); const cacheWriteLabel = cacheWriteBillingLabel(snapshot, labels.billingDisplay); const cacheWriteNote = cacheWriteBillingNote(snapshot, labels.billingDisplay); @@ -604,26 +611,84 @@ function UsageLogModelCell({ item, labels }: { item: AdminUsageLogDTO; labels: U ); } -function UsageLogTokenCell({ item, locale }: { item: AdminUsageLogDTO; locale: string }) { +function UsageLogUsageCell({ item, locale }: { item: AdminUsageLogDTO; locale: string }) { const t = useTranslations("adminLogs.usage.tokens"); + const snapshot = parseUsagePricingSnapshot(item.pricingSnapshotJSON); + const isVideoUsage = snapshot.media_type === "video" || snapshot.duration_billable === true; + if (isVideoUsage) { + const inputImageCount = typeof snapshot.input_image_count === "number" && Number.isFinite(snapshot.input_image_count) && snapshot.input_image_count >= 0 + ? Math.trunc(snapshot.input_image_count) + : null; + const mediaUsage = [ + { label: t("input"), value: inputImageCount === null ? "—" : t("imageCount", { count: inputImageCount }) }, + { label: t("output"), value: t("secondCount", { count: item.durationSeconds }) }, + ]; + return ( +
+ {mediaUsage.map((entry) => ( + + {entry.label} + {entry.value} + + ))} +
+ ); + } const tokens = [ { label: t("inputShort"), value: item.inputTokens }, - { label: t("outputShort"), value: item.outputTokens }, + { + label: t("outputShort"), + value: usageBillableOutputTokens(item), + breakdown: { + visible: item.outputTokens, + reasoning: item.reasoningTokens, + }, + }, { label: t("cacheReadShort"), value: item.cacheReadTokens }, { label: t("cacheWriteShort"), value: item.cacheWriteTokens }, ]; return (
- {tokens.map((token) => ( - - {token.label} - {formatCount(token.value, locale)} - - ))} + {tokens.map((token) => { + const badge = ( + + {token.label} + {formatCount(token.value, locale)} + + ); + if (!token.breakdown) { + return {badge}; + } + return ( + + {badge} + +
+
+ {t("output")} + {formatCount(token.breakdown.visible, locale)} +
+
+ {t("reasoning")} + {formatCount(token.breakdown.reasoning, locale)} +
+ +
+ {t("outputTotal")} + {formatCount(token.value, locale)} +
+
+ + + ); + })}
); } @@ -1318,7 +1383,7 @@ function UsageLogTable({ ID {t("columns.caller")} {t("columns.model")} - Token + {t("columns.usage")} {t("columns.billing")} {t("columns.balanceAfter")} {t("columns.latency")} @@ -1340,7 +1405,7 @@ function UsageLogTable({ - + diff --git a/frontend/features/chat/components/message/message-bot.tsx b/frontend/features/chat/components/message/message-bot.tsx index 7a7cb56a..36b8153b 100644 --- a/frontend/features/chat/components/message/message-bot.tsx +++ b/frontend/features/chat/components/message/message-bot.tsx @@ -685,21 +685,12 @@ function MessageInlineVideoPreview({ const resolveErrorMessage = useLocalizedErrorMessage(); const objectURLRef = React.useRef(null); const [state, setState] = React.useState({ status: "loading" }); - const [detectedDurationSeconds, setDetectedDurationSeconds] = React.useState(); const fileID = attachment.fileID; const fileName = attachment.fileName; const mimeType = attachment.mimeType; const detectedMime = attachment.detectedMime; const previewURL = attachment.previewURL; const sizeBytes = attachment.sizeBytes; - const displayDurationSeconds = attachment.durationSeconds ?? detectedDurationSeconds; - - const handleDurationChange = React.useCallback((durationSeconds: number) => { - setDetectedDurationSeconds( - Number.isFinite(durationSeconds) && durationSeconds > 0 ? Math.ceil(durationSeconds) : undefined, - ); - }, []); - const revokeObjectURL = React.useCallback(() => { if (!objectURLRef.current) { return; @@ -711,7 +702,6 @@ function MessageInlineVideoPreview({ React.useEffect(() => { let cancelled = false; revokeObjectURL(); - setDetectedDurationSeconds(undefined); if (previewURL) { setState({ @@ -795,23 +785,14 @@ function MessageInlineVideoPreview({ } return ( -
+
- {displayDurationSeconds && displayDurationSeconds > 0 ? ( - - {displayDurationSeconds}s - - ) : null}
); } diff --git a/frontend/i18n/messages/en-US/admin-logs.json b/frontend/i18n/messages/en-US/admin-logs.json index 63038e4a..a4934e45 100644 --- a/frontend/i18n/messages/en-US/admin-logs.json +++ b/frontend/i18n/messages/en-US/admin-logs.json @@ -104,6 +104,7 @@ "message": "Message", "caller": "Caller", "model": "Model", + "usage": "Usage", "billing": "Billing", "balanceAfter": "Balance after settlement", "latency": "Latency", @@ -203,7 +204,13 @@ "inputShort": "In", "outputShort": "Out", "cacheReadShort": "Read", - "cacheWriteShort": "Write" + "cacheWriteShort": "Write", + "input": "Input", + "output": "Output", + "reasoning": "Reasoning", + "outputTotal": "Output total", + "imageCount": "{count, plural, one {# image} other {# images}}", + "secondCount": "{count, plural, one {# second} other {# seconds}}" }, "billing": { "cacheWrite": "Cache write", diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index f1175c2f..e9633258 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -187,7 +187,6 @@ "contextCompressed": "Context was automatically compressed", "scrollToBottom": "Scroll to bottom", "processing": "Processing...", - "videoDuration": "Video duration: {seconds} seconds", "expandUserMessage": "Expand", "collapseUserMessage": "Collapse", "editCreatesBranch": "Saving creates a new branch in this conversation.", diff --git a/frontend/i18n/messages/zh-CN/admin-logs.json b/frontend/i18n/messages/zh-CN/admin-logs.json index 217bf8d9..3b0e7431 100644 --- a/frontend/i18n/messages/zh-CN/admin-logs.json +++ b/frontend/i18n/messages/zh-CN/admin-logs.json @@ -104,6 +104,7 @@ "message": "消息", "caller": "调用人", "model": "模型", + "usage": "用量", "billing": "计费", "balanceAfter": "结算后余额", "latency": "延迟", @@ -203,7 +204,13 @@ "inputShort": "入", "outputShort": "出", "cacheReadShort": "读", - "cacheWriteShort": "写" + "cacheWriteShort": "写", + "input": "输入", + "output": "输出", + "reasoning": "推理", + "outputTotal": "输出合计", + "imageCount": "{count} 张", + "secondCount": "{count} 秒" }, "billing": { "cacheWrite": "缓存(写)", diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index 9647f9a3..733dd293 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -187,7 +187,6 @@ "contextCompressed": "上下文已自动压缩", "scrollToBottom": "回到底部", "processing": "正在处理…", - "videoDuration": "视频时长：{seconds} 秒", "expandUserMessage": "展开", "collapseUserMessage": "收起", "editCreatesBranch": "保存后会在当前 conversation 内创建新分支。", diff --git a/frontend/shared/components/file-preview/preview-media.tsx b/frontend/shared/components/file-preview/preview-media.tsx index a495c110..e8232a50 100644 --- a/frontend/shared/components/file-preview/preview-media.tsx +++ b/frontend/shared/components/file-preview/preview-media.tsx @@ -17,7 +17,6 @@ type PreviewMediaProps = { contentType?: string; toolbarContainer?: HTMLElement | null; inline?: boolean; - onDurationChange?: (durationSeconds: number) => void; }; const IMAGE_PREVIEW = { @@ -78,7 +77,6 @@ export function PreviewMedia({ contentType, toolbarContainer, inline = false, - onDurationChange, }: PreviewMediaProps) { const t = useTranslations("files.previewErrors"); const tPreview = useTranslations("files.preview"); @@ -306,8 +304,7 @@ export function PreviewMedia({ const nextDuration = media.duration || 0; setDuration(nextDuration); setCurrentTime(media.currentTime || 0); - onDurationChange?.(nextDuration); - }, [onDurationChange]); + }, []); const handleMediaLoadedMetadata = React.useCallback((event: React.SyntheticEvent) => { syncMediaMetrics(event.currentTarget); From 292b21ae1dfb34bf9363461d94712587af7fb312 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:52:51 +0800 Subject: [PATCH 2/2] fix: prevent retry messages from inheriting token usage --- .../chat/hooks/use-chat-message-submit.ts | 19 +- frontend/features/chat/model/chat-thread.ts | 6 + .../features/chat/model/message-submit.ts | 19 ++ frontend/package.json | 3 +- .../markdown/markdown-table-analyzer.test.mjs | 174 ------------------ 5 files changed, 33 insertions(+), 188 deletions(-) delete mode 100644 frontend/shared/components/markdown/markdown-table-analyzer.test.mjs diff --git a/frontend/features/chat/hooks/use-chat-message-submit.ts b/frontend/features/chat/hooks/use-chat-message-submit.ts index bb500e7d..b99d78d0 100644 --- a/frontend/features/chat/hooks/use-chat-message-submit.ts +++ b/frontend/features/chat/hooks/use-chat-message-submit.ts @@ -15,6 +15,7 @@ import type { ChatSubmitBlockReason } from "@/features/chat/model/chat-task"; import { resolveChatSubmitDecision } from "@/features/chat/model/chat-task"; import { useHiddenQueuedParentRuns } from "@/features/chat/hooks/use-hidden-queued-parent-runs"; import { + resolveAssistantInputSideUsageValue, resolveDefaultSubmissionParentMessage, resolvePersistedPublicID, toPendingAttachments, @@ -102,15 +103,6 @@ function streamEventErrorToApiError( return new ApiError(event.message || fallback, 502, event.debug, event.errorCode); } -function resolveInputSideUsageValue(...values: Array): number { - for (const value of values) { - if (typeof value === "number" && Number.isFinite(value) && value > 0) { - return value; - } - } - return 0; -} - function resolveMediaStatusLabel( status: string, fallbackMessage: string, @@ -1195,18 +1187,21 @@ export function useChatMessageSubmit({ assistantUpdatedAt: completed.assistantMessage.updatedAt, assistantContentType: completed.assistantMessage.contentType || current.assistantContentType, assistantAttachments: parseAttachments(completed.assistantMessage.attachments), - assistantInputTokens: resolveInputSideUsageValue( + assistantInputTokens: resolveAssistantInputSideUsageValue( + assistantOnlyBranch, completed.assistantMessage.inputTokens, completed.userMessage.inputTokens, current.assistantInputTokens, ), assistantOutputTokens: completed.assistantMessage.outputTokens, - assistantCacheReadTokens: resolveInputSideUsageValue( + assistantCacheReadTokens: resolveAssistantInputSideUsageValue( + assistantOnlyBranch, completed.assistantMessage.cacheReadTokens, completed.userMessage.cacheReadTokens, current.assistantCacheReadTokens, ), - assistantCacheWriteTokens: resolveInputSideUsageValue( + assistantCacheWriteTokens: resolveAssistantInputSideUsageValue( + assistantOnlyBranch, completed.assistantMessage.cacheWriteTokens, completed.userMessage.cacheWriteTokens, current.assistantCacheWriteTokens, diff --git a/frontend/features/chat/model/chat-thread.ts b/frontend/features/chat/model/chat-thread.ts index 205c4ade..76f9766c 100644 --- a/frontend/features/chat/model/chat-thread.ts +++ b/frontend/features/chat/model/chat-thread.ts @@ -368,6 +368,12 @@ export function buildVisibleMessages( if (item.role !== "assistant") { return item; } + // Assistant-only retries reuse the original user message, but own the + // prompt-side usage for their generation. A zero value is authoritative + // and must not fall back to the reused user's first-run usage. + if (item.branchReason === "retry" && item.sourcePublicID?.trim()) { + return item; + } const previous = index > 0 ? withBranchNavigators[index - 1] : null; if (!previous || previous.role !== "user") { return item; diff --git a/frontend/features/chat/model/message-submit.ts b/frontend/features/chat/model/message-submit.ts index e659d488..2461c991 100644 --- a/frontend/features/chat/model/message-submit.ts +++ b/frontend/features/chat/model/message-submit.ts @@ -41,6 +41,25 @@ export function resolvePersistedPublicID(value: string | null | undefined): stri return normalized; } +export function resolveAssistantInputSideUsageValue( + assistantOwnsUsage: boolean, + assistantValue: number | null | undefined, + userValue: number | null | undefined, + liveValue: number | null | undefined, +): number { + if (assistantOwnsUsage) { + return typeof assistantValue === "number" && Number.isFinite(assistantValue) && assistantValue >= 0 + ? assistantValue + : 0; + } + for (const value of [assistantValue, userValue, liveValue]) { + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + return value; + } + } + return 0; +} + function isSuccessfulContextMessage(message: ChatAreaMessage): boolean { const status = message.status?.trim().toLowerCase() || "success"; return ( diff --git a/frontend/package.json b/frontend/package.json index 8a6c81cd..ba98f8a6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,8 +19,7 @@ "check": "pnpm lint && pnpm typecheck", "lint": "biome lint .", "lint:fix": "biome lint --write .", - "clean": "rm -rf .next out", - "test": "node --no-warnings --test shared/components/markdown/markdown-table-analyzer.test.mjs" + "clean": "rm -rf .next out" }, "dependencies": { "@base-ui/react": "^1.4.1", diff --git a/frontend/shared/components/markdown/markdown-table-analyzer.test.mjs b/frontend/shared/components/markdown/markdown-table-analyzer.test.mjs deleted file mode 100644 index eed6ff0e..00000000 --- a/frontend/shared/components/markdown/markdown-table-analyzer.test.mjs +++ /dev/null @@ -1,174 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; - -import { - classifyColumn, - classifyTableColumns, - createColumnAnalyzerConfig, - DEFAULT_COLUMN_ANALYZER_PATTERNS, - getVisualLength, - mergeColumnType, -} from "./markdown-table-analyzer.ts"; - -test("classifies a mixed table from values rather than column position", () => { - const headers = ["编号", "Name", "説明", "Date"]; - const rows = [ - ["1", "Alpha", "这是一个较长的中文说明，用于解释问题发生的原因以及推荐的处理方式。", "2026-03-20"], - ["2", "ベータ", "This description contains enough natural-language detail to remain readable on a phone.", "2026-03-21"], - ["3", "Gamma", "複数の言語を含む長い説明で、列が狭くなりすぎないことを確認します。", "2026-03-22"], - ]; - - assert.deepEqual(classifyTableColumns(headers, rows), ["numeric", "normal", "content", "date"]); -}); - -test("counts CJK and other wide characters as two visual units", () => { - assert.equal(getVisualLength("ab中文"), 6); - assert.equal(getVisualLength("テスト"), 6); -}); - -test("classifies multiple long natural-language columns as content", () => { - const types = classifyTableColumns( - ["原因", "Recommendation", "詳細"], - [ - [ - "因为移动端将所有列压缩到屏幕内，长文本被拆成了大量短行。", - "Allow the table to grow naturally and keep horizontal scrolling inside its container.", - "ユーザーが内容を読みやすいように、長文列には十分な最小幅を確保します。", - ], - [ - "第二个原因字段继续包含完整的自然语言句子，以提供稳定的统计样本。", - "Use column-level metadata instead of fixed child indexes for unknown generated schemas.", - "ストリーミング中は狭い列から広い列へのアップグレードだけを許可します。", - ], - ], - ); - - assert.deepEqual(types, ["content", "content", "content"]); -}); - -test("recognizes all-numeric values including money and percentages", () => { - assert.deepEqual( - classifyTableColumns( - ["Count", "金额", "Ratio"], - [ - ["1,200", "￥99.50", "12%"], - ["3,400", "￥125.00", "8.5%"], - ["5,600", "￥1,020.25", "100%"], - ], - ), - ["numeric", "numeric", "numeric"], - ); -}); - -test("keeps long structured-column fallback text wrap-safe", () => { - assert.equal( - classifyColumn("Amount", [ - "1", - "2", - "3", - "This value could not be calculated because the upstream response omitted pricing metadata.", - ]), - "normal", - ); - assert.equal( - classifyColumn("Date", [ - "2026-03-20", - "2026-03-21", - "2026-03-22", - "The date is unavailable because this record predates the migration.", - ]), - "normal", - ); - assert.equal(classifyColumn("Amount", ["1", "2", "3", "N/A"]), "numeric"); -}); - -test("classifies URLs and long unbroken identifiers as code", () => { - assert.deepEqual( - classifyTableColumns( - ["未知字段", "Token"], - [ - ["https://example.com/a/very/long/path?query=mobile-table", "customer_session_identifier_01HZX4RTN8Q3YJ7K9MP2W6CVBX"], - ["/srv/app/releases/2026-03-20/config.json", "customer_session_identifier_01HZX4RTN8Q3YJ7K9MP2W6CVBY"], - ], - ), - ["code", "code"], - ); -}); - -test("supports multilingual and unknown headers without relying on hints", () => { - assert.deepEqual( - classifyTableColumns( - ["名称", "Category", "更新日時", "Champ inconnu"], - [ - ["Alpha", "Tool", "2026-03-20 09:30", "Medium value"], - ["ベータ", "Library", "2026-03-21 10:45", "Another value"], - ["Gamma", "Service", "2026-03-22 11:15", "Third value"], - ], - { headerRules: false }, - ), - ["normal", "normal", "date", "normal"], - ); -}); - -test("ignores empty and missing cells while preserving uneven columns", () => { - assert.deepEqual( - classifyTableColumns( - ["ID", "Name", "Notes"], - [ - ["1", "Alpha Product"], - ["2", "", ""], - ["3", "Gamma Service", "This populated note is long enough to be handled as readable natural-language content."], - [], - ], - ), - ["numeric", "normal", "content"], - ); -}); - -test("header rules remain weak, replaceable hints", () => { - const shortDescription = ["ok", "new", "done", "hold"]; - assert.notEqual(classifyColumn("Description", shortDescription), "content"); - assert.equal( - classifyColumn("Arbitrary", ["x", "y", "z"], { - headerRules: [{ pattern: /^Arbitrary$/, type: "date", weight: 0.2 }], - }), - "compact", - ); -}); -test("normalizes complete pattern overrides with partial thresholds", () => { - const patterns = Object.fromEntries( - Object.entries(DEFAULT_COLUMN_ANALYZER_PATTERNS).map(([name, pattern]) => [name, new RegExp(pattern.source, pattern.flags)]), - ); - assert.equal( - classifyColumn("Value", ["1", "2", "3"], { - patterns, - thresholds: { numericRatio: 0.5 }, - }), - "numeric", - ); - assert.equal(classifyColumn("Value", ["1", "2", "3"], createColumnAnalyzerConfig()), "numeric"); -}); - -test("treats whitespace-free CJK prose as natural language rather than code", () => { - assert.equal( - classifyColumn("未知", [ - "这是没有空格但依然属于自然语言的中文长文本内容", - "これは空白がなくても自然言語として扱うべき日本語の長文です", - ]), - "content", - ); -}); - -test("upgrades columns without shrinking a previous streaming type", () => { - const initial = classifyColumn("Value", ["Alpha Product", "Beta Service", "Gamma Library"]); - const expanded = classifyColumn("Value", [ - "Alpha now includes a complete explanation with substantially more natural-language detail.", - "Beta now includes a second detailed sentence because the streamed answer continued growing.", - "Gamma now also contains enough descriptive prose to need a readable content-column width.", - ]); - - assert.equal(initial, "normal"); - assert.equal(expanded, "content"); - assert.equal(mergeColumnType(initial, expanded), "content"); - assert.equal(mergeColumnType("content", "normal"), "content"); -});