fix: read malformed table spans the way HTML does

snapshots/thinking-machines/model-deprecations.md

@@ -0,0 +1,39 @@
+# Model Deprecations
+
+## How does Tinker handle model deprecation?
+
+As new open-weight models are released, we aim to keep the Tinker model lineup current with the best options for different use cases. Regularly updating the model list allows us to keep throughput high and latency low on the models we offer.
+
+When a new model version supersedes an older one in the same family with comparable or better performance, we may add the new version and begin retiring the older one. We also periodically review our model lineup based on usage and overall fit. If a model sees very low usage or no longer makes sense to offer, we may retire it. We'll aim to give advance notice via email and Tinker documentation before removing a model, along with a recommended replacement.
+
+We suggest the following practices:
+
+- Switch to the recommended replacements listed below for any deprecated models.
+- Avoid building hard dependencies on any specific model, as models may be updated, replaced, or removed over time.
+- Test replacement models well ahead of the retirement date and migrate your code once you've validated they work.
+
+## Recommended replacements
+
+| Retirement date   | Deprecated model              | Recommended replacement                                                         |
+| ----------------- | ----------------------------- | ------------------------------------------------------------------------------- |
+| September 2, 2026 | Qwen3.6-27B                   | Qwen3.8-27B                                                                     |
+| July 12, 2026     | Kimi-K2.5                     | Kimi-K2.6                                                                       |
+| June 12, 2026     | Kimi-K2-Thinking              | Kimi-K2.6                                                                       |
+| June 12, 2026     | Qwen3.5-35B-A3B               | Qwen3.6-35B-A3B                                                                 |
+| June 12, 2026     | Qwen3.5-27B                   | Qwen3.6-27B                                                                     |
+| June 12, 2026     | Qwen3-30B-A3B                 | Qwen3.6-35B-A3B                                                                 |
+| June 12, 2026     | Qwen3-30B-A3B-Instruct-2507   | Qwen3.6-35B-A3B (non-thinking mode)                                             |
+| June 12, 2026     | Qwen3-VL-30B-A3B-Instruct     | Qwen3.6-35B-A3B (includes vision)                                               |
+| June 12, 2026     | Qwen3-32B                     | Qwen3.6-27B                                                                     |
+| June 12, 2026     | Qwen3-235B-A22B-Instruct-2507 | Qwen3.5-397B-A17B (non-thinking mode)                                           |
+| June 12, 2026     | Qwen3-VL-235B-A22B-Instruct   | Qwen3.5-397B-A17B (includes vision)                                             |
+| June 12, 2026     | Qwen3-4B-Instruct-2507        | Qwen3.5-4B (non-thinking mode)                                                  |
+| June 12, 2026     | Llama-3.3-70B-Instruct        | Nemotron-3-Super-120B-A12B, Qwen3.6-27B, or Qwen3.6-35B-A3B (non-thinking mode) |
+| June 12, 2026     | Qwen3-30B-A3B-Base            | Qwen3.5-35B-A3B-Base                                                            |
+| June 12, 2026     | Qwen3-8B-Base                 | Qwen3.5-9B-Base                                                                 |
+| June 12, 2026     | Llama-3.1-8B-Instruct         | Qwen3.5-9B (non-thinking mode)                                                  |
+| June 12, 2026     | Llama-3.1-70B (Base)          | Qwen3.5-35B-A3B-Base                                                            |
+| June 12, 2026     | Llama-3.1-8B (Base)           | Qwen3.5-9B-Base                                                                 |
+| June 12, 2026     | Llama-3.2-3B (Base)           | Qwen3.5-9B-Base                                                                 |
+| June 12, 2026     | Llama-3.2-1B (Base)           | Qwen3.5-9B-Base                                                                 |
+| June 12, 2026     | DeepSeek-V3.1-Base            | Qwen3.5-35B-A3B-Base                                                            |

sources.yaml

@@ -177,3 +177,12 @@ sources:
     min_bytes: 4000
     max_bytes: 200000
     required_markers: ["# Model versions and lifecycle", "### Retired models"]
+  - provider: thinking-machines
+    slug: model-deprecations
+    format: html
+    url: https://tinker-docs.thinkingmachines.ai/tinker/model-deprecations/
+    html_selector: article.md-content__inner
+    max_download_bytes: 2000000
+    min_bytes: 2000
+    max_bytes: 200000
+    required_markers: ["# Model Deprecations", "## Recommended replacements"]

src/sources/html.ts

@@ -81,11 +81,22 @@ function splitPairedTableCells(root: HTMLElement): void {
   }
 }
 
+// A table rendered inside another table's cell is part of that cell's content,
+// never the data half of a split header, so it is skipped on both sides of the
+// pairing below.
+function isNestedTable(table: HTMLElement): boolean {
+  return !!(table.parentNode as HTMLElement | null)?.closest("table");
+}
+
+// The two halves of a split header are adjacent in document order but not
+// necessarily siblings: a page may wrap each half in its own scroll container,
+// so the halves share no parent to walk between.
 function mergeSplitHeaderTables(root: HTMLElement): void {
   const tables = root.querySelectorAll("table");
   for (let index = 0; index < tables.length - 1; index += 1) {
     const headerTable = tables[index]!;
     const dataTable = tables[index + 1]!;
+    if (isNestedTable(headerTable) || isNestedTable(dataTable)) continue;
     const headerRows = directRows(headerTable);
     const dataRows = directRows(dataTable);
     if (headerRows.length !== 1 || dataRows.length === 0) continue;
@@ -116,18 +127,34 @@ function flattenedCell(cell: HTMLElement, content?: string): string {
   return clone.outerHTML;
 }
 
+// HTML reads a span it cannot parse as 1 rather than as an error, and colspan
+// has no zero. Only rowspan="0" carries meaning: span to the end of the row
+// group. Returning 0 for it defers that to flattenTableSpans, the only place
+// that knows where the group ends.
 function spanValue(cell: HTMLElement, name: "rowspan" | "colspan"): number {
-  const raw = cell.getAttribute(name);
-  if (raw === undefined) return 1;
+  const raw = cell.getAttribute(name)?.trim();
+  if (!raw) return 1;
   const value = Number(raw);
-  if (!Number.isInteger(value) || value < 1) throw new Error(`invalid table ${name}`);
+  if (!Number.isInteger(value) || value < 0) return 1;
+  if (value === 0) return name === "rowspan" ? 0 : 1;
   return value;
 }
 
+// The last row of the row group (thead, tbody, tfoot, or the table itself when
+// the markup has no groups) holding the row at rowIndex.
+function rowGroupEnd(rows: HTMLElement[], rowIndex: number): number {
+  const group = rows[rowIndex]!.parentNode;
+  let end = rowIndex;
+  while (end + 1 < rows.length && rows[end + 1]!.parentNode === group) end += 1;
+  return end;
+}
+
 function flattenTableSpans(table: HTMLElement): void {
+  const rows = directRows(table);
   const pending = new Map<number, { html: string; remaining: number }>();
 
-  for (const row of directRows(table)) {
+  for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
+    const row = rows[rowIndex]!;
     const flattened: string[] = [];
     let column = 0;
 
@@ -139,9 +166,14 @@ function flattenTableSpans(table: HTMLElement): void {
       column += 1;
     };
 
+    // A rowspan never reaches past its own row group, so an oversized one is
+    // clamped rather than failing the whole page.
+    const availableRows = rowGroupEnd(rows, rowIndex) - rowIndex + 1;
+
     for (const cell of directCells(row)) {
       while (pending.has(column)) appendPending();
-      const rowSpan = spanValue(cell, "rowspan");
+      const declared = spanValue(cell, "rowspan");
+      const rowSpan = declared === 0 ? availableRows : Math.min(declared, availableRows);
       const columnSpan = spanValue(cell, "colspan");
       for (let span = 0; span < columnSpan; span += 1) {
         while (pending.has(column)) appendPending();

tests/sources-html.test.ts

@@ -0,0 +1,248 @@
+import { describe, expect, it } from "vitest";
+import { convertHtml, HtmlConversionError } from "../src/sources/html.js";
+import type { SourceEntry } from "../src/sources/registry.js";
+
+const source: SourceEntry = {
+  provider: "bedrock",
+  slug: "model-lifecycle",
+  catalog_provider: "bedrock",
+  format: "html",
+  url: "https://example.com/model-lifecycle",
+  html_selector: "article",
+  max_download_bytes: 10_000,
+  min_bytes: 1,
+  max_bytes: 10_000,
+  required_markers: ["Model"],
+};
+
+function rows(markdown: string): string[][] {
+  return markdown
+    .split("\n")
+    .filter((line) => line.startsWith("|") && !/^\|(?:\s*-+\s*\|)+$/.test(line))
+    .map((line) =>
+      line
+        .slice(1, -1)
+        .split("|")
+        .map((cell) => cell.trim()),
+    );
+}
+
+describe("HTML source conversion", () => {
+  it("expands rowspans and repairs a missing provider cell", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Model provider</th><th>Model name</th><th>Model ID</th><th>Regions</th><th>Legacy date</th><th>EOL date</th><th>Public extended access start date</th></tr>
+        <tr><td rowspan="2">Anthropic</td><td rowspan="2">Claude 3 Haiku</td><td rowspan="2">anthropic.claude-3-haiku-v1:0</td><td>us-east-1</td><td>March 10, 2026</td><td>September 10, 2026</td><td>June 10, 2026</td></tr>
+        <tr><td>us-gov-east-1, us-gov-west-1</td><td>March 10, 2026</td><td>September 10, 2026</td><td>June 10, 2026</td></tr>
+        <tr><td>Command R</td><td>cohere.command-r-v1:0</td><td>us-east-1</td><td>February 19, 2026</td><td>August 19, 2026</td><td>May 19, 2026</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toContainEqual([
+      "Anthropic",
+      "Claude 3 Haiku",
+      "anthropic.claude-3-haiku-v1:0",
+      "us-gov-east-1, us-gov-west-1",
+      "March 10, 2026",
+      "September 10, 2026",
+      "June 10, 2026",
+    ]);
+    expect(rows(markdown)).toContainEqual([
+      "Cohere",
+      "Command R",
+      "cohere.command-r-v1:0",
+      "us-east-1",
+      "February 19, 2026",
+      "August 19, 2026",
+      "May 19, 2026",
+    ]);
+  });
+
+  it("combines split tables and separates paired date cells", () => {
+    const markdown = convertHtml(
+      source,
+      `<article>
+        <div data-slot="table-container"><table><tr><th>Model</th><th>Version</th><th>API</th><th><div class="flex justify-between">Deprecation<svg></svg>Retirement</div></th><th>Alternative</th></tr></table></div>
+        <div><span>Scroll for more</span><svg></svg></div>
+        <div data-slot="table-container"><table>
+          <tr><td>Leanstral</td><td>26.03</td><td>labs-leanstral</td><td><div class="flex justify-between"><span>5/22/2026</span><span>6/30/2026</span></div></td><td>Leanstral 1.5</td></tr>
+          <tr><td>Mathstral 7B</td><td>0.1</td><td></td><td><div class="flex justify-between"><span></span><span></span></div></td><td>Mistral Small 4</td></tr>
+        </table></div>
+      </article>`,
+    );
+
+    expect(rows(markdown)[0]).toEqual([
+      "Model",
+      "Version",
+      "API",
+      "Deprecation",
+      "Retirement",
+      "Alternative",
+    ]);
+    expect(rows(markdown)).toContainEqual([
+      "Leanstral",
+      "26.03",
+      "labs-leanstral",
+      "5/22/2026",
+      "6/30/2026",
+      "Leanstral 1.5",
+    ]);
+    expect(rows(markdown)).toContainEqual(["Mathstral 7B", "0.1", "", "", "", "Mistral Small 4"]);
+    expect(markdown).not.toContain("Scroll for more");
+    expect(markdown.match(/^\|(?:\s*-+\s*\|)+$/gm)).toHaveLength(1);
+  });
+
+  it("duplicates merged deprecation data into each affected row", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Category</th><th>Model name</th><th>Deprecation time</th><th>Replacement model</th></tr>
+        <tr><td rowspan="2">Qwen-Max</td><td>qwen3.6-max-preview</td><td rowspan="3">October 10, 2026</td><td rowspan="2">qwen3.7-max</td></tr>
+        <tr><td>qwen3-max</td></tr>
+        <tr><td>Qwen-VL</td><td>qwen3-vl-flash</td><td>qwen3.6-flash</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toContainEqual([
+      "Qwen-Max",
+      "qwen3-max",
+      "October 10, 2026",
+      "qwen3.7-max",
+    ]);
+    expect(rows(markdown)).toContainEqual([
+      "Qwen-VL",
+      "qwen3-vl-flash",
+      "October 10, 2026",
+      "qwen3.6-flash",
+    ]);
+  });
+
+  it("expands rowspan=0 to the end of the table", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Provider</th><th>Model</th></tr>
+        <tr><td rowspan="0">Anthropic</td><td>Claude</td></tr>
+        <tr><td>Haiku</td></tr>
+        <tr><td>Opus</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toContainEqual(["Anthropic", "Claude"]);
+    expect(rows(markdown)).toContainEqual(["Anthropic", "Haiku"]);
+    expect(rows(markdown)).toContainEqual(["Anthropic", "Opus"]);
+  });
+
+  it("keeps rowspan=0 inside its own row group", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <thead><tr><th>Provider</th><th>Model</th></tr></thead>
+        <tbody><tr><td rowspan="0">Anthropic</td><td>Claude</td></tr><tr><td>Haiku</td></tr></tbody>
+        <tbody><tr><td>Cohere</td><td>Command R</td></tr></tbody>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toEqual([
+      ["Provider", "Model"],
+      ["Anthropic", "Claude"],
+      ["Anthropic", "Haiku"],
+      ["Cohere", "Command R"],
+    ]);
+  });
+
+  it("clamps a rowspan that reaches past its row group", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Provider</th><th>Model</th></tr>
+        <tr><td rowspan="9">Anthropic</td><td>Claude</td></tr>
+        <tr><td>Haiku</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toEqual([
+      ["Provider", "Model"],
+      ["Anthropic", "Claude"],
+      ["Anthropic", "Haiku"],
+    ]);
+  });
+
+  it("reads an unparseable span as 1 instead of dropping the source", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Provider</th><th>Model</th></tr>
+        <tr><td rowspan="">Anthropic</td><td colspan="">Claude</td></tr>
+        <tr><td>Cohere</td><td>Command R</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toEqual([
+      ["Provider", "Model"],
+      ["Anthropic", "Claude"],
+      ["Cohere", "Command R"],
+    ]);
+  });
+
+  it("does not merge a header table that belongs to another table's cell", () => {
+    const markdown = convertHtml(
+      source,
+      `<article>
+        <table><tr><td><table><tr><th>Region</th><th>Zone</th></tr></table></td></tr></table>
+        <table><tr><td>Claude 3 Haiku</td><td>March 10, 2026</td></tr></table>
+      </article>`,
+    );
+
+    // The nested header stays inside the cell it belongs to instead of being
+    // lifted onto the table that happens to follow it.
+    expect(rows(markdown)).toContainEqual(["Claude 3 Haiku", "March 10, 2026"]);
+    expect(markdown).not.toMatch(/\| Region \| Zone \|/);
+  });
+
+  it("merges split header halves that sit in separate scroll containers", () => {
+    const markdown = convertHtml(
+      source,
+      `<article>
+        <div><div data-slot="table-container"><table><tr><th>Model</th><th>Retirement</th></tr></table></div></div>
+        <div><div data-slot="table-container"><table>
+          <tr><td>Leanstral</td><td>6/30/2026</td></tr>
+          <tr><td>Mathstral 7B</td><td>7/31/2026</td></tr>
+        </table></div></div>
+      </article>`,
+    );
+
+    expect(rows(markdown)).toEqual([
+      ["Model", "Retirement"],
+      ["Leanstral", "6/30/2026"],
+      ["Mathstral 7B", "7/31/2026"],
+    ]);
+    expect(markdown.match(/^\|(?:\s*-+\s*\|)+$/gm)).toHaveLength(1);
+  });
+  it("leaves a rowspanned provider column to the span expander", () => {
+    const markdown = convertHtml(
+      source,
+      `<article><table>
+        <tr><th>Model provider</th><th>Model name</th><th>Model ID</th></tr>
+        <tr><td rowspan="2">Amazon Web Services</td><td>Claude v2</td><td>anthropic.claude-v2</td></tr>
+        <tr><td>Claude v3</td><td>anthropic.claude-v3</td></tr>
+      </table></article>`,
+    );
+
+    expect(rows(markdown)).toEqual([
+      ["Model provider", "Model name", "Model ID"],
+      ["Amazon Web Services", "Claude v2", "anthropic.claude-v2"],
+      ["Amazon Web Services", "Claude v3", "anthropic.claude-v3"],
+    ]);
+  });
+
+  it("reports a conversion failure with the underlying cause", () => {
+    expect(() => convertHtml({ ...source, html_selector: "[" }, "<article></article>")).toThrow(
+      expect.objectContaining<Partial<HtmlConversionError>>({
+        code: "conversion_failed",
+        message: expect.stringMatching(/^HTML conversion failed: .+/),
+      }),
+    );
+  });
+});