Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions apps/docs/astro.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,9 @@ export default defineConfig({
},
],
{
// The splash landing page belongs to no topic.
exclude: ['/'],
// The splash landing page belongs to no topic, and the core pages stay
// unlisted until runntime core is published.
exclude: ['/', '/core/**'],
// The generated index pages sit outside the sidebar groups.
topics: { zoo: ['/api/**'] },
},
Expand Down
89 changes: 89 additions & 0 deletions apps/docs/src/components/QuantFile.astro
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
---
/** What a quantized safetensors file holds and how the loader reads each
* entry: the header label, an int8 weight with its F16 scales, and the float
* tensors around them. Names are all-MiniLM-L6-v2's, shortened; the floats
* are F16 as `--f16` stores them. */
const rows = [
{ name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' },
{ name: '…query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight', quant: true },
{ name: '…query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales', quant: true },
{ name: '…query.bias', dtype: 'F16', shape: '[384]', loads: 'float' },
{ name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' },
];
const rowHeight = 30;
const top = 84;
const height = top + rows.length * rowHeight + 14;
---

<svg
class="quant-file"
viewBox={`0 0 400 ${height}`}
role="img"
aria-label="A quantized file: the runntime.quant header label, then tensors. F16 tensors load as float, an I8 weight loads as an int8 weight and takes its F16 weight_scale with it."
>
<rect class="file" x="0.5" y="0.5" width="399" height={height - 1} rx="6"></rect>
<text class="header" x="12" y="22">__metadata__ runntime.quant =</text>
<text class="header strong" x="12" y="42">{'{'}"version":1,"bits":8,</text>
<text class="header strong" x="12" y="60">"groupSize":32,"scheme":"symmetric"{'}'}</text>
<line class="rule" x1="0" y1="72" x2="400" y2="72"></line>
{
rows.map((row, i) => {
const y = top + i * rowHeight + 12;
return (
<g class={row.quant ? 'row quant' : 'row'}>
{row.quant && (
<rect class="band" x="6" y={y - 20} width="388" height={rowHeight} rx="3" />
)}
<text class="name" x="12" y={y}>
{row.name}
</text>
<text class="dtype" x="212" y={y}>
{row.dtype} {row.shape}
</text>
<text class="loads" x="392" y={y} text-anchor="end">
{row.loads}
</text>
</g>
);
})
}
</svg>

<style>
.quant-file {
display: block;
height: auto;
margin: 1.5rem auto;
max-width: 30rem;
width: 100%;
}

.file {
fill: var(--surface-raised);
stroke: var(--border-strong);
}

.rule {
stroke: var(--border-strong);
}

.band {
fill: var(--surface-quiet);
}

text {
fill: var(--text-muted);
font-family: var(--__sl-font-mono, monospace);
font-size: 13px;
}

.strong,
.quant .name,
.quant .loads {
fill: var(--text-primary);
}

.loads {
font-family: var(--__sl-font, sans-serif);
}
</style>
160 changes: 160 additions & 0 deletions apps/docs/src/components/WeightsPath.astro
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
---
/** The path of a weights file, from a checkpoint to the GPU, with quantizing
* as an optional branch. Laid out in a row, and in a column at phone width. */
const BOX_HEIGHT = 56;

const checkpoint = { title: 'Checkpoint', detail: 'PyTorch / HF' };
const exported = { title: 'Export', detail: '.safetensors' };
const quantize = { title: 'Quantize', detail: 'int8 / int4', optional: true };
const modelPath = { title: 'modelPath', detail: 'create<Task>()' };
const load = { title: 'Load', detail: 'GPU upload' };

/** The quantize branch leaves Export from the top and joins modelPath from
* the top; the float path runs straight between them. The column puts the
* float path on the left and the branch on the right. */
const layouts = [
{
name: 'row',
width: 770,
height: 120,
boxes: [
{ ...checkpoint, x: 0, y: 64, width: 130 },
{ ...exported, x: 160, y: 64, width: 130 },
{ ...quantize, x: 320, y: 0, width: 130 },
{ ...modelPath, x: 480, y: 64, width: 130 },
{ ...load, x: 640, y: 64, width: 130 },
],
arrows: ['M130 92 H160', 'M225 64 V28 H320', 'M450 28 H545 V64', 'M290 92 H480', 'M610 92 H640'],
skipLabel: { x: 385, y: 84, anchor: 'middle', lines: ['float file as is'] },
},
{
name: 'column',
width: 340,
height: 440,
boxes: [
{ ...checkpoint, x: 100, y: 0, width: 240 },
{ ...exported, x: 100, y: 96, width: 240 },
{ ...quantize, x: 190, y: 192, width: 150 },
{ ...modelPath, x: 100, y: 288, width: 240 },
{ ...load, x: 100, y: 384, width: 240 },
],
arrows: ['M220 56 V96', 'M265 152 V192', 'M265 248 V288', 'M130 152 V288', 'M220 344 V384'],
skipLabel: { x: 122, y: 216, anchor: 'end', lines: ['float file', 'as is'] },
},
];
---

{
layouts.map((layout) => (
<svg
class={`weights-path ${layout.name}`}
viewBox={`-2 -2 ${layout.width + 4} ${layout.height + 4}`}
role="img"
aria-label="A checkpoint is exported to a safetensors file. The file is either quantized to int8 or int4 first, or used as it is, and then passed as modelPath and loaded to the GPU."
>
<defs>
<marker
id={`weights-path-arrow-${layout.name}`}
viewBox="0 0 10 10"
refX="10"
refY="5"
markerWidth="7"
markerHeight="7"
orient="auto"
>
<path class="arrow-head" d="M0 0 L10 5 L0 10 z" />
</marker>
</defs>
{layout.arrows.map((d) => (
<path class="arrow" d={d} marker-end={`url(#weights-path-arrow-${layout.name})`} />
))}
<text
class="skip"
x={layout.skipLabel.x}
y={layout.skipLabel.y}
text-anchor={layout.skipLabel.anchor}
>
{layout.skipLabel.lines.map((line, i) => (
<tspan x={layout.skipLabel.x} dy={i === 0 ? 0 : 16}>
{line}
</tspan>
))}
</text>
{layout.boxes.map((box) => (
<g class={box.optional ? 'box optional' : 'box'}>
<rect x={box.x} y={box.y} width={box.width} height={BOX_HEIGHT} rx="6" />
<text class="title" x={box.x + 12} y={box.y + 23}>
{box.title}
{box.optional && <tspan class="optional-label"> optional</tspan>}
</text>
<text class="detail" x={box.x + 12} y={box.y + 42}>
{box.detail}
</text>
</g>
))}
</svg>
))
}

<style>
.weights-path {
display: block;
height: auto;
margin: 1.5rem auto;
width: 100%;
}

.column {
display: none;
max-width: 21.25rem;
}

@media (max-width: 40rem) {
.row {
display: none;
}

.column {
display: block;
}
}

rect {
fill: var(--surface-raised);
stroke: var(--border-strong);
}

.optional rect {
stroke-dasharray: 4 3;
}

.arrow {
fill: none;
stroke: var(--text-faint);
stroke-width: 1.5;
}

.arrow-head {
fill: var(--text-faint);
}

.title {
fill: var(--text-primary);
font-family: var(--__sl-font, sans-serif);
font-size: 14px;
font-weight: 600;
}

.optional-label {
fill: var(--text-faint);
font-size: 11px;
font-weight: 400;
}

.detail,
.skip {
fill: var(--text-muted);
font-family: var(--__sl-font-mono, monospace);
font-size: 11px;
}
</style>
Loading
Loading