Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions .github/workflows/python-tools.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Python tools

on:
push:
branches:
- main
paths:
- tools/python/**
- .github/workflows/python-tools.yml
pull_request:
branches:
- main
- docs
paths:
- tools/python/**
- .github/workflows/python-tools.yml
workflow_dispatch:
jobs:
test:
if: github.repository == 'software-mansion/runntime'
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: python-tools-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
working-directory: tools/python
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
# --locked fails on a stale uv.lock, like pnpm's --frozen-lockfile.
- name: Tests
run: uv run --locked pytest
2 changes: 2 additions & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@ dist
node_modules
pnpm-lock.yaml
apps/docs/.astro
tools/python/.venv
tools/python/.pytest_cache
35 changes: 35 additions & 0 deletions packages/runntime/test/core/weights/safetensors.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { readFileSync } from 'node:fs';
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { WeightCache } from '../../../src/core/weights/cache.ts';
import { f16ToF32 } from '../../../src/core/weights/convert.ts';
import {
bufferSource,
chunkedSource,
Expand Down Expand Up @@ -546,6 +548,39 @@ describe('fromSafetensors', () => {
});
});

// Files written by `runntime quantize` (tools/python), whose tests check
// the same bytes, so the writer and this reader cannot drift apart.
describe('with files from the Python quantizer', () => {
const golden = (file: string) =>
fromSafetensors(
bufferSource(
readFileSync(
new URL(`../../../../../tools/python/tests/golden/${file}`, import.meta.url),
),
),
);

it.for([4, 8])('reads int%i weights within half a step of the float source', async (bits) => {
const source = await golden('tiny.safetensors');
const sd = await golden(`tiny.int${bits}.safetensors`);
const w = sd.tensors.get('layer.weight')!;
if (w.kind !== 'quant') throw new Error('expected a quantized tensor');
expect(w).toMatchObject({ shape: [8, 64], bits, groupSize: 32 });
const ints = await w.ints();
const halves = await w.scales();
const scales = halves instanceof Float32Array ? halves : f16ToF32(halves);
const want = await source.tensors.get('layer.weight')!.f32();
want.forEach((v, i) => {
const scale = scales[Math.floor(i / 32)]!;
expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6);
});
// The tensors it leaves alone keep their bytes.
for (const name of ['embed.weight', 'layer.bias', 'norm.weight']) {
expect(await sd.tensors.get(name)!.f32()).toEqual(await source.tensors.get(name)!.f32());
}
});
});

describe('with a cache', () => {
it('loads a second time without touching the source', async () => {
const { cache } = memoryCache();
Expand Down
4 changes: 4 additions & 0 deletions tools/python/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
.venv/
__pycache__/
.pytest_cache/
.ruff_cache/
1 change: 1 addition & 0 deletions tools/python/.python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.11
36 changes: 36 additions & 0 deletions tools/python/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# runntime-tools

Quantizes a safetensors checkpoint to runntime's int8/int4 weight format, so
a model downloads smaller and runs on the quantized matmul.

```bash
uv run runntime quantize model.safetensors model.int8.safetensors --bits 8
```

- The source is a `.safetensors` file or a Hugging Face model folder, sharded
or not. The output is one `.safetensors` file.
- 2D tensors named `*.weight` are quantized in groups of `--group-size`
inputs (default 32), each group with one f16 scale. Embeddings, position
embeddings and `lm_head` stay as stored. `--include` and `--exclude` take
glob patterns and replace those defaults.
- Every other tensor is copied as stored, or as F16 with `--f16`, and the
header metadata is kept.
- `--drop` takes glob patterns of tensors to leave out, like a pooler the
model never reads.
- It prints the max and mean absolute error of every quantized tensor and
why each other tensor was skipped. Exclude the layers that lose too much.

## Development

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't we want to test this in CI too, on every change to tools/python?


```bash
uv sync
uv run pytest
uv run ruff check && uv run ruff format --check
uv run pyright
```

`tests/golden` holds a small float checkpoint and its int4 and int8 versions.
The TypeScript tests in `packages/runntime` read the same files. After a
change to the output, regenerate them with
`uv run runntime quantize tests/golden/tiny.safetensors tests/golden/tiny.int8.safetensors --bits 8 --group-size 32`
(and the same with `--bits 4`).
37 changes: 37 additions & 0 deletions tools/python/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
[project]
name = "runntime-tools"
version = "0.1.0"
description = "Quantizes safetensors checkpoints to runntime's int8/int4 weight format."
readme = "README.md"
license = "MIT"
requires-python = ">=3.11"
dependencies = ["numpy>=2.0"]

[project.scripts]
runntime = "runntime_tools.cli:main"

[build-system]
requires = ["uv_build>=0.9.24,<0.10.0"]
build-backend = "uv_build"

[dependency-groups]
dev = [
"pyright>=1.1.414",
"pytest>=9.2.0",
"ruff>=0.17.3",
"safetensors>=0.8.0",
]

[tool.ruff]
line-length = 120

[tool.ruff.lint]
select = ["E", "W", "F", "I", "N", "UP", "B", "A", "C4", "SIM", "PTH", "RET", "PL", "PERF", "RUF"]
ignore = ["PLR2004", "PLR0911"]

[tool.pyright]
typeCheckingMode = "standard"
strict = ["src"]

[tool.pytest.ini_options]
testpaths = ["tests"]
1 change: 1 addition & 0 deletions tools/python/src/runntime_tools/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Tools that prepare checkpoints for runntime."""
182 changes: 182 additions & 0 deletions tools/python/src/runntime_tools/cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
"""`runntime quantize`: writes a safetensors checkpoint in runntime's int8/int4
format v1 and prints the error of every quantized tensor."""

import argparse
import json
import sys
from collections.abc import Sequence
from dataclasses import dataclass
from fnmatch import fnmatchcase
from pathlib import Path

import numpy as np

from runntime_tools.quantize import Bits, dequantize, pack_int4, quantize
from runntime_tools.safetensors_io import Tensor, from_array, read_checkpoint, to_float32, write_safetensors

QUANT_METADATA_KEY = "runntime.quant"
"""Header metadata key that marks a quantized file and holds its setting."""

FORMAT_VERSION = 1

FLOAT_DTYPES = frozenset({"F32", "F16", "BF16"})

DEFAULT_INCLUDE = ("*.weight",)

DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*")
"""runntime reads token and position embeddings as float lookup tables, and
lm_head often shares the embedding's weights."""

IN_FEATURES_MULTIPLE = 8
"""runntime's quantized matmul reads 8 input rows per tile."""

F16_MAX = float(np.finfo(np.float16).max)
"""The largest finite f16. A tensor with a bigger value cannot be stored as F16."""

PACKED_WORD_BITS = 32
"""runntime packs a weight 32 bits to a word, so out features must fill whole
words: a multiple of 4 for int8, 8 for int4."""


@dataclass(frozen=True)
class Options:
bits: Bits
group_size: int
include: Sequence[str]
exclude: Sequence[str]
drop: Sequence[str] = ()
float16: bool = False
"""Store the tensors that stay float as F16."""


def skip_reason(name: str, tensor: Tensor, options: Options) -> str | None:
"""Why `tensor` stays as stored, or None when it gets quantized."""
if not any(fnmatchcase(name, pattern) for pattern in options.include):
return "not matched by --include"
excluded_by = next((pattern for pattern in options.exclude if fnmatchcase(name, pattern)), None)
if excluded_by:
return f"excluded by {excluded_by}"
if tensor.dtype not in FLOAT_DTYPES:
return f"dtype {tensor.dtype}"
if len(tensor.shape) != 2:
return f"{len(tensor.shape)}D, needs [out, in]"
out_features, in_features = tensor.shape
if in_features % IN_FEATURES_MULTIPLE:
return f"in features {in_features} not a multiple of {IN_FEATURES_MULTIPLE}"
if in_features % options.group_size:
return f"in features {in_features} not a multiple of the group size {options.group_size}"
out_multiple = PACKED_WORD_BITS // options.bits
if out_features % out_multiple:
return f"out features {out_features} not a multiple of {out_multiple}"
return None


def to_float16(name: str, tensor: Tensor) -> Tensor:
"""`tensor` as F16, or unchanged when it is not F32 or BF16. Raises
ValueError when a value is beyond the f16 range."""
if tensor.dtype not in {"F32", "BF16"}:
return tensor
values = to_float32(tensor)
if np.abs(values[np.isfinite(values)]).max(initial=0) > F16_MAX:
raise ValueError(f"{name} has values beyond the f16 range, run without --f16")
return from_array("F16", values.astype(np.float16))


def quantize_checkpoint(source: Path, destination: Path, options: Options) -> list[str]:
"""Quantizes the selected tensors of `source` into `destination` and
returns one report line per tensor, in file order."""
checkpoint = read_checkpoint(source)
if QUANT_METADATA_KEY in checkpoint.metadata:
raise ValueError(f"{source} is already quantized, run on the float checkpoint")
tensors: dict[str, Tensor] = {}
report: list[str] = []
quantized_count = 0
for name, tensor in checkpoint.tensors.items():
shape = f"[{', '.join(map(str, tensor.shape))}]"
dropped_by = next((pattern for pattern in options.drop if fnmatchcase(name, pattern)), None)
if dropped_by:
report.append(f"dropped {name} {shape}: matched by {dropped_by}")
continue
reason = skip_reason(name, tensor, options)
if reason:
tensors[name] = to_float16(name, tensor) if options.float16 else tensor

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Running the tool on an already quantized file gives a file runntime can't load: --bits 4 on an int8 file exits 0, but the old int8 weights stay and the header says int4. I guess refuse a source that's already quantized?

report.append(f"skipped {name} {shape}: {reason}")
continue
weight = to_float32(tensor)
quantized = quantize(weight, options.bits, options.group_size)
if options.bits == 8:
tensors[name] = from_array("I8", quantized.ints)
else:
tensors[name] = from_array("U8", pack_int4(quantized.ints))
tensors[f"{name}_scale"] = from_array("F16", quantized.scales)
quantized_count += 1
error = np.abs(dequantize(quantized) - weight)
report.append(f"quantized {name} {shape}: max error {error.max():.3g}, mean {error.mean():.3g}")
if quantized_count == 0:
raise ValueError("no tensor was quantized:\n" + "\n".join(report))
setting = {"version": FORMAT_VERSION, "bits": options.bits, "groupSize": options.group_size, "scheme": "symmetric"}
metadata = checkpoint.metadata | {QUANT_METADATA_KEY: json.dumps(setting, separators=(",", ":"))}
write_safetensors(destination, tensors, metadata)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the output path is the same as the input, the input gets wiped. Runntime quantize m.safetensors m.safetensors --bits 8 crashes and leaves an empty file. Maybe write to a temp file first and rename it at the end?

return report


def group_size(value: str) -> int:
"""argparse type: a positive multiple of 4."""
size = int(value)
if size <= 0 or size % 4:
raise argparse.ArgumentTypeError(f"{value} is not a positive multiple of 4")
return size


def parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
parser = argparse.ArgumentParser(prog="runntime")
commands = parser.add_subparsers(dest="command", required=True)
command = commands.add_parser(
"quantize",
help="quantize a safetensors checkpoint to int8 or int4",
description="Quantizes the 2D weights of a safetensors checkpoint to runntime's int8/int4 format.",
)
command.add_argument("source", type=Path, help="a .safetensors file or a Hugging Face model folder")
command.add_argument("destination", type=Path, help="the .safetensors file to write")
command.add_argument("--bits", type=int, choices=(4, 8), required=True)
command.add_argument("--group-size", type=group_size, default=32, help="inputs that share one scale (default: 32)")
command.add_argument(
"--include",
action="append",
metavar="GLOB",
help=f"tensor names to quantize, repeatable, replaces the default {' '.join(DEFAULT_INCLUDE)}",
)
command.add_argument(
"--exclude",
action="append",
metavar="GLOB",
help=f"tensor names to keep as stored, repeatable, replaces the default {' '.join(DEFAULT_EXCLUDE)}",
)
command.add_argument(
"--drop",
action="append",
default=[],
metavar="GLOB",
help="tensor names to leave out of the output, repeatable",
)
command.add_argument("--f16", action="store_true", help="store the tensors that stay float as F16")
return parser.parse_args(argv)


def main(argv: Sequence[str] | None = None) -> int:
args = parse_args(argv)
options = Options(
bits=args.bits,
group_size=args.group_size,
include=args.include or DEFAULT_INCLUDE,
exclude=args.exclude or DEFAULT_EXCLUDE,
drop=args.drop,
float16=args.f16,
)
try:
report = quantize_checkpoint(args.source, args.destination, options)
except ValueError as error:
print(f"runntime quantize: {error}", file=sys.stderr)
return 1
print("\n".join(report))
return 0
Loading
Loading