Repository navigation
feat(tools): add runntime quantize, a uv Python CLI that writes int8/int4 checkpoints in format v1 #93
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(tools): add runntime quantize, a uv Python CLI that writes int8/int4 checkpoints in format v1 #93
Changes from all commits
0336af7
30a0ed1
e0a35be
b2b8c41
86e4b8b
70aec62
44e5019
0aaf88d
55d72e9
f00cee7
697eaec
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| name: Python tools | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| paths: | ||
| - tools/python/** | ||
| - .github/workflows/python-tools.yml | ||
| pull_request: | ||
| branches: | ||
| - main | ||
| - docs | ||
| paths: | ||
| - tools/python/** | ||
| - .github/workflows/python-tools.yml | ||
| workflow_dispatch: | ||
| jobs: | ||
| test: | ||
| if: github.repository == 'software-mansion/runntime' | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: read | ||
| concurrency: | ||
| group: python-tools-${{ github.ref }} | ||
| cancel-in-progress: true | ||
| defaults: | ||
| run: | ||
| working-directory: tools/python | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v7 | ||
| # --locked fails on a stale uv.lock, like pnpm's --frozen-lockfile. | ||
| - name: Tests | ||
| run: uv run --locked pytest |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,3 +2,5 @@ dist | |
| node_modules | ||
| pnpm-lock.yaml | ||
| apps/docs/.astro | ||
| tools/python/.venv | ||
| tools/python/.pytest_cache | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| .venv/ | ||
| __pycache__/ | ||
| .pytest_cache/ | ||
| .ruff_cache/ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 3.11 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| # runntime-tools | ||
|
|
||
| Quantizes a safetensors checkpoint to runntime's int8/int4 weight format, so | ||
| a model downloads smaller and runs on the quantized matmul. | ||
|
|
||
| ```bash | ||
| uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 | ||
| ``` | ||
|
|
||
| - The source is a `.safetensors` file or a Hugging Face model folder, sharded | ||
| or not. The output is one `.safetensors` file. | ||
| - 2D tensors named `*.weight` are quantized in groups of `--group-size` | ||
| inputs (default 32), each group with one f16 scale. Embeddings, position | ||
| embeddings and `lm_head` stay as stored. `--include` and `--exclude` take | ||
| glob patterns and replace those defaults. | ||
| - Every other tensor is copied as stored, or as F16 with `--f16`, and the | ||
| header metadata is kept. | ||
| - `--drop` takes glob patterns of tensors to leave out, like a pooler the | ||
| model never reads. | ||
| - It prints the max and mean absolute error of every quantized tensor and | ||
| why each other tensor was skipped. Exclude the layers that lose too much. | ||
|
|
||
| ## Development | ||
|
|
||
| ```bash | ||
| uv sync | ||
| uv run pytest | ||
| uv run ruff check && uv run ruff format --check | ||
| uv run pyright | ||
| ``` | ||
|
|
||
| `tests/golden` holds a small float checkpoint and its int4 and int8 versions. | ||
| The TypeScript tests in `packages/runntime` read the same files. After a | ||
| change to the output, regenerate them with | ||
| `uv run runntime quantize tests/golden/tiny.safetensors tests/golden/tiny.int8.safetensors --bits 8 --group-size 32` | ||
| (and the same with `--bits 4`). | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| [project] | ||
| name = "runntime-tools" | ||
| version = "0.1.0" | ||
| description = "Quantizes safetensors checkpoints to runntime's int8/int4 weight format." | ||
| readme = "README.md" | ||
| license = "MIT" | ||
| requires-python = ">=3.11" | ||
| dependencies = ["numpy>=2.0"] | ||
|
|
||
| [project.scripts] | ||
| runntime = "runntime_tools.cli:main" | ||
|
|
||
| [build-system] | ||
| requires = ["uv_build>=0.9.24,<0.10.0"] | ||
| build-backend = "uv_build" | ||
|
|
||
| [dependency-groups] | ||
| dev = [ | ||
| "pyright>=1.1.414", | ||
| "pytest>=9.2.0", | ||
| "ruff>=0.17.3", | ||
| "safetensors>=0.8.0", | ||
| ] | ||
|
|
||
| [tool.ruff] | ||
| line-length = 120 | ||
|
|
||
| [tool.ruff.lint] | ||
| select = ["E", "W", "F", "I", "N", "UP", "B", "A", "C4", "SIM", "PTH", "RET", "PL", "PERF", "RUF"] | ||
| ignore = ["PLR2004", "PLR0911"] | ||
|
|
||
| [tool.pyright] | ||
| typeCheckingMode = "standard" | ||
| strict = ["src"] | ||
|
|
||
| [tool.pytest.ini_options] | ||
| testpaths = ["tests"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| """Tools that prepare checkpoints for runntime.""" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| """`runntime quantize`: writes a safetensors checkpoint in runntime's int8/int4 | ||
| format v1 and prints the error of every quantized tensor.""" | ||
|
|
||
| import argparse | ||
| import json | ||
| import sys | ||
| from collections.abc import Sequence | ||
| from dataclasses import dataclass | ||
| from fnmatch import fnmatchcase | ||
| from pathlib import Path | ||
|
|
||
| import numpy as np | ||
|
|
||
| from runntime_tools.quantize import Bits, dequantize, pack_int4, quantize | ||
| from runntime_tools.safetensors_io import Tensor, from_array, read_checkpoint, to_float32, write_safetensors | ||
|
|
||
| QUANT_METADATA_KEY = "runntime.quant" | ||
| """Header metadata key that marks a quantized file and holds its setting.""" | ||
|
|
||
| FORMAT_VERSION = 1 | ||
|
|
||
| FLOAT_DTYPES = frozenset({"F32", "F16", "BF16"}) | ||
|
|
||
| DEFAULT_INCLUDE = ("*.weight",) | ||
|
|
||
| DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*") | ||
| """runntime reads token and position embeddings as float lookup tables, and | ||
| lm_head often shares the embedding's weights.""" | ||
|
|
||
| IN_FEATURES_MULTIPLE = 8 | ||
| """runntime's quantized matmul reads 8 input rows per tile.""" | ||
|
|
||
| F16_MAX = float(np.finfo(np.float16).max) | ||
| """The largest finite f16. A tensor with a bigger value cannot be stored as F16.""" | ||
|
|
||
| PACKED_WORD_BITS = 32 | ||
| """runntime packs a weight 32 bits to a word, so out features must fill whole | ||
| words: a multiple of 4 for int8, 8 for int4.""" | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class Options: | ||
| bits: Bits | ||
| group_size: int | ||
| include: Sequence[str] | ||
| exclude: Sequence[str] | ||
| drop: Sequence[str] = () | ||
| float16: bool = False | ||
| """Store the tensors that stay float as F16.""" | ||
|
|
||
|
|
||
| def skip_reason(name: str, tensor: Tensor, options: Options) -> str | None: | ||
| """Why `tensor` stays as stored, or None when it gets quantized.""" | ||
| if not any(fnmatchcase(name, pattern) for pattern in options.include): | ||
| return "not matched by --include" | ||
| excluded_by = next((pattern for pattern in options.exclude if fnmatchcase(name, pattern)), None) | ||
| if excluded_by: | ||
| return f"excluded by {excluded_by}" | ||
| if tensor.dtype not in FLOAT_DTYPES: | ||
| return f"dtype {tensor.dtype}" | ||
| if len(tensor.shape) != 2: | ||
| return f"{len(tensor.shape)}D, needs [out, in]" | ||
| out_features, in_features = tensor.shape | ||
| if in_features % IN_FEATURES_MULTIPLE: | ||
| return f"in features {in_features} not a multiple of {IN_FEATURES_MULTIPLE}" | ||
| if in_features % options.group_size: | ||
| return f"in features {in_features} not a multiple of the group size {options.group_size}" | ||
| out_multiple = PACKED_WORD_BITS // options.bits | ||
| if out_features % out_multiple: | ||
| return f"out features {out_features} not a multiple of {out_multiple}" | ||
| return None | ||
|
|
||
|
|
||
| def to_float16(name: str, tensor: Tensor) -> Tensor: | ||
| """`tensor` as F16, or unchanged when it is not F32 or BF16. Raises | ||
| ValueError when a value is beyond the f16 range.""" | ||
| if tensor.dtype not in {"F32", "BF16"}: | ||
| return tensor | ||
| values = to_float32(tensor) | ||
| if np.abs(values[np.isfinite(values)]).max(initial=0) > F16_MAX: | ||
| raise ValueError(f"{name} has values beyond the f16 range, run without --f16") | ||
| return from_array("F16", values.astype(np.float16)) | ||
|
|
||
|
|
||
| def quantize_checkpoint(source: Path, destination: Path, options: Options) -> list[str]: | ||
| """Quantizes the selected tensors of `source` into `destination` and | ||
| returns one report line per tensor, in file order.""" | ||
| checkpoint = read_checkpoint(source) | ||
| if QUANT_METADATA_KEY in checkpoint.metadata: | ||
| raise ValueError(f"{source} is already quantized, run on the float checkpoint") | ||
| tensors: dict[str, Tensor] = {} | ||
| report: list[str] = [] | ||
| quantized_count = 0 | ||
| for name, tensor in checkpoint.tensors.items(): | ||
| shape = f"[{', '.join(map(str, tensor.shape))}]" | ||
| dropped_by = next((pattern for pattern in options.drop if fnmatchcase(name, pattern)), None) | ||
| if dropped_by: | ||
| report.append(f"dropped {name} {shape}: matched by {dropped_by}") | ||
| continue | ||
| reason = skip_reason(name, tensor, options) | ||
| if reason: | ||
| tensors[name] = to_float16(name, tensor) if options.float16 else tensor | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Running the tool on an already quantized file gives a file runntime can't load: --bits 4 on an int8 file exits 0, but the old int8 weights stay and the header says int4. I guess refuse a source that's already quantized? |
||
| report.append(f"skipped {name} {shape}: {reason}") | ||
| continue | ||
| weight = to_float32(tensor) | ||
| quantized = quantize(weight, options.bits, options.group_size) | ||
| if options.bits == 8: | ||
| tensors[name] = from_array("I8", quantized.ints) | ||
| else: | ||
| tensors[name] = from_array("U8", pack_int4(quantized.ints)) | ||
| tensors[f"{name}_scale"] = from_array("F16", quantized.scales) | ||
| quantized_count += 1 | ||
| error = np.abs(dequantize(quantized) - weight) | ||
| report.append(f"quantized {name} {shape}: max error {error.max():.3g}, mean {error.mean():.3g}") | ||
| if quantized_count == 0: | ||
| raise ValueError("no tensor was quantized:\n" + "\n".join(report)) | ||
| setting = {"version": FORMAT_VERSION, "bits": options.bits, "groupSize": options.group_size, "scheme": "symmetric"} | ||
| metadata = checkpoint.metadata | {QUANT_METADATA_KEY: json.dumps(setting, separators=(",", ":"))} | ||
| write_safetensors(destination, tensors, metadata) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If the output path is the same as the input, the input gets wiped. Runntime quantize m.safetensors m.safetensors --bits 8 crashes and leaves an empty file. Maybe write to a temp file first and rename it at the end? |
||
| return report | ||
|
|
||
|
|
||
| def group_size(value: str) -> int: | ||
| """argparse type: a positive multiple of 4.""" | ||
| size = int(value) | ||
| if size <= 0 or size % 4: | ||
| raise argparse.ArgumentTypeError(f"{value} is not a positive multiple of 4") | ||
| return size | ||
|
|
||
|
|
||
| def parse_args(argv: Sequence[str] | None) -> argparse.Namespace: | ||
| parser = argparse.ArgumentParser(prog="runntime") | ||
| commands = parser.add_subparsers(dest="command", required=True) | ||
| command = commands.add_parser( | ||
| "quantize", | ||
| help="quantize a safetensors checkpoint to int8 or int4", | ||
| description="Quantizes the 2D weights of a safetensors checkpoint to runntime's int8/int4 format.", | ||
| ) | ||
| command.add_argument("source", type=Path, help="a .safetensors file or a Hugging Face model folder") | ||
| command.add_argument("destination", type=Path, help="the .safetensors file to write") | ||
| command.add_argument("--bits", type=int, choices=(4, 8), required=True) | ||
| command.add_argument("--group-size", type=group_size, default=32, help="inputs that share one scale (default: 32)") | ||
| command.add_argument( | ||
| "--include", | ||
| action="append", | ||
| metavar="GLOB", | ||
| help=f"tensor names to quantize, repeatable, replaces the default {' '.join(DEFAULT_INCLUDE)}", | ||
| ) | ||
| command.add_argument( | ||
| "--exclude", | ||
| action="append", | ||
| metavar="GLOB", | ||
| help=f"tensor names to keep as stored, repeatable, replaces the default {' '.join(DEFAULT_EXCLUDE)}", | ||
| ) | ||
| command.add_argument( | ||
| "--drop", | ||
| action="append", | ||
| default=[], | ||
| metavar="GLOB", | ||
| help="tensor names to leave out of the output, repeatable", | ||
| ) | ||
| command.add_argument("--f16", action="store_true", help="store the tensors that stay float as F16") | ||
| return parser.parse_args(argv) | ||
|
|
||
|
|
||
| def main(argv: Sequence[str] | None = None) -> int: | ||
| args = parse_args(argv) | ||
| options = Options( | ||
| bits=args.bits, | ||
| group_size=args.group_size, | ||
| include=args.include or DEFAULT_INCLUDE, | ||
| exclude=args.exclude or DEFAULT_EXCLUDE, | ||
| drop=args.drop, | ||
| float16=args.f16, | ||
| ) | ||
| try: | ||
| report = quantize_checkpoint(args.source, args.destination, options) | ||
| except ValueError as error: | ||
| print(f"runntime quantize: {error}", file=sys.stderr) | ||
| return 1 | ||
| print("\n".join(report)) | ||
| return 0 | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Don't we want to test this in CI too, on every change to tools/python?