tinybpe/setup.py at main · neluca/tinybpe · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
"""Minimal setup.py — only defines the C extension.

All package metadata lives in pyproject.toml.
"""

import sys
from setuptools import Extension, setup

ext_modules = [
    Extension(
        "tinybpe.bpe",
        sources=[
            "src/bpe_module.c",
            "src/_tree_core.c",
            "src/bpe_common.c",
            "src/bpe_trainer.c",
            "src/bpe_tokenizer.c",
        ],
        depends=[
            "src/_tree_core.h",
            "src/bpe_common.h",
            "src/bpe_trainer.h",
            "src/bpe_tokenizer.h",
        ],
        # NB: on 64-bit Windows, sys.platform is "win32" (historical).
        # MSVC uses /W* flags instead of -W*, so pass nothing here.
        extra_compile_args={
            "win32": [],
        }.get(sys.platform, ["-Wall", "-Wextra", "-std=c99"]),
    )
]

setup(
    ext_modules=ext_modules,
)