diff --git a/packages/postgresql-pgml/release.sh b/packages/postgresql-pgml/release.sh index 9caa5947f..7b75f5a78 100644 --- a/packages/postgresql-pgml/release.sh +++ b/packages/postgresql-pgml/release.sh @@ -72,14 +72,14 @@ build_packages() { $(package_name ${pg} ${ubuntu_version} ${ARCH}) # Upload to S3 - deb-s3 upload \ - --visibility=public \ - --bucket apt.postgresml.org \ - $(package_name ${pg} ${ubuntu_version} ${ARCH}) \ - --codename ${codename} + #deb-s3 upload \ + # --visibility=public \ + # --bucket apt.postgresml.org \ + # $(package_name ${pg} ${ubuntu_version} ${ARCH}) \ + # --codename ${codename} # Clean up the package file - rm $(package_name ${pg} ${ubuntu_version} ${ARCH}) + #rm $(package_name ${pg} ${ubuntu_version} ${ARCH}) done } @@ -97,4 +97,4 @@ else for ubuntu_version in "${!ubuntu_versions[@]}"; do build_packages "$ubuntu_version" "${ubuntu_versions[$ubuntu_version]}" done -fi \ No newline at end of file +fi diff --git a/pgml-cms/blog/serverless-llms-are-dead-long-live-serverless-llms.md b/pgml-cms/blog/serverless-llms-are-dead-long-live-serverless-llms.md index a5d15d380..0b05121f7 100644 --- a/pgml-cms/blog/serverless-llms-are-dead-long-live-serverless-llms.md +++ b/pgml-cms/blog/serverless-llms-are-dead-long-live-serverless-llms.md @@ -64,9 +64,6 @@ Because we’ve curated the best in class models, they will always be instantly Your application can instantly burst usage to massive scale without a second thought, other than the aforementioned cost of GPU usage. Financial costs are now the limiting factor, but we have an additional new lever to optimize costs even further. -### Multi-tenant continuous batching -It’s not just loading the model weights into GPU RAM the first time that’s expensive. Streaming those weights from GPU RAM to the CUDA cores for each request is actually the bottleneck for most LLM applications. Continuous batching allows us to reuse a single layer of weights for multiple different queries at the same time, further reducing costs, without significantly impacting overall latency. Thanks to vLLM team for [this impressive breakthrough](https://arxiv.org/abs/2309.06180) in performance. - ### Simplified pricing Compared to using a host of services to provide comparable functionality, our pricing is significantly simpler. We charge for: diff --git a/pgml-extension/requirements.amd64.txt b/pgml-extension/requirements.amd64.txt index 4a52d3150..40d2e35ef 100644 --- a/pgml-extension/requirements.amd64.txt +++ b/pgml-extension/requirements.amd64.txt @@ -1,180 +1,320 @@ -accelerate==1.2.1 -aiohappyeyeballs==2.4.4 -aiohttp==3.11.11 -aiohttp-cors==0.7.0 -aiosignal==1.3.2 -airportsdata==20241001 +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile +# +--extra-index-url https://download.pytorch.org/whl/cpu + +accelerate==1.10.1 + # via + # -r requirements.in + # peft + # trl +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.12.15 + # via fsspec +aiosignal==1.4.0 + # via aiohttp annotated-types==0.7.0 -anyio==4.8.0 -astor==0.8.1 -attrs==24.3.0 -auto_gptq==0.7.1 -bitsandbytes==0.45.0 -blake3==1.0.2 -cachetools==5.5.0 -catboost==1.2.7 -certifi==2024.12.14 -charset-normalizer==3.4.1 -click==8.1.8 -cloudpickle==3.1.1 + # via pydantic +anyio==4.10.0 + # via httpx +attrs==25.3.0 + # via aiohttp +catboost==1.2.8 + # via -r requirements.in +certifi==2025.8.3 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.3 + # via requests +click==8.2.1 + # via sacremoses colorama==0.4.6 -coloredlogs==15.0.1 -colorful==0.5.6 -compressed-tensors==0.8.1 -contourpy==1.3.1 + # via sacrebleu +contourpy==1.3.3 + # via matplotlib ctransformers==0.2.27 + # via -r requirements.in cycler==0.12.1 -datasets==3.2.0 -deepspeed==0.16.2 -depyf==0.18.0 + # via matplotlib +datasets==4.0.0 + # via + # -r requirements.in + # evaluate + # trl dill==0.3.8 -diskcache==5.6.3 -distlib==0.3.9 -distro==1.9.0 -einops==0.8.0 -evaluate==0.4.3 -fastapi==0.115.6 -filelock==3.16.1 -fonttools==4.55.3 -frozenlist==1.5.0 -fsspec==2024.9.0 -gekko==1.2.1 -gguf==0.10.0 -google-api-core==2.24.0 -google-auth==2.37.0 -googleapis-common-protos==1.66.0 -graphviz==0.20.3 -greenlet==3.1.1 -grpcio==1.69.0 -h11==0.14.0 -hjson==3.1.0 -httpcore==1.0.7 -httptools==0.6.4 + # via + # datasets + # evaluate + # multiprocess +einops==0.8.1 + # via -r requirements.in +evaluate==0.4.5 + # via -r requirements.in +filelock==3.19.1 + # via + # datasets + # huggingface-hub + # torch + # transformers +fonttools==4.59.2 + # via matplotlib +frozenlist==1.7.0 + # via + # aiohttp + # aiosignal +fsspec[http]==2025.3.0 + # via + # datasets + # evaluate + # huggingface-hub + # torch +graphviz==0.21 + # via catboost +greenlet==3.2.4 + # via sqlalchemy +h11==0.16.0 + # via httpcore +hf-xet==1.1.9 + # via huggingface-hub +httpcore==1.0.9 + # via httpx httpx==0.28.1 -huggingface-hub==0.27.1 -humanfriendly==10.0 + # via langsmith +huggingface-hub==0.34.4 + # via + # -r requirements.in + # accelerate + # ctransformers + # datasets + # evaluate + # optimum + # peft + # sentence-transformers + # tokenizers + # transformers idna==3.10 -importlib_metadata==8.5.0 -iniconfig==2.0.0 -interegular==0.3.3 -Jinja2==3.1.5 -jiter==0.8.2 -joblib==1.4.2 + # via + # anyio + # httpx + # requests + # yarl +jinja2==3.1.6 + # via torch +joblib==1.5.2 + # via + # sacremoses + # scikit-learn jsonpatch==1.33 + # via langchain-core jsonpointer==3.0.0 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -kiwisolver==1.4.8 -langchain==0.3.14 -langchain-core==0.3.29 -langchain-text-splitters==0.3.5 -langsmith==0.2.10 -lark==1.2.2 -lightgbm==4.5.0 -linkify-it-py==2.0.3 -lm-format-enforcer==0.10.9 -lxml==5.3.0 -markdown-it-py==3.0.0 -MarkupSafe==3.0.2 -matplotlib==3.10.0 -mdit-py-plugins==0.4.2 -mdurl==0.1.2 -memray==1.15.0 -mistral_common==1.5.1 + # via jsonpatch +kiwisolver==1.4.9 + # via matplotlib +langchain==0.3.27 + # via -r requirements.in +langchain-core==0.3.75 + # via + # langchain + # langchain-text-splitters +langchain-text-splitters==0.3.11 + # via langchain +langsmith==0.4.25 + # via + # langchain + # langchain-core +lightgbm==4.6.0 + # via -r requirements.in +lxml==6.0.1 + # via sacrebleu +markupsafe==3.0.2 + # via jinja2 +matplotlib==3.10.6 + # via catboost mpmath==1.3.0 -msgpack==1.1.0 -msgspec==0.19.0 -multidict==6.1.0 + # via sympy +multidict==6.6.4 + # via + # aiohttp + # yarl multiprocess==0.70.16 -nest-asyncio==1.6.0 -networkx==3.4.2 -ninja==1.11.1.3 -numpy==1.26.4 -nvidia-cublas-cu12==12.4.5.8 -nvidia-cuda-cupti-cu12==12.4.127 -nvidia-cuda-nvrtc-cu12==12.4.127 -nvidia-cuda-runtime-cu12==12.4.127 -nvidia-cudnn-cu12==9.1.0.70 -nvidia-cufft-cu12==11.2.1.3 -nvidia-curand-cu12==10.3.5.147 -nvidia-cusolver-cu12==11.6.1.9 -nvidia-cusparse-cu12==12.3.1.170 -nvidia-ml-py==12.560.30 -nvidia-nccl-cu12==2.21.5 -nvidia-nvjitlink-cu12==12.4.127 -nvidia-nvtx-cu12==12.4.127 -openai==1.59.7 -opencensus==0.11.4 -opencensus-context==0.1.3 -opencv-python-headless==4.10.0.84 -optimum==1.23.3 -orjson==3.10.14 -outlines==0.1.11 -outlines_core==0.1.26 -packaging==24.2 -pandas==2.2.3 -partial-json-parser==0.2.1.1.post5 -peft==0.14.0 -pillow==10.4.0 -platformdirs==4.3.6 -plotly==5.24.1 -pluggy==1.5.0 -portalocker==3.1.1 -prometheus-fastapi-instrumentator==7.0.2 -prometheus_client==0.21.1 -propcache==0.2.1 -proto-plus==1.25.0 -protobuf==5.29.3 -psutil==6.1.1 + # via + # datasets + # evaluate +narwhals==2.3.0 + # via plotly +networkx==3.5 + # via torch +numpy==2.3.2 + # via + # accelerate + # catboost + # contourpy + # datasets + # evaluate + # lightgbm + # matplotlib + # optimum + # pandas + # peft + # sacrebleu + # scikit-learn + # scipy + # torchvision + # transformers + # xgboost +nvidia-nccl-cu12==2.27.7 + # via xgboost +optimum==1.27.0 + # via -r requirements.in +orjson==3.11.3 + # via + # -r requirements.in + # langsmith +packaging==25.0 + # via + # accelerate + # datasets + # evaluate + # huggingface-hub + # langchain-core + # langsmith + # matplotlib + # optimum + # peft + # plotly + # transformers +pandas==2.3.2 + # via + # catboost + # datasets + # evaluate +peft==0.17.1 + # via -r requirements.in +pillow==11.3.0 + # via + # matplotlib + # sentence-transformers + # torchvision +plotly==6.3.0 + # via catboost +portalocker==3.2.0 + # via sacrebleu +propcache==0.3.2 + # via + # aiohttp + # yarl +psutil==7.0.0 + # via + # accelerate + # peft py-cpuinfo==9.0.0 -py-spy==0.4.0 -pyarrow==18.1.0 -pyasn1==0.6.1 -pyasn1_modules==0.4.1 -pybind11==2.13.6 -pycountry==24.6.1 -pydantic==2.10.5 -pydantic_core==2.27.2 -Pygments==2.19.1 -pyparsing==3.2.1 -pytest==8.3.4 + # via ctransformers +pyarrow==21.0.0 + # via datasets +pydantic==2.11.7 + # via + # langchain + # langchain-core + # langsmith +pydantic-core==2.33.2 + # via pydantic +pyparsing==3.2.3 + # via matplotlib python-dateutil==2.9.0.post0 -python-dotenv==1.0.1 -pytz==2024.2 -PyYAML==6.0.2 -pyzmq==26.2.0 -ray==2.40.0 -referencing==0.35.1 -regex==2024.11.6 -requests==2.32.3 + # via + # matplotlib + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # accelerate + # datasets + # huggingface-hub + # langchain + # langchain-core + # peft + # transformers +regex==2025.9.1 + # via + # sacrebleu + # sacremoses + # transformers +requests==2.32.5 + # via + # datasets + # evaluate + # huggingface-hub + # langchain + # langsmith + # requests-toolbelt + # transformers requests-toolbelt==1.0.0 -rich==13.9.4 + # via langsmith rouge==1.0.1 -rpds-py==0.22.3 -rsa==4.9 + # via -r requirements.in sacrebleu==2.5.1 + # via -r requirements.in sacremoses==0.1.1 -safetensors==0.5.2 -scikit-learn==1.6.1 -scipy==1.15.1 -sentence-transformers==3.3.1 -sentencepiece==0.2.0 + # via -r requirements.in +safetensors==0.6.2 + # via + # accelerate + # peft + # transformers +scikit-learn==1.7.1 + # via sentence-transformers +scipy==1.16.1 + # via + # catboost + # lightgbm + # scikit-learn + # sentence-transformers + # xgboost +sentence-transformers==5.1.0 + # via -r requirements.in six==1.17.0 -smart-open==7.1.0 + # via + # catboost + # python-dateutil + # rouge sniffio==1.3.1 -SQLAlchemy==2.0.37 -starlette==0.41.3 -sympy==1.13.1 + # via anyio +sqlalchemy==2.0.43 + # via langchain +sympy==1.14.0 + # via torch tabulate==0.9.0 -tenacity==9.0.0 -textual==1.0.0 -threadpoolctl==3.5.0 -tiktoken==0.7.0 -tokenizers==0.21.0 -torch==2.5.1 -torchaudio==2.5.1 -torchvision==0.20.1 + # via sacrebleu +tenacity==9.1.2 + # via langchain-core +threadpoolctl==3.6.0 + # via scikit-learn +tokenizers==0.22.0 + # via + # -r requirements.in + # transformers +torch==2.8.0+cpu + # via + # -r requirements.in + # accelerate + # optimum + # peft + # sentence-transformers + # torchaudio + # torchvision +torchaudio==2.8.0+cpu + # via -r requirements.in +torchvision==0.23.0+cpu + # via -r requirements.in tqdm==4.67.1 +<<<<<<< Updated upstream transformers==4.48.0 transformers-stream-generator==0.0.5 triton==3.1.0 @@ -186,13 +326,59 @@ urllib3==2.3.0 uvicorn==0.34.0 uvloop==0.21.0 virtualenv==20.28.1 -vllm==0.6.6.post1 watchfiles==1.0.4 websockets==14.1 wrapt==1.17.2 xformers==0.0.28.post3 xgboost==2.1.3 xgrammar==0.1.9 +======= + # via + # datasets + # evaluate + # huggingface-hub + # peft + # sacremoses + # sentence-transformers + # transformers +transformers==4.56.1 + # via + # -r requirements.in + # optimum + # peft + # sentence-transformers + # trl +trl==0.22.2 + # via -r requirements.in +typing-extensions==4.15.0 + # via + # aiosignal + # anyio + # huggingface-hub + # langchain-core + # pydantic + # pydantic-core + # sentence-transformers + # sqlalchemy + # torch + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +tzdata==2025.2 + # via pandas +urllib3==2.5.0 + # via requests +xgboost==3.0.4 + # via -r requirements.in +>>>>>>> Stashed changes xxhash==3.5.0 -yarl==1.18.3 -zipp==3.21.0 + # via + # datasets + # evaluate +yarl==1.20.1 + # via aiohttp +zstandard==0.24.0 + # via langsmith + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/pgml-extension/requirements.in b/pgml-extension/requirements.in new file mode 100644 index 000000000..fd5933a21 --- /dev/null +++ b/pgml-extension/requirements.in @@ -0,0 +1,34 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +catboost +lightgbm +torch +torchvision +torchaudio +xgboost + +# Transformers +accelerate +ctransformers +huggingface-hub +einops +optimum +peft +tokenizers +transformers + +# Embeddings +sentence-transformers + +# Ratings +rouge +sacrebleu +sacremoses +evaluate +trl + +# Utils +datasets +orjson +langchain +evaluate +trl diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt index 4a52d3150..46d803132 100644 --- a/pgml-extension/requirements.linux.txt +++ b/pgml-extension/requirements.linux.txt @@ -186,7 +186,6 @@ urllib3==2.3.0 uvicorn==0.34.0 uvloop==0.21.0 virtualenv==20.28.1 -vllm==0.6.6.post1 watchfiles==1.0.4 websockets==14.1 wrapt==1.17.2 diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index 2830711ec..08eee0ceb 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,11 +1,366 @@ -# Use a Python 3.11 virtualenv with PostgresML: -# $ virtualenv -p python3.11 pgml-venv -# $ source pgml-vev/bin/activate -# $ pip install -r requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile +# +--extra-index-url https://download.pytorch.org/whl/cpu -# Python 3.12 can't resolve all dependencies: -# catboost and autogptq fail to build +accelerate==1.10.1 + # via + # -r requirements.in + # peft + # trl +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.12.15 + # via fsspec +aiosignal==1.4.0 + # via aiohttp +annotated-types==0.7.0 + # via pydantic +anyio==4.10.0 + # via httpx +attrs==25.3.0 + # via aiohttp +catboost==1.2.8 + # via -r requirements.in +certifi==2025.8.3 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.3 + # via requests +click==8.2.1 + # via sacremoses +colorama==0.4.6 + # via sacrebleu +contourpy==1.3.3 + # via matplotlib +ctransformers==0.2.27 + # via -r requirements.in +cycler==0.12.1 + # via matplotlib +datasets==4.0.0 + # via + # -r requirements.in + # evaluate + # trl +dill==0.3.8 + # via + # datasets + # evaluate + # multiprocess +einops==0.8.1 + # via -r requirements.in +evaluate==0.4.5 + # via -r requirements.in +filelock==3.19.1 + # via + # datasets + # huggingface-hub + # torch + # transformers +fonttools==4.59.2 + # via matplotlib +frozenlist==1.7.0 + # via + # aiohttp + # aiosignal +fsspec[http]==2025.3.0 + # via + # datasets + # evaluate + # huggingface-hub + # torch +graphviz==0.21 + # via catboost +greenlet==3.2.4 + # via sqlalchemy +h11==0.16.0 + # via httpcore +hf-xet==1.1.9 + # via huggingface-hub +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via langsmith +huggingface-hub==0.34.4 + # via + # -r requirements.in + # accelerate + # ctransformers + # datasets + # evaluate + # optimum + # peft + # sentence-transformers + # tokenizers + # transformers +idna==3.10 + # via + # anyio + # httpx + # requests + # yarl +jinja2==3.1.6 + # via torch +joblib==1.5.2 + # via + # sacremoses + # scikit-learn +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +kiwisolver==1.4.9 + # via matplotlib +langchain==0.3.27 + # via -r requirements.in +langchain-core==0.3.75 + # via + # langchain + # langchain-text-splitters +langchain-text-splitters==0.3.11 + # via langchain +langsmith==0.4.25 + # via + # langchain + # langchain-core +lightgbm==4.6.0 + # via -r requirements.in +lxml==6.0.1 + # via sacrebleu +markupsafe==3.0.2 + # via jinja2 +matplotlib==3.10.6 + # via catboost +mpmath==1.3.0 + # via sympy +multidict==6.6.4 + # via + # aiohttp + # yarl +multiprocess==0.70.16 + # via + # datasets + # evaluate +narwhals==2.3.0 + # via plotly +networkx==3.5 + # via torch +numpy==2.3.2 + # via + # accelerate + # catboost + # contourpy + # datasets + # evaluate + # lightgbm + # matplotlib + # optimum + # pandas + # peft + # sacrebleu + # scikit-learn + # scipy + # torchvision + # transformers + # xgboost +nvidia-nccl-cu12==2.27.7 + # via xgboost +optimum==1.27.0 + # via -r requirements.in +orjson==3.11.3 + # via + # -r requirements.in + # langsmith +packaging==25.0 + # via + # accelerate + # datasets + # evaluate + # huggingface-hub + # langchain-core + # langsmith + # matplotlib + # optimum + # peft + # plotly + # transformers +pandas==2.3.2 + # via + # catboost + # datasets + # evaluate +peft==0.17.1 + # via -r requirements.in +pillow==11.3.0 + # via + # matplotlib + # sentence-transformers + # torchvision +plotly==6.3.0 + # via catboost +portalocker==3.2.0 + # via sacrebleu +propcache==0.3.2 + # via + # aiohttp + # yarl +psutil==7.0.0 + # via + # accelerate + # peft +py-cpuinfo==9.0.0 + # via ctransformers +pyarrow==21.0.0 + # via datasets +pydantic==2.11.7 + # via + # langchain + # langchain-core + # langsmith +pydantic-core==2.33.2 + # via pydantic +pyparsing==3.2.3 + # via matplotlib +python-dateutil==2.9.0.post0 + # via + # matplotlib + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # accelerate + # datasets + # huggingface-hub + # langchain + # langchain-core + # peft + # transformers +regex==2025.9.1 + # via + # sacrebleu + # sacremoses + # transformers +requests==2.32.5 + # via + # datasets + # evaluate + # huggingface-hub + # langchain + # langsmith + # requests-toolbelt + # transformers +requests-toolbelt==1.0.0 + # via langsmith +rouge==1.0.1 + # via -r requirements.in +sacrebleu==2.5.1 + # via -r requirements.in +sacremoses==0.1.1 + # via -r requirements.in +safetensors==0.6.2 + # via + # accelerate + # peft + # transformers +scikit-learn==1.7.1 + # via sentence-transformers +scipy==1.16.1 + # via + # catboost + # lightgbm + # scikit-learn + # sentence-transformers + # xgboost +sentence-transformers==5.1.0 + # via -r requirements.in +six==1.17.0 + # via + # catboost + # python-dateutil + # rouge +sniffio==1.3.1 + # via anyio +sqlalchemy==2.0.43 + # via langchain +sympy==1.14.0 + # via torch +tabulate==0.9.0 + # via sacrebleu +tenacity==9.1.2 + # via langchain-core +threadpoolctl==3.6.0 + # via scikit-learn +tokenizers==0.22.0 + # via + # -r requirements.in + # transformers +torch==2.8.0+cpu + # via + # -r requirements.in + # accelerate + # optimum + # peft + # sentence-transformers + # torchaudio + # torchvision +torchaudio==2.8.0+cpu + # via -r requirements.in +torchvision==0.23.0+cpu + # via -r requirements.in +tqdm==4.67.1 + # via + # datasets + # evaluate + # huggingface-hub + # peft + # sacremoses + # sentence-transformers + # transformers +transformers==4.56.1 + # via + # -r requirements.in + # optimum + # peft + # sentence-transformers + # trl +trl==0.22.2 + # via -r requirements.in +typing-extensions==4.15.0 + # via + # aiosignal + # anyio + # huggingface-hub + # langchain-core + # pydantic + # pydantic-core + # sentence-transformers + # sqlalchemy + # torch + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +tzdata==2025.2 + # via pandas +urllib3==2.5.0 + # via requests +xgboost==3.0.4 + # via -r requirements.in +xxhash==3.5.0 + # via + # datasets + # evaluate +yarl==1.20.1 + # via aiohttp +zstandard==0.24.0 + # via langsmith +<<<<<<< Updated upstream # Only the immediate dependencies of PostgresML are maintained here. # See requirements.linux.txt or requirements.macos.txt for complete and locked versions. # If you update this file, make sure to pip freeze into platform specific dependencies @@ -33,7 +388,7 @@ tokenizers transformers transformers-stream-generator xformers; sys_platform == 'linux' # only runs on nvidia hardware -vllm; sys_platform == 'linux' # only runs on linux +#vllm; sys_platform == 'linux' # only runs on linux # Embeddings sentence-transformers @@ -51,3 +406,7 @@ orjson langchain evaluate trl +======= +# The following packages are considered to be unsafe in a requirements file: +# setuptools +>>>>>>> Stashed changes