From c5ff8fd134d442a10a58a63573a27ec7231756ac Mon Sep 17 00:00:00 2001 From: Yannik Zbick Date: Thu, 20 Aug 2026 15:52:10 +0200 Subject: [PATCH 1/2] feat(proxy): setup headroom sidecar --- pyproject.toml | 1 - .../litellm/aws/config/prod/terraform.tfvars | 15 ++++++- terraform/litellm/aws/ecs.tf | 34 ++++++++++++++- uv.lock | 42 ------------------- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index acde14552765..e15bf1351dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,6 @@ proxy = [ "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "pydantic-settings>=2.14.1,<3.0", "expression>=5.6.0,<6.0", - "headroom-ai>=0.33.0,<1.0", ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports (fastapi, cryptography, ...) are all guarded, so it runs on the base diff --git a/terraform/litellm/aws/config/prod/terraform.tfvars b/terraform/litellm/aws/config/prod/terraform.tfvars index a68eea250f72..62d3b9baf554 100644 --- a/terraform/litellm/aws/config/prod/terraform.tfvars +++ b/terraform/litellm/aws/config/prod/terraform.tfvars @@ -217,8 +217,20 @@ proxy_config = { log_to_console = false } }, + guardrails = [ + { + guardrail_name = "headroom-compression" + litellm_params = { + guardrail = "headroom" + mode = "pre_call" + api_base = "http://127.0.0.1:8787" + default_on = false + unreachable_fallback = "fail_open" + } + } + ], litellm_settings = { - callbacks = ["smtp_email", "headroom.integrations.litellm_callback.HeadroomCallback"] + callbacks = ["smtp_email"] # Should solve https://github.com/BerriAI/litellm/issues/14194 modify_params = true route_all_chat_openai_to_responses = true # Recommended @@ -290,4 +302,3 @@ gateway_extra_secrets = { SLACK_WEBHOOK_URL = "arn:aws:secretsmanager:eu-central-1:751812493785:secret:data-reply/litellm/slack/webhook-reports-vXNOwY" OPENAI_API_KEY = "arn:aws:secretsmanager:eu-central-1:751812493785:secret:data-reply/litellm/openai-api-key-nNpCF1" } - diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 6a4d7f284e97..399914fb8570 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -273,6 +273,8 @@ resource "aws_ecs_task_definition" "gateway" { image = var.gateway_image essential = true + dependsOn = [{ containerName = "headroom", condition = "HEALTHY" }] + portMappings = [{ containerPort = 4000, protocol = "tcp" }] environment = concat( local.shared_env, @@ -297,7 +299,37 @@ resource "aws_ecs_task_definition" "gateway" { } }, local.gateway_proxy_overrides, - ) + ), + { + name = "headroom" + image = "ghcr.io/headroomlabs-ai/headroom@sha256:6b34905489e3a5e4458cd95d2db2b1ac84fba64429c06fc653eb31e59376b079" + essential = true + + command = ["--host", "127.0.0.1", "--port", "8787", "--workers", "1"] + + environment = [ + { name = "HEADROOM_LOSSLESS", value = "true" }, + { name = "HEADROOM_STATELESS", value = "true" }, + { name = "HEADROOM_TELEMETRY", value = "off" }, + ] + + healthCheck = { + command = ["CMD", "python3", "-c", "from urllib.request import urlopen; urlopen('http://127.0.0.1:8787/readyz', timeout=3)"] + interval = 30 + retries = 3 + startPeriod = 120 + timeout = 5 + } + + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.gateway.name + awslogs-region = var.region + awslogs-stream-prefix = "headroom" + } + } + }, ]) tags = local.tags diff --git a/uv.lock b/uv.lock index d2728b56e191..b781e540a394 100644 --- a/uv.lock +++ b/uv.lock @@ -413,21 +413,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/00/24d6c9d60e4f2036cee7cc28fb73a7ab5e71c0171201586c9fbc383021ea/assemblyai-0.52.4-py3-none-any.whl", hash = "sha256:4d358c9cc4c6685cde296ccfa0a544417272a2b74a248e92273db7af6ea60901", size = 53900, upload-time = "2026-02-25T15:30:21.337Z" }, ] -[[package]] -name = "ast-grep-cli" -version = "0.45.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/c3/a4f4dba7f601d096e69539a3e569346ab05a5d0f3177fa971526b410a012/ast_grep_cli-0.45.0.tar.gz", hash = "sha256:d07bfe87c09697e5011b2ce065f61cbc165ca5b0c0a567000f4ce9e62f2fde32", size = 279290, upload-time = "2026-07-23T16:18:32.824Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/20/abe4ec0cd4f7e1b32c495fc0e4a37243c26c09a1d22959d20311c6894644/ast_grep_cli-0.45.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2fa9b424b0cf82770154cfdfd9aafe1eacd4598073e63c8482962df008c99b41", size = 16018089, upload-time = "2026-07-23T16:18:19.242Z" }, - { url = "https://files.pythonhosted.org/packages/16/a8/1273fd3a4f7a30d45fbcb0ff051be24e7ee31bfacbd0bb48ea7a8a9b46ef/ast_grep_cli-0.45.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a39b1674c8686bf21e252da59d82ec8d5fea543f5c934734f596ddf1c0552961", size = 8006267, upload-time = "2026-07-23T16:18:21.705Z" }, - { url = "https://files.pythonhosted.org/packages/7f/da/e6db92897e2f2ffca80604d422f2cd837f7375be5adfb90add30fa94244d/ast_grep_cli-0.45.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:99c908930e181bcdf4d0e9b8bb4b3e52ee6403f59ff6fc16ba8054bcff90dda2", size = 7865207, upload-time = "2026-07-23T16:18:23.192Z" }, - { url = "https://files.pythonhosted.org/packages/58/ca/c6140ca63d6a117d83296d46c3052facf893bde899d8f734ee08b9e8864e/ast_grep_cli-0.45.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:a1e9e83e15e7f2290e51770b0629618b68b3fb628396f86648d99015782b7374", size = 8161265, upload-time = "2026-07-23T16:18:24.644Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/9073558b1cdf5c735d59695b364f2c8fa03235ee089929a69cad123e7e3c/ast_grep_cli-0.45.0-py3-none-win32.whl", hash = "sha256:609e77bfb4a61c047d23c081b5bf862b1e05c9d06e7d26b8e4df4879d508c973", size = 15197816, upload-time = "2026-07-23T16:18:26.292Z" }, - { url = "https://files.pythonhosted.org/packages/45/f7/03d920cb9c1e850b55b60732ae0b9d9080218b4a5620d1499a77e613446a/ast_grep_cli-0.45.0-py3-none-win_amd64.whl", hash = "sha256:63026862481b2f480792498c8c1bad13f09fccd8fa34ae9913c5743eb3165150", size = 16175179, upload-time = "2026-07-23T16:18:28.622Z" }, - { url = "https://files.pythonhosted.org/packages/64/28/dde9d9c2435faf43b596cc22a6a39ae9a0f952560ab6917abb201d5d15bb/ast_grep_cli-0.45.0-py3-none-win_arm64.whl", hash = "sha256:5c3aea8e2a06938350f342f0c5be4399a4fe05e12ba80a5c037313e28bc18b00", size = 15554486, upload-time = "2026-07-23T16:18:30.887Z" }, -] - [[package]] name = "astroid" version = "4.0.4" @@ -3147,31 +3132,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, ] -[[package]] -name = "headroom-ai" -version = "0.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ast-grep-cli" }, - { name = "click" }, - { name = "litellm", marker = "python_full_version < '3.14'" }, - { name = "opentelemetry-api" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "rich" }, - { name = "tiktoken" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "tomlkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/2c/d3aeeb62d8f61430c9cf5b84c1bd0227362e43eaaaf710d6bb1759fec153/headroom_ai-0.33.0.tar.gz", hash = "sha256:97d817e5923903d72bed24f75e0424e9cb7f86b3ddde0fc1acec4f3f85deeb5a", size = 2462330, upload-time = "2026-07-29T23:24:15.245Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/99/06bb6ef86c7bce575aa2e78b1fdd848ebeeb6e2a718c0dbbf41f93f60e33/headroom_ai-0.33.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:41bfc3daf7f1088e9b0b356533b3b88d71b11ea84fa49652344e01a8f77e7642", size = 13244384, upload-time = "2026-07-29T23:24:03.401Z" }, - { url = "https://files.pythonhosted.org/packages/4a/85/dbe382e0166b24dddba88f49664a2af31af085dd80e611b281f673a73c00/headroom_ai-0.33.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd4402d4d29356e76393822f7d163ad32b92134dcc93bd1007d39833de8f6265", size = 13232213, upload-time = "2026-07-29T23:24:05.901Z" }, - { url = "https://files.pythonhosted.org/packages/44/9d/2a52d1f9f278c1092853d2db17bb597040575a8c92abb7116be5f5bdb819/headroom_ai-0.33.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:798cd88ada03541affe4733b7301891aa10dadeb2f80a05b75f7c73efc0e9cc8", size = 13257768, upload-time = "2026-07-29T23:24:08.085Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/cf1e7bfd84d9acac2941ebdc90dbb6c9bc114b4e36da2e5714662b5cd319/headroom_ai-0.33.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e4a4b9aaa6b115452c77b29954a4b357fc0bf242810f02aa92536a3f0f235079", size = 13393046, upload-time = "2026-07-29T23:24:10.291Z" }, - { url = "https://files.pythonhosted.org/packages/53/aa/e9c716cc48523f68d8390a4548bb0f2700a7bf2d920d389437fdf5b44187/headroom_ai-0.33.0-cp310-abi3-win_amd64.whl", hash = "sha256:30669b6109a645efab95819685056ec74efc3096e7b6de4feee46c6a6790cb91", size = 13339743, upload-time = "2026-07-29T23:24:12.903Z" }, -] - [[package]] name = "hf-xet" version = "1.5.0" @@ -4223,7 +4183,6 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, - { name = "headroom-ai" }, { name = "inquirerpy" }, { name = "litellm-enterprise" }, { name = "litellm-proxy-extras" }, @@ -4406,7 +4365,6 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, - { name = "headroom-ai", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, From b9f9f3e02381156b8605aeef38bbfa5b8fb1b5c2 Mon Sep 17 00:00:00 2001 From: Yannik Zbick Date: Fri, 21 Aug 2026 14:40:27 +0200 Subject: [PATCH 2/2] fix(proxy): setup headroom sidecar --- terraform/litellm/aws/ecs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 399914fb8570..3cdbdb694287 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -302,7 +302,7 @@ resource "aws_ecs_task_definition" "gateway" { ), { name = "headroom" - image = "ghcr.io/headroomlabs-ai/headroom@sha256:6b34905489e3a5e4458cd95d2db2b1ac84fba64429c06fc653eb31e59376b079" + image = "ghcr.io/headroomlabs-ai/headroom@sha256:6c654cb0268a7699e68172fe3b43fda31a381576870fb2ed1691451a64a1bfe2" essential = true command = ["--host", "127.0.0.1", "--port", "8787", "--workers", "1"]