fix(proxy): setup headroom sidecar

pyproject.toml

@@ -72,7 +72,6 @@ proxy = [
     "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'",
     "pydantic-settings>=2.14.1,<3.0",
     "expression>=5.6.0,<6.0",
-    "headroom-ai>=0.33.0,<1.0",
 ]
 # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy
 # imports (fastapi, cryptography, ...) are all guarded, so it runs on the base

terraform/litellm/aws/config/prod/terraform.tfvars

@@ -217,8 +217,20 @@ proxy_config = {
       log_to_console               = false
     }
   },
+  guardrails = [
+    {
+      guardrail_name = "headroom-compression"
+      litellm_params = {
+        guardrail            = "headroom"
+        mode                 = "pre_call"
+        api_base             = "http://127.0.0.1:8787"
+        default_on           = false
+        unreachable_fallback = "fail_open"
+      }
+    }
+  ],
   litellm_settings = {
-    callbacks = ["smtp_email", "headroom.integrations.litellm_callback.HeadroomCallback"]
+    callbacks = ["smtp_email"]
     # Should solve https://github.com/BerriAI/litellm/issues/14194
     modify_params                      = true
     route_all_chat_openai_to_responses = true # Recommended
@@ -290,4 +302,3 @@ gateway_extra_secrets = {
   SLACK_WEBHOOK_URL = "arn:aws:secretsmanager:eu-central-1:751812493785:secret:data-reply/litellm/slack/webhook-reports-vXNOwY"
   OPENAI_API_KEY    = "arn:aws:secretsmanager:eu-central-1:751812493785:secret:data-reply/litellm/openai-api-key-nNpCF1"
 }
-

terraform/litellm/aws/ecs.tf

@@ -273,6 +273,8 @@ resource "aws_ecs_task_definition" "gateway" {
         image     = var.gateway_image
         essential = true
 
+        dependsOn = [{ containerName = "headroom", condition = "HEALTHY" }]
+
         portMappings = [{ containerPort = 4000, protocol = "tcp" }]
         environment = concat(
           local.shared_env,
@@ -297,7 +299,37 @@ resource "aws_ecs_task_definition" "gateway" {
         }
       },
       local.gateway_proxy_overrides,
-    )
+    ),
+    {
+      name      = "headroom"
+      image     = "ghcr.io/headroomlabs-ai/headroom@sha256:6c654cb0268a7699e68172fe3b43fda31a381576870fb2ed1691451a64a1bfe2"
+      essential = true
+
+      command = ["--host", "127.0.0.1", "--port", "8787", "--workers", "1"]
+
+      environment = [
+        { name = "HEADROOM_LOSSLESS", value = "true" },
+        { name = "HEADROOM_STATELESS", value = "true" },
+        { name = "HEADROOM_TELEMETRY", value = "off" },
+      ]
+
+      healthCheck = {
+        command     = ["CMD", "python3", "-c", "from urllib.request import urlopen; urlopen('http://127.0.0.1:8787/readyz', timeout=3)"]
+        interval    = 30
+        retries     = 3
+        startPeriod = 120
+        timeout     = 5
+      }
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.gateway.name
+          awslogs-region        = var.region
+          awslogs-stream-prefix = "headroom"
+        }
+      }
+    },
   ])
 
   tags = local.tags