Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4200
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 7000
decode_tokens_per_s: 140
registration:
last_mean_input_tps: 220
- count: 4
profile:
name: balanced
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4200
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120

degradation:
actions:
Expand Down
8 changes: 0 additions & 8 deletions src/libraries/rust/stargate/benches/bursty-8-backends.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4200
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 7200
decode_tokens_per_s: 150
registration:
last_mean_input_tps: 220
- count: 4
profile:
name: balanced
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4200
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
- count: 2
profile:
name: constrained
Expand All @@ -83,8 +77,6 @@ backends:
decode_jitter_ms: 10
prefill_tokens_per_s: 2200
decode_tokens_per_s: 50
registration:
last_mean_input_tps: 60

traffic_pattern:
kind: bursty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 3500
decode_tokens_per_s: 85
registration:
last_mean_input_tps: 100

traffic_pattern:
kind: zipf_hotset
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 7000
decode_tokens_per_s: 140
registration:
last_mean_input_tps: 220
- count: 4
profile:
name: balanced
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
- count: 2
profile:
name: slow
Expand All @@ -83,8 +77,6 @@ backends:
decode_jitter_ms: 8
prefill_tokens_per_s: 2200
decode_tokens_per_s: 55
registration:
last_mean_input_tps: 70

traffic_pattern:
kind: zipf_hotset
Expand Down
8 changes: 0 additions & 8 deletions src/libraries/rust/stargate/benches/hotset-8-backends.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 7000
decode_tokens_per_s: 140
registration:
last_mean_input_tps: 220
- count: 4
profile:
name: balanced
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
- count: 2
profile:
name: slow
Expand All @@ -83,8 +77,6 @@ backends:
decode_jitter_ms: 8
prefill_tokens_per_s: 2200
decode_tokens_per_s: 55
registration:
last_mean_input_tps: 70

traffic_pattern:
kind: zipf_hotset
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 7000
decode_tokens_per_s: 170
registration:
last_mean_input_tps: 160
profiles:
- count: 2
profile:
Expand All @@ -57,8 +55,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 12000
decode_tokens_per_s: 260
registration:
last_mean_input_tps: 280
- count: 4
profile:
name: balanced-cluster
Expand All @@ -71,8 +67,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 7000
decode_tokens_per_s: 170
registration:
last_mean_input_tps: 160
- count: 2
profile:
name: constrained-cluster
Expand All @@ -85,8 +79,6 @@ backends:
decode_jitter_ms: 7
prefill_tokens_per_s: 3400
decode_tokens_per_s: 85
registration:
last_mean_input_tps: 80

traffic_pattern:
kind: bursty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 8200
decode_tokens_per_s: 190
registration:
last_mean_input_tps: 185
profiles:
- count: 4
profile:
Expand All @@ -57,8 +55,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 14000
decode_tokens_per_s: 285
registration:
last_mean_input_tps: 310
- count: 8
profile:
name: balanced-cluster
Expand All @@ -71,8 +67,6 @@ backends:
decode_jitter_ms: 3
prefill_tokens_per_s: 8200
decode_tokens_per_s: 190
registration:
last_mean_input_tps: 185
- count: 4
profile:
name: constrained-cluster
Expand All @@ -85,8 +79,6 @@ backends:
decode_jitter_ms: 6
prefill_tokens_per_s: 4000
decode_tokens_per_s: 100
registration:
last_mean_input_tps: 90

traffic_pattern:
kind: zipf_hotset
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 32768
decode_tokens_per_s: 220
registration:
last_mean_input_tps: 32768
profiles:
- count: 2
profile:
Expand All @@ -57,8 +55,6 @@ backends:
decode_jitter_ms: 1
prefill_tokens_per_s: 65536
decode_tokens_per_s: 320
registration:
last_mean_input_tps: 65536
- count: 4
profile:
name: balanced-prefix-cluster
Expand All @@ -71,8 +67,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 32768
decode_tokens_per_s: 220
registration:
last_mean_input_tps: 32768
- count: 2
profile:
name: constrained-prefix-cluster
Expand All @@ -85,8 +79,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 16384
decode_tokens_per_s: 120
registration:
last_mean_input_tps: 16384

traffic_pattern:
kind: prefix_reuse
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 32768
decode_tokens_per_s: 220
registration:
last_mean_input_tps: 32768
profiles:
- count: 2
profile:
Expand All @@ -57,8 +55,6 @@ backends:
decode_jitter_ms: 1
prefill_tokens_per_s: 65536
decode_tokens_per_s: 320
registration:
last_mean_input_tps: 65536
- count: 4
profile:
name: balanced-prefix-cluster
Expand All @@ -71,8 +67,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 32768
decode_tokens_per_s: 220
registration:
last_mean_input_tps: 32768
- count: 2
profile:
name: constrained-prefix-cluster
Expand All @@ -85,8 +79,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 16384
decode_tokens_per_s: 120
registration:
last_mean_input_tps: 16384

traffic_pattern:
kind: prefix_reuse
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 1
prefill_tokens_per_s: 32768
decode_tokens_per_s: 512
registration:
last_mean_input_tps: 32768

traffic_pattern:
kind: prefix_reuse
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 9000
decode_tokens_per_s: 220
registration:
last_mean_input_tps: 200

traffic_pattern:
kind: uniform
Expand Down
8 changes: 0 additions & 8 deletions src/libraries/rust/stargate/benches/mixed-size-pulsar.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 8000
decode_tokens_per_s: 160
registration:
last_mean_input_tps: 260
- count: 3
profile:
name: balanced
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4000
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 120
- count: 1
profile:
name: small-cache
Expand All @@ -83,8 +77,6 @@ backends:
decode_jitter_ms: 6
prefill_tokens_per_s: 2600
decode_tokens_per_s: 70
registration:
last_mean_input_tps: 80

traffic_pattern:
kind: mixed_size
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 8
prefill_tokens_per_s: 1800
decode_tokens_per_s: 45
registration:
last_mean_input_tps: 60
profiles:
- count: 2
profile:
Expand All @@ -55,8 +53,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4200
decode_tokens_per_s: 90
registration:
last_mean_input_tps: 130
- count: 4
profile:
name: constrained
Expand All @@ -69,8 +65,6 @@ backends:
decode_jitter_ms: 8
prefill_tokens_per_s: 1800
decode_tokens_per_s: 45
registration:
last_mean_input_tps: 60

traffic_pattern:
kind: bursty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@

name: queue-mismatch-retry-ab
metadata:
description: A/B diagnostic for pylon rejection of stale Stargate queue estimates under burst load. The scenario runs 2048 requests per arm for benchmark evidence. Mock backends enforce real concurrent-service, prefill, and decode delays; benchmark pylons pin prompt throughput to the matching 2200 tokens/s profile value. Queue admission intentionally models prompt work only because output residency is unknowable at admission time.
description: A/B diagnostic for pylon rejection of stale Stargate queue estimates under burst load. The scenario runs 2048 requests per arm for benchmark evidence. Mock backends enforce real concurrent-service, prefill, and decode delays, and pylons measure the resulting prompt throughput. Queue admission intentionally models prompt work only because output residency is unknowable at admission time.
tags: [queue-admission, queue-mismatch, ab, wait-and-widen, diagnostic]
expected_runtime: medium
expected_signal: With the same fixed prompt-throughput estimate and real mock service delays in both arms, the enabled row should record pylon rejections and Stargate queue-mismatch retries; the disabled row should record disabled decisions without those retries.
expected_signal: With the same mock service delays in both arms, the enabled row should record pylon rejections and Stargate queue-mismatch retries; the disabled row should record disabled decisions without those retries.
model: dummy-model
seed: 154
request_count: 2048
Expand All @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 2
prefill_tokens_per_s: 2200
decode_tokens_per_s: 100
registration:
last_mean_input_tps: 2200

traffic_pattern:
kind: bursty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ backends:
decode_jitter_ms: 4
prefill_tokens_per_s: 4500
decode_tokens_per_s: 95
registration:
last_mean_input_tps: 130

traffic_pattern:
kind: stair_step
Expand Down
Loading
Loading